diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 8be43bd8e350fae05244ead4d752759268eb81aa..cf5c5c5637c26d76f181a626990ad12e65f3ffbe 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -51,7 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" ) // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -78,8 +78,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) @@ -964,18 +963,6 @@ typedef struct { char subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; // include subqueries' index, Obj IDs and states(C-complete/I-imcomplete) } SQueryDesc; -typedef struct { - char sql[TSDB_SHOW_SQL_LEN]; - char dstTable[TSDB_TABLE_NAME_LEN]; - int32_t streamId; - int64_t num; // number of computing/cycles - int64_t useconds; - int64_t ctime; - int64_t stime; - int64_t slidingTime; - int64_t interval; -} SStreamDesc; - typedef struct { int32_t connId; int32_t pid; @@ -1112,6 +1099,7 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSchedulerQueryMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; uint32_t contentLen; @@ -1119,15 +1107,39 @@ typedef struct SSchedulerQueryMsg { } SSchedulerQueryMsg; typedef struct SSchedulerReadyMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerReadyMsg; typedef struct SSchedulerFetchMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerFetchMsg; +typedef struct SSchedulerStatusMsg { + uint64_t schedulerId; +} SSchedulerStatusMsg; + +typedef struct STaskStatus { + uint64_t queryId; + uint64_t taskId; + int8_t status; +} STaskStatus; + +typedef struct SSchedulerStatusRsp { + uint32_t num; + STaskStatus status[]; +} SSchedulerStatusRsp; + + +typedef struct SSchedulerCancelMsg { + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; +} SSchedulerCancelMsg; + #pragma pack(pop) diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 65779b209938bc9766f2d0d11881c2354d0c5092..aa4c3af392dfb2e94198d0062801fe405624752c 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif - +#include "trpc.h" typedef struct { uint64_t numOfStartTask; @@ -32,48 +32,6 @@ typedef struct { uint64_t numOfErrors; } SQnodeStat; -/* start Task msg */ -typedef struct { - uint32_t schedulerIp; - uint16_t schedulerPort; - int64_t taskId; - int64_t queryId; - uint32_t srcIp; - uint16_t srcPort; -} SQnodeStartTaskMsg; - -/* stop Task msg */ -typedef struct { - int64_t taskId; -} SQnodeStopTaskMsg; - -/* start/stop Task msg response */ -typedef struct { - int64_t taskId; - int32_t code; -} SQnodeTaskRespMsg; - -/* Task status msg */ -typedef struct { - int64_t taskId; - int32_t status; - int64_t queryId; -} SQnodeTaskStatusMsg; - -/* Qnode/Scheduler heartbeat msg */ -typedef struct { - int32_t status; - int32_t load; - -} SQnodeHeartbeatMsg; - -/* Qnode sent/received msg */ -typedef struct { - int8_t msgType; - int32_t msgLen; - char msg[]; -} SQnodeMsg; - /** * Start one Qnode in Dnode. diff --git a/include/libs/index/index.h b/include/libs/index/index.h index f4d45477c11af7161221e493f8c15118d358d252..f93c46da0c412e3529dd0c016ef7c42dc8de1686 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -44,41 +44,46 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = * @param: oper * */ -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); /* * @param: * @param: */ -int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); -void indexClose(SIndex *index); -int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); -int indexDelete(SIndex *index, SIndexMultiTermQuery *query); -int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); -int indexRebuild(SIndex *index, SIndexOpts *opt); +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +void indexClose(SIndex* index); +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +int indexRebuild(SIndex* index, SIndexOpts* opt); /* * @param * @param */ -SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +SIndexMultiTerm* indexMultiTermCreate(); +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term); +void indexMultiTermDestroy(SIndexMultiTerm* terms); /* * @param: * @param: */ -SIndexOpts *indexOptsCreate(); -void indexOptsDestroy(SIndexOpts *opts); +SIndexOpts* indexOptsCreate(); +void indexOptsDestroy(SIndexOpts* opts); /* * @param: * @param: */ -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName, - int32_t nColName, const char *colVal, int32_t nColVal); -void indexTermDestroy(SIndexTerm *p); +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn operType, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal); +void indexTermDestroy(SIndexTerm* p); #ifdef __cplusplus } diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h new file mode 100644 index 0000000000000000000000000000000000000000..62f36abcee9c45c0c254848c57e8d24a108999ba --- /dev/null +++ b/include/libs/parser/parsenodes.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PARSENODES_H_ +#define _TD_PARSENODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "common.h" +#include "function.h" +#include "tmsgtype.h" +#include "tname.h" +#include "tvariant.h" + +/* + * The first field of a node of any type is guaranteed to be the int16_t. + * Hence the type of any node can be gotten by casting it to SQueryNode. + */ +typedef struct SQueryNode { + int16_t type; +} SQueryNode; + +#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) + +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int16_t bytes; +} SField; + +typedef struct SParseBasicCtx { + const char *db; + int32_t acctId; + uint64_t requestId; +} SParseBasicCtx; + +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SField *final; + SArray *internalField; // SArray +} SFieldInfo; + +typedef struct SCond { + uint64_t uid; + int32_t len; // length of tag query condition data + char * cond; +} SCond; + +typedef struct SJoinNode { + uint64_t uid; + int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; +} SJoinInfo; + +typedef struct STagCond { + int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR + SCond tbnameCond; // tbname query condition, only support tbname query condition on one table + SJoinInfo joinInfo; // join condition, only support two tables join currently + SArray *pCond; // for different table, the query condition must be seperated +} STagCond; + +typedef struct STableMetaInfo { + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + SVgroupsInfo *vgroupList; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns +} STableMetaInfo; + +typedef struct SColumnIndex { + int16_t tableIndex; + int16_t columnIndex; + int16_t type; // normal column/tag/ user input constant column +} SColumnIndex; + +// select statement +typedef struct SQueryStmtInfo { + int16_t command; // the command may be different for each subclause, so keep it seperately. + uint32_t type; // query/insert type + STimeWindow window; // the whole query time window + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window + SStateWindow stateWindow; // state window query + SGroupbyExpr groupbyExpr; // groupby tags info + SArray * colList; // SArray + SFieldInfo fieldsInfo; + SArray** exprList; // SArray + SLimit limit; + SLimit slimit; + STagCond tagCond; + SArray * colCond; + SArray * order; + int16_t numOfTables; + int16_t curTableIdx; + STableMetaInfo **pTableMetaInfo; + struct STSBuf *tsBuf; + + int16_t fillType; // final result fill type + int64_t * fillVal; // default value for fill + int32_t numOfFillVal; // fill value size + + char * msg; // pointer to the pCmd->payload to keep error message temporarily + int64_t clauseLimit; // limit for current sub clause + + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit + + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX + int32_t bufLen; + char* buf; + SArray *pUdfInfo; + + struct SQueryStmtInfo *sibling; // sibling + struct SQueryStmtInfo *pDownstream; + SMultiFunctionsDesc info; + SArray *pUpstream; // SArray + int32_t havingFieldNum; + int32_t exprListLevelIndex; +} SQueryStmtInfo; + +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +typedef struct SVgDataBlocks { + SVgroupInfo vg; + int32_t numOfTables; // number of tables in current submit block + uint32_t size; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... +} SVgDataBlocks; + +typedef struct SInsertStmtInfo { + int16_t nodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + int8_t schemaAttache; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position +} SInsertStmtInfo; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PARSENODES_H_*/ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index a92398602b3a980e332eab11c7153c28bf167bc3..03750a16a9e8cb3bce6acafe91d0305fff3ed49e 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,115 +20,7 @@ extern "C" { #endif -#include "catalog.h" -#include "common.h" -#include "tname.h" -#include "tvariant.h" -#include "function.h" - -typedef struct SField { - char name[TSDB_COL_NAME_LEN]; - uint8_t type; - int32_t bytes; -} SField; - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SField *final; - SArray *internalField; // SArray -} SFieldInfo; - -// TODO merged with SParseContext -typedef struct SParseCtx { - char *db; - int32_t acctId; - uint64_t requestId; -} SParseCtx; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR - SCond tbnameCond; // tbname query condition, only support tbname query condition on one table - SJoinInfo joinInfo; // join condition, only support two tables join currently - SArray *pCond; // for different table, the query condition must be seperated -} STagCond; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - SVgroupsInfo *vgroupList; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; - -typedef struct SQueryStmtInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - SStateWindow stateWindow; // state window query - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray** exprList; // SArray - SLimit limit; - SLimit slimit; - STagCond tagCond; - SArray * colCond; - SArray * order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - int32_t bufLen; - char* buf; - SArray *pUdfInfo; - - struct SQueryStmtInfo *sibling; // sibling - struct SQueryStmtInfo *pDownstream; - SMultiFunctionsDesc info; - SArray *pUpstream; // SArray - int32_t havingFieldNum; - int32_t exprListLevelIndex; -} SQueryStmtInfo; - -typedef struct SColumnIndex { - int16_t tableIndex; - int16_t columnIndex; - int16_t type; // normal column/tag/ user input constant column -} SColumnIndex; - -struct SInsertStmtInfo; +#include "parsenodes.h" /** * True will be returned if the input sql string is insert, false otherwise. @@ -139,17 +31,16 @@ struct SInsertStmtInfo; bool qIsInsertSql(const char* pStr, size_t length); typedef struct SParseContext { - const char* pAcctId; - const char* pDbname; - void *pRpc; - const char* pClusterId; - const SEpSet* pEpSet; - int64_t id; // query id, generated by uuid generator - int8_t schemaAttached; // denote if submit block is built with table schema or not - const char* pSql; // sql string - size_t sqlLen; // length of the sql string - char* pMsg; // extended error message if exists to help avoid the problem in sql statement. - int32_t msgLen; // max length of the msg + SParseBasicCtx ctx; + void *pRpc; + struct SCatalog *pCatalog; + const SEpSet *pEpSet; + int64_t id; // query id, generated by uuid generator + int8_t schemaAttached; // denote if submit block is built with table schema or not + const char *pSql; // sql string + size_t sqlLen; // length of the sql string + char *pMsg; // extended error message if exists to help avoid the problem in sql statement. + int32_t msgLen; // max length of the msg } SParseContext; /** @@ -160,27 +51,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, SParseCtx* pParseCtx, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); - -typedef enum { - PAYLOAD_TYPE_KV = 0, - PAYLOAD_TYPE_RAW = 1, -} EPayloadType; - -typedef struct SVgDataBlocks { - int64_t vgId; // virtual group id - int32_t numOfTables; // number of tables in current submit block - uint32_t size; - char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... -} SVgDataBlocks; - -typedef struct SInsertStmtInfo { - SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position -} SInsertStmtInfo; +int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); /** * Parse the insert sql statement. @@ -217,9 +88,9 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo); -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 8b54b88b28fd908bf2ad9040918cdcf9a4c4eea0..153cf0bb3ec2e4ad33f1e04ebff53a709e3e555c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -25,6 +25,7 @@ extern "C" { #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_SCAN 3 +#define QUERY_TYPE_MODIFY 4 enum OPERATOR_TYPE_E { OP_Unknown, @@ -58,18 +59,17 @@ typedef struct SQueryNodeBasicInfo { typedef struct SDataSink { SQueryNodeBasicInfo info; - SDataBlockSchema schema; } SDataSink; typedef struct SDataDispatcher { SDataSink sink; - // todo } SDataDispatcher; typedef struct SDataInserter { SDataSink sink; - uint64_t uid; // unique id of the table - // todo data field + int32_t numOfTables; + uint32_t size; + char *pData; } SDataInserter; typedef struct SPhyNode { @@ -119,12 +119,13 @@ typedef struct SSubplanId { typedef struct SSubplan { SSubplanId id; // unique id of the subplan - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN - int32_t level; // the execution level of current subplan, starting from 0. - SEpSet execEpSet; // for the scan sub plan, the optional execution node - SArray *pChildern; // the datasource subplan,from which to fetch the result - SArray *pParents; // the data destination subplan, get data from current subplan - SPhyNode *pNode; // physical plan of current subplan + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t level; // the execution level of current subplan, starting from 0. + SEpSet execEpSet; // for the scan sub plan, the optional execution node + SArray *pChildern; // the datasource subplan,from which to fetch the result + SArray *pParents; // the data destination subplan, get data from current subplan + SPhyNode *pNode; // physical plan of current subplan + SDataSink *pDataSink; // data of the subplan flow into the datasink } SSubplan; typedef struct SQueryDag { @@ -133,10 +134,12 @@ typedef struct SQueryDag { SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; +struct SQueryNode; + /** * Create the physical plan for the query, according to the AST. */ -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule @@ -144,7 +147,7 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* // @ep one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); -int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); +int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); /** * Convert to subplan to string for the scheduler to send to the executor diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 70dcc9428087ae73b1297e67bf60adbbe161d086..002e736c37f1992af6a7693c07b1f64fa879ccb3 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -24,6 +24,15 @@ extern "C" { #include "thash.h" #include "tlog.h" +enum { + JOB_TASK_STATUS_NOT_START = 1, + JOB_TASK_STATUS_EXECUTING, + JOB_TASK_STATUS_SUCCEED, + JOB_TASK_STATUS_FAILED, + JOB_TASK_STATUS_CANCELLING, + JOB_TASK_STATUS_CANCELLED +}; + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h new file mode 100644 index 0000000000000000000000000000000000000000..63a6b6f89bee7100abd91426b979163a99cfd641 --- /dev/null +++ b/include/libs/qworker/qworker.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_H_ +#define _TD_QWORKER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "trpc.h" + +typedef struct SQWorkerCfg { + uint32_t maxSchedulerNum; + uint32_t maxResCacheNum; + uint32_t maxSchTaskNum; +} SQWorkerCfg; + +typedef struct { + uint64_t numOfStartTask; + uint64_t numOfStopTask; + uint64_t numOfRecvedFetch; + uint64_t numOfSentHb; + uint64_t numOfSentFetch; + uint64_t numOfTaskInQueue; + uint64_t numOfFetchInQueue; + uint64_t numOfErrors; +} SQWorkerStat; + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt); + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_H_*/ diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 2cbf26f8772927f67c3a882465cd66414be20a93..975b10353871f269abecb70d4974836d3e4a44b2 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -25,6 +25,7 @@ extern "C" { typedef struct SSchedulerCfg { int32_t clusterType; + int32_t maxJobNum; } SSchedulerCfg; typedef struct SQueryProfileSummary { diff --git a/include/util/thash.h b/include/util/thash.h index d5038802c079f3ed3cddcef6c30c219459fc9917..ebdc91f0546a7ef9de3801f4eabf13f18c2cdf4c 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -193,11 +193,10 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); /** * Get the corresponding key information for a given data in hash table - * @param pHashObj * @param data * @return */ -int32_t taosHashGetKey(SHashObj *pHashObj, void *data, void** key, size_t* keyLen); +int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0deaf5ad8e17e35f39de4de422883ffa5f3e04d8..42de97fe6a2387e824355cd7d491a717cc2d1744 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -152,7 +152,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { void* output = NULL; int32_t outputLen = 0; - SParseCtx c = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)}; + SParseBasicCtx c = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)}; code = qParseQuerySql(pRequest->sqlstr, sqlLen, &c, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT || @@ -167,8 +167,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); - - destroyRequestMsgBody(&body); } else { assert(0); diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c deleted file mode 100644 index b7a622c2e8f0c06859154b2c212f9a4a9f19b659..0000000000000000000000000000000000000000 --- a/source/client/src/tscEnv.c +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosmsg.h" -#include "query.h" -#include "clientInt.h" -#include "clientLog.h" -#include "tcache.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "tref.h" -#include "trpc.h" -#include "ttime.h" -#include "ttimezone.h" - -#define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 - -SAppInfo appInfo; -int32_t tscReqRef = -1; -int32_t tscConnRef = -1; - -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; - -static void registerRequest(SRequestObj* pRequest) { - STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id); - assert(pTscObj != NULL); - - // connection has been released already, abort creating request. - pRequest->self = taosAddRef(tscReqRef, pRequest); - - int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); - - if (pTscObj->pAppInfo) { - SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary; - - int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, - pRequest->pTscObj->id, num, currentInst, total); - } -} - -static void deregisterRequest(SRequestObj* pRequest) { - assert(pRequest != NULL); - - STscObj* pTscObj = pRequest->pTscObj; - SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; - - int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); - taosReleaseRef(tscConnRef, pTscObj->id); -} - -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} - -void closeTransporter(STscObj* pTscObj) { - if (pTscObj == NULL || pTscObj->pTransporter == NULL) { - return; - } - - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); - rpcClose(pTscObj->pTransporter); - pTscObj->pTransporter = NULL; -} - -// TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = numOfThread; - rpcInit.cfp = processMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; -// rpcInit.spi = 1; - rpcInit.secret = (char *)auth; - - void* pDnodeConn = rpcOpen(&rpcInit); - if (pDnodeConn == NULL) { - tscError("failed to init connection to server"); - return NULL; - } - - return pDnodeConn; -} - -void destroyTscObj(void *pObj) { - STscObj *pTscObj = pObj; - - atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - pthread_mutex_destroy(&pTscObj->mutex); - tfree(pTscObj); -} - -void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); - if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pObj->pAppInfo = pAppInfo; - if (pAppInfo != NULL) { - pObj->pTransporter = pAppInfo->pTransporter; - } - - tstrncpy(pObj->user, user, sizeof(pObj->user)); - memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); - - pthread_mutex_init(&pObj->mutex, NULL); - pObj->id = taosAddRef(tscConnRef, pObj); - - tscDebug("connObj created, 0x%"PRIx64, pObj->id); - return pObj; -} - -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { - assert(pObj != NULL); - - SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); - if (NULL == pRequest) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - // TODO generated request uuid - pRequest->requestId = 0; - - pRequest->metric.start = taosGetTimestampMs(); - - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; -// pRequest->body.requestMsg. = param; - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); - tsem_init(&pRequest->body.rspSem, 0, 0); - - registerRequest(pRequest); - return pRequest; -} - -static void doDestroyRequest(void* p) { - assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; - - assert(RID_VALID(pRequest->self)); - - tfree(pRequest->msgBuf); - tfree(pRequest->sqlstr); - tfree(pRequest->pInfo); - - tfree(pRequest->body.resInfo.pRspMsg); - - deregisterRequest(pRequest); - tfree(pRequest); -} - -void destroyRequest(SRequestObj* pRequest) { - if (pRequest == NULL) { - return; - } - - taosReleaseRef(tscReqRef, pRequest->self); -} - -void taos_init_imp(void) { - // In the APIs of other program language, taos_cleanup is not available yet. - // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. - atexit(taos_cleanup); - - errno = TSDB_CODE_SUCCESS; - srand(taosGetTimestampSec()); - - deltaToUtcInitOnce(); - taosInitGlobalCfg(); - taosReadCfgFromFile(); - - tscInitLogFile(); - if (taosCheckAndPrintCfg()) { - tscInitRes = -1; - return; - } - - taosInitNotes(); - initMsgHandleFp(); - - rpcInit(); - - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); - - taosSetCoreDump(true); - - initTaskQueue(); - - tscConnRef = taosOpenRef(200, destroyTscObj); - tscReqRef = taosOpenRef(40960, doDestroyRequest); - - taosGetAppName(appInfo.appName, NULL); - appInfo.pid = taosGetPId(); - appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - - tscDebug("client is initialized successfully"); -} - -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -int taos_options_imp(TSDB_OPTION option, const char *str) { - SGlobalCfg *cfg = NULL; - - switch (option) { - case TSDB_OPTION_CONFIGDIR: - cfg = taosGetConfigOption("configDir"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, str, TSDB_FILENAME_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - cfg = taosGetConfigOption("shellActivityTimer"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(str); - if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; - if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); - } - break; - - case TSDB_OPTION_LOCALE: { // set locale - cfg = taosGetConfigOption("locale"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - char sep = '.'; - - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); - - // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. - // The launch of current system will abort soon. - if (defaultLocale == NULL) { - tscError("failed to get default locale, please set the correct locale in current OS"); - return -1; - } - - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); - } - - // set the user specified locale - char *locale = setlocale(LC_CTYPE, str); - - if (locale != NULL) { // failed to set the user specified locale - tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale - locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); - } - - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - - char *charset = strrchr(tsLocale, sep); - if (charset != NULL) { - charset += 1; - - charset = taosCharsetReplace(charset); - - if (taosValidateEncodec(charset)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset set:%s", charset); - } else { - tscInfo("charset changed from %s to %s", tsCharset, charset); - } - - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - - } else { - tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); - } - - free(charset); - } else { // it may be windows system - tscInfo("charset remains:%s", tsCharset); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - } - - case TSDB_OPTION_CHARSET: { - /* set charset will override the value of charset, assigned during system locale changed */ - cfg = taosGetConfigOption("charset"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("failed to set charset:%s", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(str)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", str); - } else { - tscInfo("charset changed from %s to %s", tsCharset, str); - } - - tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { - tscInfo("charset:%s not valid", str); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - - break; - } - - case TSDB_OPTION_TIMEZONE: - cfg = taosGetConfigOption("timezone"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); - tsSetTimeZone(); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - default: - // TODO return the correct error code to client in the format for taos_errstr() - tscError("Invalid option %d", option); - return -1; - } - - return 0; -} - -#if 0 -#include "cJSON.h" -static setConfRet taos_set_config_imp(const char *config){ - setConfRet ret = {SET_CONF_RET_SUCC, {0}}; - static bool setConfFlag = false; - if (setConfFlag) { - ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; - strcpy(ret.retMsg, "configuration can only set once"); - return ret; - } - taosInitGlobalCfg(); - cJSON *root = cJSON_Parse(config); - if (root == NULL){ - ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; - strcpy(ret.retMsg, "parse json error"); - return ret; - } - - int size = cJSON_GetArraySize(root); - if(!cJSON_IsObject(root) || size == 0) { - ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; - strcpy(ret.retMsg, "json content is invalid, must be not empty object"); - return ret; - } - - if(size >= 1000) { - ret.retCode = SET_CONF_RET_ERR_TOO_LONG; - strcpy(ret.retMsg, "json object size is too long"); - return ret; - } - - for(int i = 0; i < size; i++){ - cJSON *item = cJSON_GetArrayItem(root, i); - if(!item) { - ret.retCode = SET_CONF_RET_ERR_INNER; - strcpy(ret.retMsg, "inner error"); - return ret; - } - if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ - ret.retCode = SET_CONF_RET_ERR_PART; - if (strlen(ret.retMsg) == 0){ - snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); - }else{ - int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); - size_t leftSize = tmp >= 0 ? tmp : 0; - strncat(ret.retMsg, "|", leftSize); - tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); - leftSize = tmp >= 0 ? tmp : 0; - strncat(ret.retMsg, item->string, leftSize); - } - } - } - cJSON_Delete(root); - setConfFlag = true; - return ret; -} - -setConfRet taos_set_config(const char *config){ - pthread_mutex_lock(&setConfMutex); - setConfRet ret = taos_set_config_imp(config); - pthread_mutex_unlock(&setConfMutex); - return ret; -} -#endif \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 50b1a1cf20fa2db10c401911aff6b8178f374d64..be60d2fcae26fc0a054b83c24a72f9cd7272d600 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -73,7 +73,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index a5ece72f425f8d00bd6d4328350cb48dab186cc5..a29926c8026950028690bf3d07fa224ee1dd5084 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -8,8 +8,8 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) # add_subdirectory(mnode) -# add_subdirectory(profile) -# add_subdirectory(show) +add_subdirectory(profile) +add_subdirectory(show) add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) @@ -17,4 +17,4 @@ add_subdirectory(stb) add_subdirectory(user) add_subdirectory(vgroup) -# add_subdirectory(common) +add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index 8b160fc446b2f501e8f2ece4e76e1dbc27a77a74..a06becd1279b2f66f73cb3d8fbf347c9e4848e2b 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_acct "") - -target_sources(dnode_test_acct - PRIVATE - "acct.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. ACCT_SRC) +add_executable(dnode_test_acct ${ACCT_SRC}) target_link_libraries( dnode_test_acct - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_acct - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index e2b819da2828dc4f7d4d0a915edf76c1bc29a06c..69ddae675d12e353a610769592565b18d0078ada 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -9,103 +9,59 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestAcct : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9012"; - pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9012, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9012); - taosMsleep(300); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_acct", 9012); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} }; -SServer* DndTestAcct::pServer; -SClient* DndTestAcct::pClient; -int32_t DndTestAcct::connId; +Testbase DndTestAcct::test; TEST_F(DndTestAcct, 01_CreateAcct) { - ASSERT_NE(pClient, nullptr); - - SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg)); + int32_t contLen = sizeof(SCreateAcctMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_ACCT; + SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(contLen); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 02_AlterAcct) { - ASSERT_NE(pClient, nullptr); + int32_t contLen = sizeof(SCreateAcctMsg); - SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg)); + SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(contLen); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_ACCT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 03_DropAcct) { - ASSERT_NE(pClient, nullptr); - - SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg)); + int32_t contLen = sizeof(SDropAcctMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_ACCT; + SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(contLen); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 04_ShowAcct) { - ASSERT_NE(pClient, nullptr); + int32_t contLen = sizeof(SShowMsg); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); pReq->type = TSDB_MGMT_TABLE_ACCT; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt index e97ef7688140bc4d85a7d1a734b21a6b586c1297..a63f3106e6299299adf963427604c7384931f35c 100644 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_cluster "") - -target_sources(dnode_test_cluster - PRIVATE - "cluster.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. CLUSTER_SRC) +add_executable(dnode_test_cluster ${CLUSTER_SRC}) target_link_libraries( dnode_test_cluster - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_cluster - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 0aca2fde4badaa17b6fa70186ed9ad157c588678..7230c3eb74ca2e0cb08454f1f170b046918c4cc7 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -9,162 +9,33 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestCluster : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_cluster", 9030); } + static void TearDownTestSuite() { test.Cleanup(); } - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9030"; - pServer = CreateServer("/tmp/dnode_test_cluster", fqdn, 9030, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9030); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt32() { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_GT(data, 0); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestCluster::pServer; -SClient* DndTestCluster::pClient; -int32_t DndTestCluster::connId; +Testbase DndTestCluster::test; TEST_F(DndTestCluster, 01_ShowCluster) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, "show cluster", 3, NULL); - CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, ""); + CHECK_META( "show cluster", 3); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendThenCheckShowRetrieveMsg(1); - CheckInt32(); - CheckBinary(TSDB_CLUSTER_ID_LEN); + IgnoreInt32(); + IgnoreBinary(TSDB_CLUSTER_ID_LEN); CheckTimestamp(); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt index db96e2bad32eff4e99e6a3c68421608f655f18ec..cb9f1600fc2a76951a5cb81cf94c5d1ef48d5eb1 100644 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_db "") - -target_sources(dnode_test_db - PRIVATE - "db.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. DB_SRC) +add_executable(dnode_test_db ${DB_SRC}) target_link_libraries( dnode_test_db - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_db - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index 204afa111fdd509aac44937774410a8165905314..378f46aa4defe07fa739b170b3cf04f442d99d60 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -9,199 +9,52 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestDb : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9040"; - pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9040); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_db", 9040); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestDb::pServer; -SClient* DndTestDb::pClient; -int32_t DndTestDb::connId; +Testbase DndTestDb::test; TEST_F(DndTestDb, 01_ShowDb) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); - CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); - CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache"); - CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks"); - CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows"); - CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); - CheckSchema(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); - CheckSchema(12, TSDB_DATA_TYPE_INT, 4, "fsync"); - CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); - CheckSchema(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); - CheckSchema(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CheckSchema(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); - - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CHECK_SCHEMA(7, TSDB_DATA_TYPE_INT, 4, "cache"); + CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "blocks"); + CHECK_SCHEMA(9, TSDB_DATA_TYPE_INT, 4, "minrows"); + CHECK_SCHEMA(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CHECK_SCHEMA(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CHECK_SCHEMA(12, TSDB_DATA_TYPE_INT, 4, "fsync"); + CHECK_SCHEMA(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CHECK_SCHEMA(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CHECK_SCHEMA(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CHECK_SCHEMA(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -223,20 +76,16 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - // taosMsleep(1000000); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -255,12 +104,15 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("ms", 3); // precision CheckInt8(0); // update - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "show vgroups", 4, "1.d1"); - CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "vgId"); - CheckSchema(1, TSDB_DATA_TYPE_INT, 4, "tables"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); - CheckSchema(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); - SendThenCheckShowRetrieveMsg(2); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "1.d1"); + CHECK_META("show vgroups", 4); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "vgId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_INT, 4, "tables"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); CheckInt32(1); CheckInt32(2); CheckInt32(0); @@ -271,7 +123,9 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + int32_t contLen = sizeof(SAlterDbMsg); + + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->totalBlocks = htonl(12); pReq->daysToKeep0 = htonl(300); @@ -282,19 +136,14 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->quorum = 2; pReq->cacheLastRow = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -314,19 +163,14 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update // restart - stopServer(pServer); - pServer = NULL; - - uInfo("start all server"); + test.Restart(); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9040"; - pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); - uInfo("all server is running"); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -346,27 +190,28 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SDropDbMsg)); - strcpy(pReq->db, "1.d1"); + int32_t contLen = sizeof(SDropDbMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + strcpy(pReq->db, "1.d1"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -388,33 +233,26 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d2", TSDB_DB_NAME_LEN - 1); { - SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); + int32_t contLen = sizeof(SUseDbMsg); + + SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->vgVersion = htonl(-1); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SUseDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index 48b74a4eff8c0e07f115c8cb4cfcd04448413319..5796590865a0b4d5ec3e66b38441ea67465c79df 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_dnode "") - -target_sources(dnode_test_dnode - PRIVATE - "dnode.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. DTEST_SRC) +add_executable(dnode_test_dnode ${DTEST_SRC}) target_link_libraries( dnode_test_dnode - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_dnode - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 1fa91f26cb6edb13aa9588ef26d0f4136d0621c9..c9708ec1cfcb8fdec5ddf10cd63918100187c010 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -9,189 +9,62 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestDnode : public ::testing::Test { public: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } + void SetUp() override {} + void TearDown() override {} + public: static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - + test.Init("/tmp/dnode_test_dnode1", 9041); const char* fqdn = "localhost"; const char* firstEp = "localhost:9041"; - pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); - pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); - pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); - pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); - pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9041); + + server2.Start("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); + server3.Start("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + server4.Start("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + server5.Start("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); taosMsleep(300); } static void TearDownTestSuite() { - stopServer(pServer1); - stopServer(pServer2); - stopServer(pServer3); - stopServer(pServer4); - stopServer(pServer5); - dropClient(pClient); - pServer1 = NULL; - pServer2 = NULL; - pServer3 = NULL; - pServer4 = NULL; - pServer5 = NULL; - pClient = NULL; - } - - static SServer* pServer1; - static SServer* pServer2; - static SServer* pServer3; - static SServer* pServer4; - static SServer* pServer5; - static SClient* pClient; - - public: - void SetUp() override {} - void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; + server2.Stop(); + server3.Stop(); + server4.Stop(); + server5.Stop(); + test.Cleanup(); } - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } + static Testbase test; + static TestServer server2; + static TestServer server3; + static TestServer server4; + static TestServer server5; +}; - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } +Testbase DndTestDnode::test; +TestServer DndTestDnode::server2; +TestServer DndTestDnode::server3; +TestServer DndTestDnode::server4; +TestServer DndTestDnode::server5; - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } +TEST_F(DndTestDnode, 01_ShowDnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; -}; + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); -SServer* DndTestDnode::pServer1; -SServer* DndTestDnode::pServer2; -SServer* DndTestDnode::pServer3; -SServer* DndTestDnode::pServer4; -SServer* DndTestDnode::pServer5; -SClient* DndTestDnode::pClient; + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); -TEST_F(DndTestDnode, 01_ShowDnode) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); - CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); - CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); - - SendThenCheckShowRetrieveMsg(1); CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); @@ -202,40 +75,36 @@ TEST_F(DndTestDnode, 01_ShowDnode) { } TEST_F(DndTestDnode, 02_ConfigDnode) { - SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); + int32_t contLen = sizeof(SCfgDnodeMsg); + + SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCfgDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONFIG_DNODE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONFIG_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9042"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9042"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(2); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + CheckInt16(1); CheckInt16(2); CheckBinary("localhost:9041", TSDB_EP_LEN); @@ -252,22 +121,21 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); - pReq->dnodeId = htonl(2); + int32_t contLen = sizeof(SDropDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); @@ -277,53 +145,44 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9043"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9043"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9044"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9044"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9045"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9045"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(4); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 4); + CheckInt16(1); CheckInt16(3); CheckInt16(4); @@ -355,31 +214,18 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { // restart uInfo("stop all server"); - stopServer(pServer1); - stopServer(pServer2); - stopServer(pServer3); - stopServer(pServer4); - stopServer(pServer5); - pServer1 = NULL; - pServer2 = NULL; - pServer3 = NULL; - pServer4 = NULL; - pServer5 = NULL; - - uInfo("start all server"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9041"; - pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); - pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); - pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); - pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); - - uInfo("all server is running"); + test.Restart(); + server2.Restart(); + server3.Restart(); + server4.Restart(); + server5.Restart(); taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(4); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 4); + CheckInt16(1); CheckInt16(3); CheckInt16(4); diff --git a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt index d8e4eae0426037ecf20a7888a18360ccdd023391..0edd63172034b364e448f5164a4334d1a06f34d4 100644 --- a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt @@ -1,27 +1,11 @@ -add_executable(dndTestProfile "") - -target_sources(dndTestProfile - PRIVATE - "profile.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. PROFILE_SRC) +add_executable(dnode_test_profile ${PROFILE_SRC}) target_link_libraries( - dndTestProfile - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dndTestProfile - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + dnode_test_profile + PUBLIC sut ) add_test( - NAME dndTestProfile - COMMAND dndTestProfile + NAME dnode_test_profile + COMMAND dnode_test_profile ) diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index f1e4704777b1794ce8ff3bc183eeb34551b3db53..2e82d4d1aa6d1c5007ef7453cb4fe956118823c5 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -9,55 +9,35 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestProfile : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_profile", 9080); } + static void TearDownTestSuite() { test.Cleanup(); } - static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestProfile"; - const char* fqdn = "localhost"; - uint16_t port = 9522; + static Testbase test; - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - } + public: + void SetUp() override {} + void TearDown() override {} - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + int32_t connId; }; -SServer* DndTestProfile::pServer; -SClient* DndTestProfile::pClient; -int32_t DndTestProfile::connId; +Testbase DndTestProfile::test; -TEST_F(DndTestProfile, SConnectMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 01_ConnectMsg) { + int32_t contLen = sizeof(SConnectMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -69,183 +49,65 @@ TEST_F(DndTestProfile, SConnectMsg_01) { EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); EXPECT_EQ(pRsp->connId, 1); - EXPECT_EQ(pRsp->superAuth, 1); - EXPECT_EQ(pRsp->readAuth, 1); - EXPECT_EQ(pRsp->writeAuth, 1); + EXPECT_EQ(pRsp->superUser, 1); EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } -TEST_F(DndTestProfile, SConnectMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { + int32_t contLen = sizeof(SConnectMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, "invalid_db"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_DB); ASSERT_EQ(pMsg->contLen, 0); } -TEST_F(DndTestProfile, SConnectMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CONNS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 7); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "program"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "pid"); - - pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - - pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "login_time"); - - pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "last_access"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->compLen, 0); - } +TEST_F(DndTestProfile, 03_ConnectMsg_Show) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + CHECK_META("show connections", 7); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "connId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "user"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, "program"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "pid"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, "ip:port"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "login_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_TIMESTAMP, 8, "last_access"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckInt32(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("dnode_test_profile", TSDB_APP_NAME_LEN); + CheckInt32(1234); + IgnoreBinary(TSDB_IPv4ADDR_LEN + 6); + CheckTimestamp(); + CheckTimestamp(); } -TEST_F(DndTestProfile, SHeartBeatMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 04_HeartBeatMsg) { + int32_t contLen = sizeof(SHeartBeatMsg); - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -265,62 +127,49 @@ TEST_F(DndTestProfile, SHeartBeatMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } -TEST_F(DndTestProfile, SKillConnMsg_01) { - ASSERT_NE(pClient, nullptr); - +TEST_F(DndTestProfile, 05_KillConnMsg) { { - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); - pReq->connId = htonl(connId); + int32_t contLen = sizeof(SKillConnMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillConnMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); + pReq->connId = htonl(connId); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + int32_t contLen = sizeof(SHeartBeatMsg); + + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + strcpy(pReq->app, "dnode_test_profile"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONNECTION); ASSERT_EQ(pMsg->contLen, 0); } { - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + int32_t contLen = sizeof(SConnectMsg); + + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -332,71 +181,55 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); EXPECT_GT(pRsp->connId, connId); - EXPECT_EQ(pRsp->readAuth, 1); - EXPECT_EQ(pRsp->writeAuth, 1); + EXPECT_EQ(pRsp->superUser, 1); EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } } -TEST_F(DndTestProfile, SKillConnMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { + int32_t contLen = sizeof(SKillConnMsg); - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillConnMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, SKillQueryMsg_01) { - ASSERT_NE(pClient, nullptr); - +TEST_F(DndTestProfile, 07_KillQueryMsg) { { - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + int32_t contLen = sizeof(SKillQueryMsg); + + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->queryId = htonl(1234); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillQueryMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); ASSERT_EQ(pMsg->contLen, 0); } { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + int32_t contLen = sizeof(SHeartBeatMsg); + + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -416,314 +249,42 @@ TEST_F(DndTestProfile, SKillQueryMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } -TEST_F(DndTestProfile, SKillQueryMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { + int32_t contLen = sizeof(SKillQueryMsg); - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); pReq->queryId = htonl(1234); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillQueryMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, SKillQueryMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_QUERIES; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 14); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "queryId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 0); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } -} - -TEST_F(DndTestProfile, SKillStreamMsg_01) { - ASSERT_NE(pClient, nullptr); - - { - SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); - pReq->connId = htonl(connId); - pReq->streamId = htonl(3579); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillStreamMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_STREAM; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - ASSERT_EQ(pMsg->contLen, 0); - } - - { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); - pReq->connId = htonl(connId); - pReq->pid = htonl(1234); - pReq->numOfQueries = htonl(0); - pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->connId = htonl(pRsp->connId); - pRsp->queryId = htonl(pRsp->queryId); - pRsp->streamId = htonl(pRsp->streamId); - pRsp->totalDnodes = htonl(pRsp->totalDnodes); - pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); - pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); - - EXPECT_EQ(pRsp->connId, connId); - EXPECT_EQ(pRsp->queryId, 0); - EXPECT_EQ(pRsp->streamId, 3579); - EXPECT_EQ(pRsp->totalDnodes, 1); - EXPECT_EQ(pRsp->onlineDnodes, 1); - EXPECT_EQ(pRsp->killConnection, 0); - - EXPECT_EQ(pRsp->epSet.inUse, 0); - EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); - } -} - -TEST_F(DndTestProfile, SKillStreamMsg_02) { - ASSERT_NE(pClient, nullptr); - - SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); - pReq->connId = htonl(2345); - pReq->streamId = htonl(1234); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillStreamMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); -} - -TEST_F(DndTestProfile, SKillStreamMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_STREAMS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 10); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "streamId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 0); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } +TEST_F(DndTestProfile, 09_KillQueryMsg) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QUERIES, ""); + CHECK_META("show queries", 14); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "queryId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_INT, 4, "connId"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "user"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, "ip:port"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 22 + VARSTR_HEADER_SIZE, "qid"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "created_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BIGINT, 8, "time"); + CHECK_SCHEMA(7, TSDB_DATA_TYPE_BINARY, 18 + VARSTR_HEADER_SIZE, "sql_obj_id"); + CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "pid"); + CHECK_SCHEMA(9, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "ep"); + CHECK_SCHEMA(10, TSDB_DATA_TYPE_BOOL, 1, "stable_query"); + CHECK_SCHEMA(11, TSDB_DATA_TYPE_INT, 4, "sub_queries"); + CHECK_SCHEMA(12, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, "sub_query_info"); + CHECK_SCHEMA(13, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, "sql"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt index b3dc3297a8a8926707fc3d409555e0cdefff9c70..9786f65605ca25686b8d88e236d8834ce24274d9 100644 --- a/source/dnode/mgmt/impl/test/show/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_show "") - -target_sources(dnode_test_show - PRIVATE - "show.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. SHOW_SRC) +add_executable(dnode_test_show ${SHOW_SRC}) target_link_libraries( dnode_test_show - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_show - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index f0c20f71e572640a190c724bab5cf9cf343dab7f..bec681cb8fed2115f70ebbb147a3c21926060a36 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -9,226 +9,79 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestShow : public ::testing::Test { protected: + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_show", 9091); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: void SetUp() override {} void TearDown() override {} - - static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestShow"; - const char* fqdn = "localhost"; - uint16_t port = 9523; - - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; }; -SServer* DndTestShow::pServer; -SClient* DndTestShow::pClient; -int32_t DndTestShow::connId; +Testbase DndTestShow::test; -TEST_F(DndTestShow, SShowMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 01_ShowMsg_InvalidMsgMax) { + int32_t contLen = sizeof(SShowMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); - pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestShow"); + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); + pReq->type = TSDB_MGMT_TABLE_MAX; strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); - - SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->connId = htonl(pRsp->connId); - - EXPECT_EQ(pRsp->connId, 1); - connId = pRsp->connId; + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, SShowMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 02_ShowMsg_InvalidMsgStart) { + int32_t contLen = sizeof(SShowMsg); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_MAX; + pReq->type = TSDB_MGMT_TABLE_START; strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, SShowMsg_03) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 02_ShowMsg_Conn) { + int32_t contLen = sizeof(SConnectMsg); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_START; + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + pReq->pid = htonl(1234); + strcpy(pReq->app, "dnode_test_show"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); -} - -TEST_F(DndTestShow, SShowMsg_04) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CONNS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 7); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "program"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "pid"); - - pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - - pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "login_time"); - - pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "last_access"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + + STableMetaMsg* pMeta = test.GetShowMeta(); + EXPECT_STREQ(pMeta->tbFname, "show connections"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, 7); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + + test.SendShowRetrieveMsg(); + + SRetrieveTableRsp* pRetrieveRsp = test.GetRetrieveRsp(); + EXPECT_EQ(pRetrieveRsp->numOfRows, 1); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); } diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt index 524e3e5c7ee5fdb65830c7cdcb32e5cfa985606b..68e4dfb07f114e7644638938130445549aa023ec 100644 --- a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_stb "") - -target_sources(dnode_test_stb - PRIVATE - "stb.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. STB_SRC) +add_executable(dnode_test_stb ${STB_SRC}) target_link_libraries( dnode_test_stb - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_stb - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index c12e8eadf45111af660126837fa1157212f8bd57..992ebaf902668c8d646d71969891c9d16831a601 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -9,176 +9,27 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestStb : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9101); } + static void TearDownTestSuite() { test.Cleanup(); } - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9101"; - pServer = CreateServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9101); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestStb::pServer; -SClient* DndTestStb::pClient; -int32_t DndTestStb::connId; +Testbase DndTestStb::test; TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -200,13 +51,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -214,9 +59,9 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { int32_t cols = 2; int32_t tags = 3; - int32_t size = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); + int32_t contLen = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); - SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(size); + SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); pReq->numOfTags = htonl(tags); pReq->numOfColumns = htonl(cols); @@ -261,24 +106,21 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { strcpy(pSchema->name, "tag3"); } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = size; - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_STB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(2, TSDB_DATA_TYPE_INT, 4, "columns"); - CheckSchema(3, TSDB_DATA_TYPE_INT, 4, "tags"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "columns"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "tags"); - SendThenCheckShowRetrieveMsg(1); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); CheckInt32(2); @@ -286,16 +128,12 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // ----- meta ------ { - STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(sizeof(STableInfoMsg)); - strcpy(pReq->tableFname, "1.d1.stb"); + int32_t contLen = sizeof(STableInfoMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(STableInfoMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_TABLE_META; + STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(contLen); + strcpy(pReq->tableFname, "1.d1.stb"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -336,39 +174,31 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } // restart - stopServer(pServer); - pServer = NULL; + test.Restart(); - uInfo("start all server"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9101"; - pServer = startServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); - - uInfo("all server is running"); - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - SendThenCheckShowRetrieveMsg(1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); CheckInt32(2); CheckInt32(3); { - SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(sizeof(SDropStbMsg)); - strcpy(pReq->name, "1.d1.stb"); + int32_t contLen = sizeof(SDropStbMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropStbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_STB; + SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(contLen); + strcpy(pReq->name, "1.d1.stb"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d20c680fad3066e03a0ba6d93ab577b48e959755 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SUT_SRC) +add_library(sut STATIC ${SUT_SRC}) +target_link_libraries( + sut + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories( + sut + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp deleted file mode 100644 index 7cf469f8e253f0354b74cbdda7d2e8cf6cc8a2c5..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "deploy.h" - -void initLog(const char* path) { - dDebugFlag = 207; - vDebugFlag = 0; - mDebugFlag = 207; - cDebugFlag = 0; - jniDebugFlag = 0; - tmrDebugFlag = 0; - uDebugFlag = 143; - rpcDebugFlag = 0; - odbcDebugFlag = 0; - qDebugFlag = 0; - wDebugFlag = 0; - sDebugFlag = 0; - tsdbDebugFlag = 0; - cqDebugFlag = 0; - tscEmbeddedInUtil = 1; - - taosRemoveDir(path); - taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - } -} - -void* runServer(void* param) { - SServer* pServer = (SServer*)param; - while (1) { - taosMsleep(100); - pthread_testcancel(); - } -} - -void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - pOption->sver = 1; - pOption->numOfCores = 1; - pOption->numOfSupportMnodes = 1; - pOption->numOfSupportVnodes = 1; - pOption->numOfSupportQnodes = 1; - pOption->statusInterval = 1; - pOption->numOfThreadsPerCore = 1; - pOption->ratioOfQueryCores = 1; - pOption->maxShellConns = 1000; - pOption->shellActivityTimer = 30; - pOption->serverPort = port; - strcpy(pOption->dataDir, path); - snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp); -} - -SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - taosMkDir(path); - - SDnodeOpt option = {0}; - initOption(&option, path, fqdn, port, firstEp); - - SDnode* pDnode = dndInit(&option); - ASSERT(pDnode); - - SServer* pServer = (SServer*)calloc(1, sizeof(SServer)); - ASSERT(pServer); - - pServer->pDnode = pDnode; - pServer->threadId = taosCreateThread(runServer, pServer); - ASSERT(pServer->threadId); - - return pServer; -} - -SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - taosRemoveDir(path); - return startServer(path, fqdn, port, firstEp); -} - -void stopServer(SServer* pServer) { - if (pServer == NULL) return; - if (pServer->threadId != NULL) { - taosDestoryThread(pServer->threadId); - } - - if (pServer->pDnode != NULL) { - dndCleanup(pServer->pDnode); - pServer->pDnode = NULL; - } -} - -void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SClient* pClient = (SClient*)parent; - pClient->pRsp = pMsg; - uInfo("response:%s from dnode, pCont:%p contLen:%d code:0x%X", taosMsg[pMsg->msgType], pMsg->pCont, pMsg->contLen, - pMsg->code); - tsem_post(&pClient->sem); -} - -SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port) { - SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); - ASSERT(pClient); - - char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = (char*)"DND-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processClientRsp; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = 30 * 1000; - rpcInit.user = (char*)user; - rpcInit.ckey = (char*)"key"; - rpcInit.parent = pClient; - rpcInit.secret = (char*)secretEncrypt; - rpcInit.parent = pClient; - // rpcInit.spi = 1; - - pClient->clientRpc = rpcOpen(&rpcInit); - ASSERT(pClient->clientRpc); - - tsem_init(&pClient->sem, 0, 0); - strcpy(pClient->fqdn, fqdn); - pClient->port = port; - - return pClient; -} - -void dropClient(SClient* pClient) { - tsem_destroy(&pClient->sem); - rpcClose(pClient->clientRpc); -} - -void sendMsg(SClient* pClient, SRpcMsg* pMsg) { - SEpSet epSet = {0}; - epSet.inUse = 0; - epSet.numOfEps = 1; - epSet.port[0] = pClient->port; - memcpy(epSet.fqdn[0], pClient->fqdn, TSDB_FQDN_LEN); - - rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); - tsem_wait(&pClient->sem); -} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h deleted file mode 100644 index 88d9b06fbb52ca421bec4a100fb5849bc2626b1f..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include "os.h" - -#include "dnode.h" -#include "taosmsg.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "trpc.h" -#include "tthread.h" -#include "ulog.h" -#include "tdataformat.h" - -typedef struct { - SDnode* pDnode; - pthread_t* threadId; -} SServer; - -typedef struct { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; - void* clientRpc; - SRpcMsg* pRsp; - tsem_t sem; -} SClient; - -void initLog(const char* path); -SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); -SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); -void stopServer(SServer* pServer); -SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); -void dropClient(SClient* pClient); -void sendMsg(SClient* pClient, SRpcMsg* pMsg); - diff --git a/source/dnode/mgmt/impl/test/sut/inc/base.h b/source/dnode/mgmt/impl/test/sut/inc/base.h new file mode 100644 index 0000000000000000000000000000000000000000..b0de6f5c0c2b878bf57b7eede971126642959156 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/inc/base.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TEST_BASE_H_ +#define _TD_TEST_BASE_H_ + +#include +#include "os.h" + +#include "dnode.h" +#include "taosmsg.h" +#include "tconfig.h" +#include "tdataformat.h" +#include "tglobal.h" +#include "tnote.h" +#include "trpc.h" +#include "tthread.h" +#include "ulog.h" + +#include "client.h" +#include "server.h" + +class Testbase { + public: + void Init(const char* path, int16_t port); + void Cleanup(); + void Restart(); + SRpcMsg* SendMsg(int8_t msgType, void* pCont, int32_t contLen); + + private: + void InitLog(const char* path); + + private: + TestServer server; + TestClient client; + int32_t connId; + + public: + void SendShowMetaMsg(int8_t showType, const char* db); + void SendShowRetrieveMsg(); + + STableMetaMsg* GetShowMeta(); + SRetrieveTableRsp* GetRetrieveRsp(); + + int32_t GetMetaNum(); + const char* GetMetaTbName(); + int32_t GetMetaColId(int32_t index); + int8_t GetMetaType(int32_t index); + int32_t GetMetaBytes(int32_t index); + const char* GetMetaName(int32_t index); + + const char* GetShowName(); + int32_t GetShowRows(); + int8_t GetShowInt8(); + int16_t GetShowInt16(); + int32_t GetShowInt32(); + int64_t GetShowInt64(); + int64_t GetShowTimestamp(); + const char* GetShowBinary(int32_t len); + + private: + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +#define CHECK_META(tbName, numOfColumns) \ + { \ + EXPECT_EQ(test.GetMetaNum(), numOfColumns); \ + EXPECT_STREQ(test.GetMetaTbName(), tbName); \ + } + +#define CHECK_SCHEMA(colId, type, bytes, colName) \ + { \ + EXPECT_EQ(test.GetMetaType(colId), type); \ + EXPECT_EQ(test.GetMetaBytes(colId), bytes); \ + EXPECT_STREQ(test.GetMetaName(colId), colName); \ + } + +#define CheckBinary(val, len) \ + { EXPECT_STREQ(test.GetShowBinary(len), val); } + +#define CheckInt8(val) \ + { EXPECT_EQ(test.GetShowInt8(), val); } + +#define CheckInt16(val) \ + { EXPECT_EQ(test.GetShowInt16(), val); } + +#define CheckInt32(val) \ + { EXPECT_EQ(test.GetShowInt32(), val); } + +#define CheckInt64(val) \ + { EXPECT_EQ(test.GetShowInt64(), val); } + +#define CheckTimestamp() \ + { EXPECT_GT(test.GetShowTimestamp(), 0); } + +#define IgnoreBinary(len) \ + { test.GetShowBinary(len); } + +#define IgnoreInt8() \ + { test.GetShowInt8(); } + +#define IgnoreInt16() \ + { test.GetShowInt16(); } + +#define IgnoreInt32() \ + { test.GetShowInt32(); } + +#define IgnoreInt64() \ + { test.GetShowInt64(); } + +#define IgnoreTimestamp() \ + { test.GetShowTimestamp(); } + +#endif /* _TD_TEST_BASE_H_ */ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/impl/test/sut/inc/client.h new file mode 100644 index 0000000000000000000000000000000000000000..e1e430318e891d2cf8c91740ebbc9d8d5de183cb --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/inc/client.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TEST_CLIENT_H_ +#define _TD_TEST_CLIENT_H_ + +class TestClient { + public: + bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); + void Cleanup(); + + SRpcMsg* SendMsg(SRpcMsg* pMsg); + void SetRpcRsp(SRpcMsg* pRsp); + tsem_t* GetSem(); + + private: + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + void* clientRpc; + SRpcMsg* pRsp; + tsem_t sem; +}; + +#endif /* _TD_TEST_CLIENT_H_ */ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/impl/test/sut/inc/server.h new file mode 100644 index 0000000000000000000000000000000000000000..aa7b37f22118ca0ca4081235624903dc010f85af --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/inc/server.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TEST_SERVER_H_ +#define _TD_TEST_SERVER_H_ + +class TestServer { + public: + bool Start(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + void Stop(); + void Restart(); + + private: + SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + bool DoStart(); + + private: + SDnode* pDnode; + pthread_t* threadId; + char path[PATH_MAX]; + char fqdn[TSDB_FQDN_LEN]; + char firstEp[TSDB_EP_LEN]; + uint16_t port; +}; + +#endif /* _TD_TEST_SERVER_H_ */ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp new file mode 100644 index 0000000000000000000000000000000000000000..59e99b1e1d9b740ac78079d2c899dd00e36b141b --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +void Testbase::InitLog(const char* path) { + dDebugFlag = 207; + vDebugFlag = 0; + mDebugFlag = 207; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 0; + uDebugFlag = 143; + rpcDebugFlag = 0; + odbcDebugFlag = 0; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + + taosRemoveDir(path); + taosMkDir(path); + + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } +} + +void Testbase::Init(const char* path, int16_t port) { + char fqdn[] = "localhost"; + char firstEp[TSDB_EP_LEN] = {0}; + snprintf(firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + + InitLog("/tmp/tdlog"); + server.Start(path, fqdn, port, firstEp); + client.Init("root", "taosdata", fqdn, port); + taosMsleep(1100); +} + +void Testbase::Cleanup() { + server.Stop(); + client.Cleanup(); +} + +void Testbase::Restart() { server.Restart(); } + +SRpcMsg* Testbase::SendMsg(int8_t msgType, void* pCont, int32_t contLen) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pCont; + rpcMsg.contLen = contLen; + rpcMsg.msgType = msgType; + + return client.SendMsg(&rpcMsg); +} + +void Testbase::SendShowMetaMsg(int8_t showType, const char* db) { + int32_t contLen = sizeof(SShowMsg); + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(contLen); + pShow->type = showType; + strcpy(pShow->db, db); + + SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW, pShow, contLen); + SShowRsp* pShowRsp = (SShowRsp*)pMsg->pCont; + + ASSERT(pShowRsp != nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; +} + +int32_t Testbase::GetMetaColId(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->colId = htonl(pSchema->colId); + return pSchema->colId; +} + +int8_t Testbase::GetMetaType(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + return pSchema->type; +} + +int32_t Testbase::GetMetaBytes(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htonl(pSchema->bytes); + return pSchema->bytes; +} + +const char* Testbase::GetMetaName(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + return pSchema->name; +} + +int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } + +const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } + +void Testbase::SendShowRetrieveMsg() { + int32_t contLen = sizeof(SRetrieveTableMsg); + + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(contLen); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW_RETRIEVE, pRetrieve, contLen); + pRetrieveRsp = (SRetrieveTableRsp*)pMsg->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + pData = pRetrieveRsp->data; + pos = 0; +} + +const char* Testbase::GetShowName() { return pMeta->tbFname; } + +int8_t Testbase::GetShowInt8() { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + return data; +} + +int16_t Testbase::GetShowInt16() { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + return data; +} + +int32_t Testbase::GetShowInt32() { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + return data; +} + +int64_t Testbase::GetShowInt64() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + return data; +} + +int64_t Testbase::GetShowTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + return data; +} + +const char* Testbase::GetShowBinary(int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + return data; +} + +int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } + +STableMetaMsg* Testbase::GetShowMeta() { return pMeta; } + +SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fce3da15073e830908390fe95d9b98f0fcc5bada --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + TestClient* client = (TestClient*)parent; + client->SetRpcRsp(pMsg); + uInfo("response:%s from dnode, code:0x%x", taosMsg[pMsg->msgType], pMsg->code); + tsem_post(client->GetSem()); +} + +void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; + +tsem_t* TestClient::GetSem() { return &sem; } + +bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = (char*)"DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processClientRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = 30 * 1000; + rpcInit.user = (char*)user; + rpcInit.ckey = (char*)"key"; + rpcInit.parent = this; + rpcInit.secret = (char*)secretEncrypt; + // rpcInit.spi = 1; + + clientRpc = rpcOpen(&rpcInit); + ASSERT(clientRpc); + + tsem_init(&sem, 0, 0); + strcpy(this->fqdn, fqdn); + this->port = port; + + return true; +} + +void TestClient::Cleanup() { + tsem_destroy(&sem); + rpcClose(clientRpc); +} + +SRpcMsg* TestClient::SendMsg(SRpcMsg* pMsg) { + SEpSet epSet = {0}; + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN); + + rpcSendRequest(clientRpc, &epSet, pMsg, NULL); + tsem_wait(&sem); + + return pRsp; +} diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e96f7deaf5e1c9466a5b94a29cc8d1ad25c8f6d4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +void* serverLoop(void* param) { + while (1) { + taosMsleep(100); + pthread_testcancel(); + } +} + +SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeOpt option = {0}; + option.sver = 1; + option.numOfCores = 1; + option.numOfSupportMnodes = 1; + option.numOfSupportVnodes = 1; + option.numOfSupportQnodes = 1; + option.statusInterval = 1; + option.numOfThreadsPerCore = 1; + option.ratioOfQueryCores = 1; + option.maxShellConns = 1000; + option.shellActivityTimer = 30; + option.serverPort = port; + strcpy(option.dataDir, path); + snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); + return option; +} + +bool TestServer::DoStart() { + SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); + taosMkDir(path); + + pDnode = dndInit(&option); + if (pDnode != NULL) { + return false; + } + + threadId = taosCreateThread(serverLoop, NULL); + if (threadId != NULL) { + return false; + } + return true; +} + +void TestServer::Restart() { + uInfo("start all server"); + Stop(); + DoStart(); + uInfo("all server is running"); +} + +bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + strcpy(this->path, path); + strcpy(this->fqdn, fqdn); + this->port = port; + strcpy(this->firstEp, firstEp); + + taosRemoveDir(path); + return DoStart(); +} + +void TestServer::Stop() { + if (threadId != NULL) { + taosDestoryThread(threadId); + threadId = NULL; + } + + if (pDnode != NULL) { + dndCleanup(pDnode); + pDnode = NULL; + } +} diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index 0d3215a4d0445ab99c8180b31716b12841e21c34..5068660eba1fea5964664819b9a51686cbde2adf 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_user "") - -target_sources(dnode_test_user - PRIVATE - "user.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. USER_SRC) +add_executable(dnode_test_user ${USER_SRC}) target_link_libraries( dnode_test_user - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_user - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 32309b106e8f6aef905fb8fd8fb5c72e6d28ecda..a1810e7adaaa7011775dd2e9fbcc3892cce720e6 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -9,168 +9,34 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestUser : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9140"; - pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9140); - taosMsleep(300); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_user", 9140); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestUser::pServer; -SClient* DndTestUser::pClient; -int32_t DndTestUser::connId; +Testbase DndTestUser::test; TEST_F(DndTestUser, 01_ShowUser) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendThenCheckShowRetrieveMsg(1); CheckBinary("root", TSDB_USER_LEN); CheckBinary("super", 10); CheckTimestamp(); @@ -179,39 +45,35 @@ TEST_F(DndTestUser, 01_ShowUser) { TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + int32_t contLen = sizeof(SCreateUserMsg); + + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + int32_t contLen = sizeof(SCreateUserMsg); + + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u2"); strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(3); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 3); + CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); @@ -226,22 +88,23 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); { - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + int32_t contLen = sizeof(SAlterUserMsg); + + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(3); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 3); + CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); @@ -256,21 +119,22 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); { - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); + int32_t contLen = sizeof(SDropUserMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); CheckBinary("super", 10); @@ -281,19 +145,14 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); // restart - stopServer(pServer); - pServer = NULL; - - uInfo("start all server"); + test.Restart(); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9140"; - pServer = startServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); - uInfo("all server is running"); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); CheckBinary("super", 10); diff --git a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt index 5670f9dbf2c4ac2c3609494f50bd4c82bd77fef1..b864b0593c26d15c4468c381f95c1ac337e1174f 100644 --- a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_vgroup "") - -target_sources(dnode_test_vgroup - PRIVATE - "vgroup.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. VGROUP_SRC) +add_executable(dnode_test_vgroup ${VGROUP_SRC}) target_link_libraries( dnode_test_vgroup - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_vgroup - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index 9149be9cd838be6ddd708a1fa56e85f6dc5f2dae..c534fcbabe96b28af1a917bbca718ad6210070b5 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -9,177 +9,28 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestVgroup : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9150"; - pServer = CreateServer("/tmp/dnode_test_vgroup", fqdn, 9150, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9150); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vgroup", 9150); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestVgroup::pServer; -SClient* DndTestVgroup::pClient; -int32_t DndTestVgroup::connId; +Testbase DndTestVgroup::test; TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); + int32_t contLen = sizeof(SCreateVnodeMsg); + + SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -209,13 +60,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_VNODE_IN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -223,7 +68,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(sizeof(SAlterVnodeMsg)); + int32_t contLen = sizeof(SAlterVnodeMsg); + + SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -253,13 +100,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_VNODE_IN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_VNODE_IN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -267,7 +108,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(sizeof(SDropVnodeMsg)); + int32_t contLen = sizeof(SDropVnodeMsg); + + SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -278,8 +121,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { rpcMsg.contLen = sizeof(SDropVnodeMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_VNODE_IN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bf657fd27aabdbb22ed3b1c01fb85052b60cb506..a4b73e0f3f2547bf6a013eae16c927e1658a0ad7 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -23,29 +23,25 @@ #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 #define SUBQUERY_INFO_SIZE 6 -#define QUERY_STREAM_SAVE_SIZE 20 +#define QUERY_SAVE_SIZE 20 typedef struct { - char user[TSDB_USER_LEN]; - char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc - int32_t pid; // pid of app that invokes taosc - int64_t appStartTime; // app start time - int32_t id; - int8_t killed; - int8_t align; - uint16_t port; - uint32_t ip; - int64_t stime; - int64_t lastAccess; - int32_t queryId; - int32_t streamId; - int32_t numOfQueries; - int32_t numOfStreams; - SStreamDesc *pStreams; - SQueryDesc *pQueries; + int32_t id; + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int64_t appStartTimeMs; // app start time + int32_t pid; // pid of app that invokes taosc + uint32_t ip; + uint16_t port; + int8_t killed; + int64_t loginTimeMs; + int64_t lastAccessTimeMs; + int32_t queryId; + int32_t numOfQueries; + SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); +static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -54,16 +50,12 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg); static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg); static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg); -static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg); static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -79,7 +71,6 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_STREAM, mndProcessKillStreamMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); @@ -88,9 +79,6 @@ int32_t mndInitProfile(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndGetQueryMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndGetStreamMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStreams); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); return 0; } @@ -103,46 +91,43 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { +static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); + if (startTime == 0) startTime = taosGetTimestampMs(); - SConnObj connObj = {.pid = pid, - .appStartTime = startTime, - .id = connId, + SConnObj connObj = {.id = connId, + .appStartTimeMs = startTime, + .pid = pid, + .ip = pInfo->clientIp, + .port = pInfo->clientPort, .killed = 0, - .port = port, - .ip = ip, - .stime = taosGetTimestampMs(), - .lastAccess = 0, + .loginTimeMs = taosGetTimestampMs(), + .lastAccessTimeMs = 0, .queryId = 0, - .streamId = 0, .numOfQueries = 0, - .numOfStreams = 0, - .pStreams = NULL, .pQueries = NULL}; - connObj.lastAccess = connObj.stime; - tstrncpy(connObj.user, user, TSDB_USER_LEN); + connObj.lastAccessTimeMs = connObj.loginTimeMs; + tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, user, terrstr()); + mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, pInfo->user, terrstr()); return NULL; } else { - mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, user); + mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, pInfo->user); return pConn; } } static void mndFreeConn(SConnObj *pConn) { tfree(pConn->pQueries); - tfree(pConn->pStreams); mTrace("conn:%d, data:%p destroyed", pConn->id, pConn); } @@ -156,7 +141,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { } int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; - pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn); return pConn; @@ -219,7 +204,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); + SConnObj *pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -254,16 +239,14 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { pConn->numOfQueries = 0; - pConn->numOfStreams = 0; int32_t numOfQueries = htonl(pMsg->numOfQueries); - int32_t numOfStreams = htonl(pMsg->numOfStreams); if (numOfQueries > 0) { if (pConn->pQueries == NULL) { - pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); + pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } - pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries); + pConn->numOfQueries = MIN(QUERY_SAVE_SIZE, numOfQueries); int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { @@ -271,19 +254,6 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { } } - if (numOfStreams > 0) { - if (pConn->pStreams == NULL) { - pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); - } - - pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams); - - int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); - if (saveSize > 0 && pConn->pStreams != NULL) { - memcpy(pConn->pStreams, pMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); - } - } - return TSDB_CODE_SUCCESS; } @@ -303,7 +273,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); + pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -343,11 +313,6 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { pRsp->killConnection = 1; } - if (pConn->streamId != 0) { - pRsp->streamId = htonl(pConn->streamId); - pConn->streamId = 0; - } - if (pConn->queryId != 0) { pRsp->queryId = htonl(pConn->queryId); pConn->queryId = 0; @@ -395,37 +360,6 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { } } -static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - - SKillStreamMsg *pKill = pMsg->rpcMsg.pCont; - int32_t connId = htonl(pKill->connId); - int32_t streamId = htonl(pKill->streamId); - mDebug("kill stream msg is received, streamId:%d", streamId); - - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); - if (pConn == NULL) { - mError("connId:%d, failed to kill streamId:%d, conn not exist", connId, streamId); - terrno = TSDB_CODE_MND_INVALID_CONN_ID; - return -1; - } else { - mInfo("connId:%d, streamId:%d is killed by user:%s", connId, streamId, pMsg->user); - pConn->streamId = streamId; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); - return TSDB_CODE_SUCCESS; - } -} - static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -525,6 +459,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -567,12 +502,12 @@ static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pConn->stime; + *(int64_t *)pWrite = pConn->loginTimeMs; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pConn->lastAccess < pConn->stime) pConn->lastAccess = pConn->stime; - *(int64_t *)pWrite = pConn->lastAccess; + if (pConn->lastAccessTimeMs < pConn->loginTimeMs) pConn->lastAccessTimeMs = pConn->loginTimeMs; + *(int64_t *)pWrite = pConn->lastAccessTimeMs; cols++; numOfRows++; @@ -623,7 +558,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 24; + pShow->bytes[cols] = 22 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); @@ -693,6 +628,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = 1000000; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -790,6 +726,7 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, } } + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -798,173 +735,3 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } - -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - - int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "streamId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "destination"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "exec"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; - strcpy(pSchema[cols].name, "time(us)"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cycles"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htonl(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = 1000000; - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - - return 0; -} - -static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - int32_t numOfRows = 0; - SConnObj *pConn = NULL; - int32_t cols = 0; - char *pWrite; - void *pIter; - char ipStr[TSDB_IPv4ADDR_LEN + 6]; - - while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); - if (pConn == NULL) { - pShow->pIter = pIter; - break; - } - - if (numOfRows + pConn->numOfStreams >= rows) { - mndCancelGetNextConn(pMnode, pIter); - break; - } - - pShow->pIter = pIter; - for (int32_t i = 0; i < pConn->numOfStreams; ++i) { - SStreamDesc *pDesc = pConn->pStreams + i; - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->streamId); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pConn->id); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConn->ip), pConn->port); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->ctime); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->stime); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->useconds); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = (int32_t)htobe64(pDesc->num); - cols++; - - numOfRows++; - } - } - - pShow->numOfReads += numOfRows; - return numOfRows; -} - -static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index db6777ebf8129d41fac21b95d373246e96127bb8..41f09e85a99b55637a44388d2bd1106ba43b961d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -138,7 +138,7 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { ShowMetaFp metaFp = pMgmt->metaFps[type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta msg:%s since no message handle", mndShowStr(type)); + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); return -1; } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 636cf0a9a84453458ec0d06b84231f926a4637b5..027532bbb1d1514604fc8277862e764cec13ec51 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -11,3 +11,4 @@ add_subdirectory(executor) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) +add_subdirectory(qworker) diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c index 48662f34436d05c9c2ef873bb52d6e1aa515f4f3..97dccb5c7b69405f91934362262e339985a5060d 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/executor/src/tfilter.c @@ -1175,7 +1175,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g void* key = NULL; len = 0; - taosHashGetKey((SHashObj *)data, p, &key, &len); + taosHashGetKey(p, &key, &len); void *fdata = NULL; if (IS_VAR_DATA_TYPE(type)) { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 72415a971674790f1b8ce5d9f21261c82eb16b89..2584a847ff2343db27a2f2acaf00a3f68c7aad68 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -42,11 +42,11 @@ typedef struct SIndexStat { struct SIndex { #ifdef USE_LUCENE - index_t *index; + index_t* index; #endif - void * cache; - void * tindex; - SHashObj *colObj; // < field name, field id> + void* cache; + void* tindex; + SHashObj* colObj; // < field name, field id> int64_t suid; // current super table id, -1 is normal table int colId; // field id allocated to cache @@ -58,7 +58,7 @@ struct SIndex { struct SIndexOpts { #ifdef USE_LUCENE - void *opts; + void* opts; #endif #ifdef USE_INVERTED_INDEX @@ -69,7 +69,7 @@ struct SIndexOpts { struct SIndexMultiTermQuery { EIndexOperatorType opera; - SArray * query; + SArray* query; }; // field and key; @@ -77,14 +77,14 @@ typedef struct SIndexTerm { int64_t suid; SIndexOperOnColumn operType; // oper type, add/del/update uint8_t colType; // term data type, str/interger/json - char * colName; + char* colName; int32_t nColName; - char * colVal; + char* colVal; int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { - SIndexTerm * term; + SIndexTerm* term; EIndexQueryType qType; } SIndexTermQuery; diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 9d5083d23d14c3e2490f3228b28672cdf8417f08..fb4f478ae99fdc7654536a98d67d5e3d8ac407fa 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -34,19 +34,18 @@ extern "C" { typedef struct IndexCache { T_REF_DECLARE() - SSkipList *skiplist; + SSkipList* skiplist; } IndexCache; // -IndexCache *indexCacheCreate(); +IndexCache* indexCacheCreate(); -void indexCacheDestroy(void *cache); +void indexCacheDestroy(void* cache); -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch( - void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 67438c092b3a99df876e445e2abf71d1b1a126c1..3c572787fc4b4fa17f7e870b5af44aa95b3ef115 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -40,10 +40,10 @@ typedef struct FstBoundWithData { } FstBoundWithData; typedef struct FstStreamBuilder { - Fst * fst; - AutomationCtx * aut; - FstBoundWithData *min; - FstBoundWithData *max; + Fst* fst; + AutomationCtx* aut; + FstBoundWithData* min; + FstBoundWithData* max; } FstStreamBuilder, FstStreamWithStateBuilder; typedef struct FstRange { @@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; -FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data); -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data); +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice); +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound); +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound); typedef struct FstOutput { bool null; @@ -71,46 +71,46 @@ typedef struct FstOutput { * TODO: simple function name */ typedef struct FstUnFinishedNodes { - SArray *stack; // } FstUnFinishedNodes; + SArray* stack; // } FstUnFinishedNodes; } FstUnFinishedNodes; #define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) -FstUnFinishedNodes *fstUnFinishedNodesCreate(); -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); -FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); -FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); -FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); +FstUnFinishedNodes* fstUnFinishedNodesCreate(); +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* node); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* node, Output out); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* node, CompiledAddr addr); +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* node, FstSlice bs, Output out); +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs); +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr); +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes); -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out); typedef struct FstBuilder { - FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`. - FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry * registry; // A map of finished nodes. + FstCountingWriter* wrt; // The FST raw data is written directly to `wtr`. + FstUnFinishedNodes* unfinished; // The stack of unfinished nodes + FstRegistry* registry; // A map of finished nodes. FstSlice last; // The last word added CompiledAddr lastAddr; // The address of the last compiled node uint64_t len; // num of keys added } FstBuilder; -FstBuilder *fstBuilderCreate(void *w, FstType ty); +FstBuilder* fstBuilderCreate(void* w, FstType ty); -void fstBuilderDestroy(FstBuilder *b); -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); -void * fstBuilerIntoInner(FstBuilder *b); -void fstBuilderFinish(FstBuilder *b); -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); +void fstBuilderDestroy(FstBuilder* b); +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in); +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate); +void* fstBuilerIntoInner(FstBuilder* b); +void fstBuilderFinish(FstBuilder* b); +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup); +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn); typedef struct FstTransitions { - FstNode *node; + FstNode* node; FstRange range; } FstTransitions; @@ -121,56 +121,55 @@ typedef struct FstState { uint8_t val; } FstState; -FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr); +FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr); FstState fstStateCreate(State state); // compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp); +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn); +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node); // set_comm_input -void fstStateSetCommInput(FstState *state, uint8_t inp); +void fstStateSetCommInput(FstState* state, uint8_t inp); // comm_input -uint8_t fstStateCommInput(FstState *state, bool *null); +uint8_t fstStateCommInput(FstState* state, bool* null); // input_len -uint64_t fstStateInputLen(FstState *state); +uint64_t fstStateInputLen(FstState* state); // end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data); -uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans( - FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); +uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); // input -uint8_t fstStateInput(FstState *state, FstNode *node); -uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +uint8_t fstStateInput(FstState* state, FstNode* node); +uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // trans_addr -CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); -CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); +CompiledAddr fstStateTransAddr(FstState* state, FstNode* node); +CompiledAddr fstStateTransAddrForAnyTrans(FstState* state, FstNode* node, uint64_t i); // sizes -PackSizes fstStateSizes(FstState *state, FstSlice *data); +PackSizes fstStateSizes(FstState* state, FstSlice* data); // Output -Output fstStateOutput(FstState *state, FstNode *node); -Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +Output fstStateOutput(FstState* state, FstNode* node); +Output fstStateOutputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // anyTrans specify function -void fstStateSetFinalState(FstState *state, bool yes); -bool fstStateIsFinalState(FstState *state); -void fstStateSetStateNtrans(FstState *state, uint8_t n); +void fstStateSetFinalState(FstState* state, bool yes); +bool fstStateIsFinalState(FstState* state); +void fstStateSetStateNtrans(FstState* state, uint8_t n); // state_ntrans -uint8_t fstStateStateNtrans(FstState *state, bool *null); -uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); -uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); -uint64_t fstStateNtransLen(FstState *state); -uint64_t fstStateNtrans(FstState *state, FstSlice *slice); -Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); -uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); +uint8_t fstStateStateNtrans(FstState* state, bool* null); +uint64_t fstStateTotalTransSize(FstState* state, uint64_t version, PackSizes size, uint64_t nTrans); +uint64_t fstStateTransIndexSize(FstState* state, uint64_t version, uint64_t nTrans); +uint64_t fstStateNtransLen(FstState* state); +uint64_t fstStateNtrans(FstState* state, FstSlice* slice); +Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) @@ -187,13 +186,13 @@ typedef struct FstLastTransition { * TODO: simple function name */ typedef struct FstBuilderNodeUnfinished { - FstBuilderNode * node; - FstLastTransition *last; + FstBuilderNode* node; + FstLastTransition* last; } FstBuilderNodeUnfinished; -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* node, CompiledAddr addr); -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* node, Output out); /* * FstNode and helper function @@ -224,18 +223,18 @@ typedef struct FstNode { // Return the address of this node. #define FST_NODE_ADDR(node) node->start -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); -void fstNodeDestroy(FstNode *fstNode); +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* data); +void fstNodeDestroy(FstNode* fstNode); -FstTransitions fstNodeTransitionIter(FstNode *node); -FstTransitions *fstNodeTransitions(FstNode *node); -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); +FstTransitions fstNodeTransitionIter(FstNode* node); +FstTransitions* fstNodeTransitions(FstNode* node); +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* res); +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res); +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res); -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode); -FstSlice fstNodeAsSlice(FstNode *node); +FstSlice fstNodeAsSlice(FstNode* node); // ops @@ -244,8 +243,8 @@ typedef struct FstIndexedValue { uint64_t value; } FstIndexedValue; -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); -void fstLastTransitionDestroy(FstLastTransition *trn); +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out); +void fstLastTransitionDestroy(FstLastTransition* trn); typedef struct FstMeta { uint64_t version; @@ -256,75 +255,74 @@ typedef struct FstMeta { } FstMeta; typedef struct Fst { - FstMeta * meta; - FstSlice *data; // - FstNode * root; // + FstMeta* meta; + FstSlice* data; // + FstNode* root; // } Fst; // refactor simple function -Fst *fstCreate(FstSlice *data); -void fstDestroy(Fst *fst); +Fst* fstCreate(FstSlice* data); +void fstDestroy(Fst* fst); -bool fstGet(Fst *fst, FstSlice *b, Output *out); -FstNode * fstGetNode(Fst *fst, CompiledAddr); -FstNode * fstGetRoot(Fst *fst); -FstType fstGetType(Fst *fst); -CompiledAddr fstGetRootAddr(Fst *fst); -Output fstEmptyFinalOutput(Fst *fst, bool *null); -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); +bool fstGet(Fst* fst, FstSlice* b, Output* out); +FstNode* fstGetNode(Fst* fst, CompiledAddr); +FstNode* fstGetRoot(Fst* fst); +FstType fstGetType(Fst* fst); +CompiledAddr fstGetRootAddr(Fst* fst); +Output fstEmptyFinalOutput(Fst* fst, bool* null); +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx); -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx); // into stream to expand later -StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb); +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb); -bool fstVerify(Fst *fst); +bool fstVerify(Fst* fst); // refactor this function -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr); typedef struct StreamState { - FstNode * node; + FstNode* node; uint64_t trans; FstOutput out; - void * autState; + void* autState; } StreamState; -void streamStateDestroy(void *s); +void streamStateDestroy(void* s); typedef struct StreamWithState { - Fst * fst; - AutomationCtx * aut; - SArray * inp; + Fst* fst; + AutomationCtx* aut; + SArray* inp; FstOutput emptyOutput; - SArray * stack; // - FstBoundWithData *endAt; + SArray* stack; // + FstBoundWithData* endAt; } StreamWithState; typedef struct StreamWithStateResult { FstSlice data; FstOutput out; - void * state; + void* state; } StreamWithStateResult; -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); -void swsResultDestroy(StreamWithStateResult *result); +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state); +void swsResultDestroy(StreamWithStateResult* result); -typedef void *(*StreamCallback)(void *); -StreamWithState *streamWithStateCreate( - Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max); +typedef void* (*StreamCallback)(void*); +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max); -void streamWithStateDestroy(StreamWithState *sws); +void streamWithStateDestroy(StreamWithState* sws); -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min); -StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback); -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut); // set up bound range // refator, simple code by marco -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 4e5309bf07abfb653a1ff69d0ef789b5bfce6739..c7269eda0f30a845c5b9a1810a5a4aea4baa56eb 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx; typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef struct StartWith { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } StartWith; typedef struct Complement { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } Complement; // automation typedef struct AutomationCtx { AutomationType type; - void * stdata; - char * data; + void* stdata; + char* data; } AutomationCtx; typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; @@ -48,27 +48,27 @@ typedef struct StartWithStateValue { ValueType type; union { int val; - char * ptr; - SArray *arr; + char* ptr; + SArray* arr; // add more type }; } StartWithStateValue; -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); -void startWithStateValueDestroy(void *sv); +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val); +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv); +void startWithStateValueDestroy(void* sv); typedef struct AutomationFunc { - void *(*start)(AutomationCtx *ctx); - bool (*isMatch)(AutomationCtx *ctx, void *); - bool (*canMatch)(AutomationCtx *ctx, void *data); - bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); - void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte); - void *(*acceptEof)(AutomationCtx *ct, void *state); + void* (*start)(AutomationCtx* ctx); + bool (*isMatch)(AutomationCtx* ctx, void*); + bool (*canMatch)(AutomationCtx* ctx, void* data); + bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state); + void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte); + void* (*acceptEof)(AutomationCtx* ct, void* state); } AutomationFunc; -AutomationCtx *automCtxCreate(void *data, AutomationType atype); -void automCtxDestroy(AutomationCtx *ctx); +AutomationCtx* automCtxCreate(void* data, AutomationType atype); +void automCtxDestroy(AutomationCtx* ctx); extern AutomationFunc automFuncs[]; #ifdef __cplusplus diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index b8a0384fb87584bbd8ad7da9193b19cf7165b772..ba0bd52e10280cd0139a6dc852b61c7f75b91a77 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -28,9 +28,10 @@ static char tmpFile[] = "./index"; typedef enum WriterType { TMemory, TFile } WriterType; typedef struct WriterCtx { - int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*flush)(struct WriterCtx *ctx); + int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*flush)(struct WriterCtx* ctx); + int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); WriterType type; union { struct { @@ -39,41 +40,42 @@ typedef struct WriterCtx { } file; struct { int32_t capa; - char * buf; + char* buf; } mem; }; int32_t offset; int32_t limit; } WriterCtx; -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoFlush(WriterCtx *ctx); +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); +static int writeCtxDoFlush(WriterCtx* ctx); -WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); -void writerCtxDestroy(WriterCtx *w); +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); +void writerCtxDestroy(WriterCtx* w); typedef uint32_t CheckSummer; typedef struct FstCountingWriter { - void * wrt; // wrap any writer that counts and checksum bytes written + void* wrt; // wrap any writer that counts and checksum bytes written uint64_t count; CheckSummer summer; } FstCountingWriter; -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len); -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len); -int fstCountingWriterFlush(FstCountingWriter *write); +int fstCountingWriterFlush(FstCountingWriter* write); -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write); -FstCountingWriter *fstCountingWriterCreate(void *wtr); -void fstCountingWriterDestroy(FstCountingWriter *w); +FstCountingWriter* fstCountingWriterCreate(void* wtr); +void fstCountingWriterDestroy(FstCountingWriter* w); -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes); +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n); #define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index a2041fad408745dbc502bd1c206dcaff29f64de7..32de7ea6211d4ab449fc6ef0d0fe8355ff380299 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -36,20 +36,20 @@ typedef struct FstTransition { typedef struct FstBuilderNode { bool isFinal; Output finalOutput; - SArray *trans; // + SArray* trans; // } FstBuilderNode; -FstBuilderNode *fstBuilderNodeDefault(); +FstBuilderNode* fstBuilderNodeDefault(); -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src); -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src); // bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, // CompiledAddr lastAddr, CompiledAddr startAddr); -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2); -void fstBuilderNodeDestroy(FstBuilderNode *node); +void fstBuilderNodeDestroy(FstBuilderNode* node); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 1b0922c7241cfa1c5f92863f459b5cf999baac99..f94a21d8dd7fd6887f531793af74f2d5023f66b7 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -25,7 +25,7 @@ extern "C" { typedef struct FstRegistryCell { CompiledAddr addr; - FstBuilderNode *node; + FstBuilderNode* node; } FstRegistryCell; #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) @@ -45,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState; typedef struct FstRegistryEntry { FstRegistryEntryState state; CompiledAddr addr; - FstRegistryCell * cell; + FstRegistryCell* cell; } FstRegistryEntry; // Registry relation function typedef struct FstRegistry { - SArray * table; // + SArray* table; // uint64_t tableSize; // num of rows uint64_t mruSize; // num of columns } FstRegistry; // -FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); -void fstRegistryDestroy(FstRegistry *registry); +FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); +void fstRegistryDestroy(FstRegistry* registry); -FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); -void fstRegistryEntryDestroy(FstRegistryEntry *entry); +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode); +void fstRegistryEntryDestroy(FstRegistryEntry* entry); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index c41977e6377f97a007c221cc63bf28182cae40ba..71a378584d401e966d7b2966ff215916c40c8232 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD; // uint8_t commonIdx(uint8_t v, uint8_t max); uint8_t packSize(uint64_t n); -uint64_t unpackUint64(uint8_t *ch, uint8_t sz); +uint64_t unpackUint64(uint8_t* ch, uint8_t sz); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr); typedef struct FstString { - uint8_t *data; + uint8_t* data; uint32_t len; int32_t ref; } FstString; typedef struct FstSlice { - FstString *str; + FstString* str; int32_t start; int32_t end; } FstSlice; -FstSlice fstSliceCreate(uint8_t *data, uint64_t len); -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); -bool fstSliceIsEmpty(FstSlice *s); -int fstSliceCompare(FstSlice *s1, FstSlice *s2); -void fstSliceDestroy(FstSlice *s); -uint8_t *fstSliceData(FstSlice *s, int32_t *sz); +FstSlice fstSliceCreate(uint8_t* data, uint64_t len); +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end); +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end); +bool fstSliceIsEmpty(FstSlice* s); +int fstSliceCompare(FstSlice* s1, FstSlice* s2); +void fstSliceDestroy(FstSlice* s); +uint8_t* fstSliceData(FstSlice* s, int32_t* sz); #define FST_SLICE_LEN(s) (s->end - s->start + 1) diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 7c6261aacf7ad14d58c883443cfadc16d50269d4..b19b887e9f86a482634b8c0877c1e4c079ac9f44 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -26,84 +26,93 @@ extern "C" { #endif -// tfile header -// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| -// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| +// tfile header content +// |<---suid--->|<---version--->|<-------colName------>|<---type-->|<--fstOffset->| +// |<-uint64_t->|<---int32_t--->|<--TSDB_COL_NAME_LEN-->|<-uint8_t->|<---int32_t-->| -typedef struct TFileReadHeader { +#pragma pack(push, 1) +typedef struct TFileHeader { uint64_t suid; int32_t version; - char colName[128]; // + char colName[TSDB_COL_NAME_LEN]; // uint8_t colType; -} TFileReadHeader; + int32_t fstOffset; +} TFileHeader; +#pragma pack(pop) -#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); -#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) +#define TFILE_HEADER_SIZE (sizeof(TFileHeader)) +#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t)) +//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) typedef struct TFileCacheKey { - uint64_t suid; - uint8_t colType; - int32_t version; - const char *colName; - int32_t nColName; + uint64_t suid; + uint8_t colType; + char* colName; + int32_t nColName; } TFileCacheKey; // table cache // refactor to LRU cache later typedef struct TFileCache { - SHashObj *tableCache; + SHashObj* tableCache; int16_t capacity; // add more param } TFileCache; typedef struct TFileWriter { - FstBuilder *fb; - WriterCtx * ctx; + FstBuilder* fb; + WriterCtx* ctx; + TFileHeader header; + uint32_t offset; } TFileWriter; +// multi reader and single write typedef struct TFileReader { T_REF_DECLARE() - Fst * fst; - WriterCtx * ctx; - TFileReadHeader header; + Fst* fst; + WriterCtx* ctx; + TFileHeader header; } TFileReader; typedef struct IndexTFile { - char * path; - TFileCache * cache; - TFileWriter *tw; + char* path; + TFileCache* cache; + TFileWriter* tw; } IndexTFile; typedef struct TFileWriterOpt { uint64_t suid; int8_t colType; - char * colName; + char* colName; int32_t nColName; int32_t version; } TFileWriterOpt; typedef struct TFileReaderOpt { uint64_t suid; - char * colName; + char* colName; int32_t nColName; } TFileReaderOpt; // tfile cache, manage tindex reader -TFileCache * tfileCacheCreate(const char *path); -void tfileCacheDestroy(TFileCache *tcache); -TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); -void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); +TFileCache* tfileCacheCreate(const char* path); +void tfileCacheDestroy(TFileCache* tcache); +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); -TFileReader *tfileReaderCreate(); -void TFileReaderDestroy(TFileReader *reader); +TFileReader* tfileReaderCreate(WriterCtx* ctx); +void tfileReaderDestroy(TFileReader* reader); +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); -TFileWriter *tfileWriterCreate(const char *suid, const char *colName); -void tfileWriterDestroy(TFileWriter *tw); +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); +void tfileWriterDestroy(TFileWriter* tw); +int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterFinish(TFileWriter* tw); // -IndexTFile *indexTFileCreate(const char *path); -int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); +IndexTFile* indexTFileCreate(const char* path); +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index f708e71f57364a673d3f1c7327f932464702350c..21c5ca155b3bb252429be2dd6c30a344694b168a 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -19,29 +19,29 @@ extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void *)buf, (void *)key->mem, len); \ + memcpy((void*)buf, (void*)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(var) == sizeof(type)); \ - memcpy((void *)buf, (void *)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type)); \ + memcpy((void*)buf, (void*)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void *)buf, (void *)var, len); \ + memcpy((void*)buf, (void*)var, len); \ buf += len; \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index cc68324017f44960d14537c782806a8351c65f89..e88cecfd623c86d39ae28c9906c976f60c27927f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -23,9 +23,9 @@ #include "lucene++/Lucene_c.h" #endif -static int uidCompare(const void *a, const void *b) { - uint64_t u1 = *(uint64_t *)a; - uint64_t u2 = *(uint64_t *)b; +static int uidCompare(const void* a, const void* b) { + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; if (u1 == u2) { return 0; } else { @@ -40,25 +40,26 @@ typedef struct SIdxColInfo { static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); -static int indexMergeCacheIntoTindex(SIndex *sIdx); +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); +static int indexMergeCacheIntoTindex(SIndex* sIdx); -static void indexInterResultsDestroy(SArray *results); -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); +static void indexInterResultsDestroy(SArray* results); +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); -int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { +int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); - SIndex *sIdx = calloc(1, sizeof(SIndex)); + SIndex* sIdx = calloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE - index_t *index = index_open(path); + index_t* index = index_open(path); sIdx->index = index; #endif - sIdx->cache = (void *)indexCacheCreate(); +#ifdef USE_INVERTED_INDEX + sIdx->cache = (void*)indexCacheCreate(); sIdx->tindex = NULL; sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colId = 1; @@ -67,9 +68,13 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { *index = sIdx; return 0; +#endif + + *index = NULL; + return -1; } -void indexClose(SIndex *sIdx) { +void indexClose(SIndex* sIdx) { #ifdef USE_LUCENE index_close(sIdex->index); sIdx->index = NULL; @@ -84,16 +89,16 @@ void indexClose(SIndex *sIdx) { return; } -int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { +int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { #ifdef USE_LUCENE - index_document_t *doc = index_document_create(); + index_document_t* doc = index_document_create(); char buf[16] = {0}; sprintf(buf, "%d", uid); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); + SIndexTerm* p = taosArrayGetP(fVals, i); + index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1); } index_document_add(doc, NULL, 0, buf, strlen(buf), 0); @@ -106,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { // TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm * p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); if (fi == NULL) { SIdxColInfo tfi = {.colId = index->colId}; index->cVersion++; @@ -120,8 +125,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm * p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); assert(fi != NULL); int32_t colId = fi->colId; int32_t version = index->cVersion; @@ -134,18 +139,18 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { return 0; } -int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { +int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { #ifdef USE_LUCENE EIndexOperatorType opera = multiQuerys->opera; int nQuery = taosArrayGetSize(multiQuerys->query); - char **fields = malloc(sizeof(char *) * nQuery); - char **keys = malloc(sizeof(char *) * nQuery); - int * types = malloc(sizeof(int) * nQuery); + char** fields = malloc(sizeof(char*) * nQuery); + char** keys = malloc(sizeof(char*) * nQuery); + int* types = malloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { - SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); - SIndexTerm * term = p->field_value; + SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); + SIndexTerm* term = p->field_value; fields[i] = calloc(1, term->nKey + 1); keys[i] = calloc(1, term->nVal + 1); @@ -154,9 +159,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result memcpy(keys[i], term->val, term->nVal); types[i] = (int)(p->type); } - int *tResult = NULL; + int* tResult = NULL; int tsz = 0; - index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); + index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); @@ -174,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result #ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys - SArray *interResults = taosArrayInit(4, POINTER_BYTES); + SArray* interResults = taosArrayInit(4, POINTER_BYTES); int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); - SArray * tResult = NULL; + SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); + SArray* tResult = NULL; indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void *)&tResult); + taosArrayPush(interResults, (void*)&tResult); } indexMergeFinalResults(interResults, opera, result); indexInterResultsDestroy(interResults); @@ -189,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result return 1; } -int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { +int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { #ifdef USE_INVERTED_INDEX #endif return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts){ +int indexRebuild(SIndex* index, SIndexOpts* opts){ #ifdef USE_INVERTED_INDEX #endif } -SIndexOpts *indexOptsCreate() { +SIndexOpts* indexOptsCreate() { #ifdef USE_LUCENE #endif return NULL; } -void indexOptsDestroy(SIndexOpts *opts){ +void indexOptsDestroy(SIndexOpts* opts){ #ifdef USE_LUCENE #endif } /* @@ -214,8 +219,8 @@ void indexOptsDestroy(SIndexOpts *opts){ * */ -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } @@ -223,23 +228,28 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { - SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); + SIndexTermQuery* p = (SIndexTermQuery*)taosArrayGet(pQuery->query, i); indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) { +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, - int32_t nColName, const char *colVal, int32_t nColVal) { - SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn oper, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal) { + SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); if (t == NULL) { return NULL; } @@ -248,30 +258,32 @@ SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy t->operType = oper; t->colType = colType; - t->colName = (char *)calloc(1, nColName + 1); + t->colName = (char*)calloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char *)calloc(1, nColVal + 1); + t->colVal = (char*)calloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } -void indexTermDestroy(SIndexTerm *p) { +void indexTermDestroy(SIndexTerm* p) { free(p->colName); free(p->colVal); free(p); } -SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } +SIndexMultiTerm* indexMultiTermCreate() { + return taosArrayInit(4, sizeof(SIndexTerm*)); +} -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { taosArrayPush(terms, &term); return 0; } -void indexMultiTermDestroy(SIndexMultiTerm *terms) { +void indexMultiTermDestroy(SIndexMultiTerm* terms) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { - SIndexTerm *p = taosArrayGetP(terms, i); + SIndexTerm* p = taosArrayGetP(terms, i); indexTermDestroy(p); } taosArrayDestroy(terms); @@ -280,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) { void indexInit() { // do nothing } -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) { int32_t version = -1; int16_t colId = -1; - SIdxColInfo *colInfo = NULL; + SIdxColInfo* colInfo = NULL; - SIndexTerm *term = query->term; - const char *colName = term->colName; + SIndexTerm* term = query->term; + const char* colName = term->colName; int32_t nColName = term->nColName; pthread_mutex_lock(&sIdx->mtx); @@ -319,23 +331,24 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result } return 0; } -static void indexInterResultsDestroy(SArray *results) { +static void indexInterResultsDestroy(SArray* results) { if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { - SArray *p = taosArrayGetP(results, i); + SArray* p = taosArrayGetP(results, i); taosArrayDestroy(p); } taosArrayDestroy(results); } -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray *first = taosArrayGetP(interResults, 0); + SArray* first = taosArrayGetP(interResults, 0); taosArraySort(first, uidCompare); taosArrayRemoveDuplicate(first, uidCompare, NULL); + if (oType == MUST) { // just one column index, enhance later taosArrayAddAll(fResults, first); @@ -350,7 +363,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType } return 0; } -static int indexMergeCacheIntoTindex(SIndex *sIdx) { +static int indexMergeCacheIntoTindex(SIndex* sIdx) { if (sIdx == NULL) { return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index dd8a8bcbb6e31031429e7181793fafc65cc004c8..6170642707434b4daaf22bc14c9f8058f841e611 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,14 +20,15 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later // ref index_cache.h:22 -#define CACHE_KEY_LEN(p) \ - (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \ - sizeof(p->operType)) +#define CACHE_KEY_LEN(p) \ + (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) -static char * getIndexKey(const void *pData) { return NULL; } -static int32_t compareKey(const void *l, const void *r) { - char *lp = (char *)l; - char *rp = (char *)r; +static char* getIndexKey(const void* pData) { + return NULL; +} +static int32_t compareKey(const void* l, const void* r) { + char* lp = (char*)l; + char* rp = (char*)r; // skip total len, not compare int32_t ll, rl; // len @@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) { int16_t lf, rf; // field id memcpy(&lf, lp, sizeof(lf)); memcpy(&rf, rp, sizeof(rf)); - if (lf != rf) { - return lf < rf ? -1 : 1; - } + if (lf != rf) { return lf < rf ? -1 : 1; } lp += sizeof(lf); rp += sizeof(rf); @@ -89,41 +88,41 @@ static int32_t compareKey(const void *l, const void *r) { int32_t lv, rv; memcpy(&lv, lp, sizeof(lv)); memcpy(&rv, rp, sizeof(rv)); - if (lv != rv) { - return lv > rv ? -1 : 1; - } + if (lv != rv) { return lv > rv ? -1 : 1; } + lp += sizeof(lv); rp += sizeof(rv); // not care item type return 0; } -IndexCache *indexCacheCreate() { - IndexCache *cache = calloc(1, sizeof(IndexCache)); - cache->skiplist = tSkipListCreate( - MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); +IndexCache* indexCacheCreate() { + IndexCache* cache = calloc(1, sizeof(IndexCache)); + if (cache == NULL) { + indexError("failed to create index cache"); + return NULL; + } + cache->skiplist = + tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); return cache; } -void indexCacheDestroy(void *cache) { - IndexCache *pCache = cache; - if (pCache == NULL) { - return; - } +void indexCacheDestroy(void* cache) { + IndexCache* pCache = cache; + if (pCache == NULL) { return; } tSkipListDestroy(pCache->skiplist); free(pCache); } -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { - if (cache == NULL) { - return -1; - } +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { + if (cache == NULL) { return -1; } - IndexCache *pCache = cache; + IndexCache* pCache = cache; // encode data int32_t total = CACHE_KEY_LEN(term); - char * buf = calloc(1, total); - char * p = buf; + + char* buf = calloc(1, total); + char* p = buf; SERIALIZE_VAR_TO_BUF(p, total, int32_t); SERIALIZE_VAR_TO_BUF(p, colId, int16_t); @@ -137,30 +136,31 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, SERIALIZE_MEM_TO_BUF(p, term, operType); - tSkipListPut(pCache->skiplist, (void *)buf); + tSkipListPut(pCache->skiplist, (void*)buf); return 0; // encode end } -int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { - IndexCache *pCache = cache; +int indexCacheDel(void* cache, int32_t fieldId, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + IndexCache* pCache = cache; return 0; } -int indexCacheSearch( - void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { - if (cache == NULL) { - return -1; - } - IndexCache * pCache = cache; - SIndexTerm * term = query->term; +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; + SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; int32_t keyLen = CACHE_KEY_LEN(term); - char *buf = calloc(1, keyLen); + char* buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { + // } else if (qtype == QUERY_PREFIX) { + // } else if (qtype == QUERY_SUFFIX) { + // } else if (qtype == QUERY_REGEX) { + // } return 0; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index adb002c2b74301403e599a71935a66c7d5a6759b..18514cd0d5498eba9a31d8d28b197557bdf23bd8 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -19,43 +19,39 @@ #include "tchecksum.h" #include "tcoding.h" -static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); } -static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); return nBytes; } -FstUnFinishedNodes *fstUnFinishedNodesCreate() { - FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); - if (nodes == NULL) { - return NULL; - } +FstUnFinishedNodes* fstUnFinishedNodesCreate() { + FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); + if (nodes == NULL) { return NULL; } - nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); + nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -void unFinishedNodeDestroyElem(void *elem) { - FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished *)elem; +void unFinishedNodeDestroyElem(void* elem) { + FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); free(b->last); b->last = NULL; } -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { - if (nodes == NULL) { - return; - } +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { + if (nodes == NULL) { return; } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); } -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -63,56 +59,53 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); } -FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { assert(taosArrayGetSize(nodes->stack) == 1); - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); assert(un->last == NULL); return un->node; } -FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); // free(un->last); // TODO add func FstLastTransitionFree() // un->last = NULL; return un->node; } -FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); assert(un->last == NULL); return un->node; } -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); un->node->isFinal = true; un->node->finalOutput = out; // un->node->trans = NULL; } -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); fstBuilderNodeUnfinishedLastCompiled(un, addr); } -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { - FstSlice *s = &bs; - if (fstSliceIsEmpty(s)) { - return; - } +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { + FstSlice* s = &bs; + if (fstSliceIsEmpty(s)) { return; } size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; // trn->out = out; int32_t len = 0; - uint8_t *data = fstSliceData(s, &len); + uint8_t* data = fstSliceData(s, &len); un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -120,7 +113,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // trn->inp = s->data[i]; // trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); + FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; taosArrayPush(nodes->stack, &un); @@ -128,15 +121,15 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output fstUnFinishedNodesPushEmpty(nodes, true); } -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { - FstSlice *s = &bs; +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { + FstSlice* s = &bs; size_t ssz = taosArrayGetSize(node->stack); // stack size uint64_t count = 0; int32_t lsz; // data len - uint8_t *data = fstSliceData(s, &lsz); + uint8_t* data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); if (un->last->inp == data[i]) { count++; } else { @@ -145,19 +138,19 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) } return count; } -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) { - FstSlice *s = &bs; +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { + FstSlice* s = &bs; size_t lsz = (size_t)(s->end - s->start + 1); // data len size_t ssz = taosArrayGetSize(node->stack); // stack size *out = in; uint64_t i = 0; for (i = 0; i < lsz && i < ssz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); - FstLastTransition *t = un->last; + FstLastTransition* t = un->last; uint64_t addPrefix = 0; - uint8_t * data = fstSliceData(s, NULL); + uint8_t* data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { uint64_t commPrefix = MIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; @@ -169,7 +162,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, } if (addPrefix != 0) { if (i + 1 < ssz) { - FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); + FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); } } @@ -177,13 +170,11 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, return i; } -FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { +FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; - if (addr == EMPTY_ADDRESS) { - return fs; - } + if (addr == EMPTY_ADDRESS) { return fs; } - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); uint8_t v = data[addr]; uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { @@ -197,17 +188,19 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { return fs; } -static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000}, - {.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}}; +static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, + {.state = OneTrans, .val = 0b10000000}, + {.state = AnyTrans, .val = 0b00000000}, + {.state = EmptyFinal, .val = 0b00000000}}; // debug -static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; FstState fstStateCreate(State state) { uint8_t idx = (uint8_t)state; return fstStateDict[idx]; } // compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp) { +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { FstState s = fstStateCreate(OneTransNext); fstStateSetCommInput(&s, inp); @@ -221,7 +214,7 @@ void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uin // w->write_all(&[s.val]) return; } -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn) { +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { Output out = trn->out; uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); @@ -229,20 +222,18 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); - fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); + fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); bool null = false; uint8_t inp = fstStateCommInput(&st, &null); - if (null == true) { - fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp)); - } - fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val)); + if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } + fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); return; } -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { size_t sz = taosArrayGetSize(node->trans); assert(sz <= 256); @@ -252,7 +243,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // finalOutput.is_zero() bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); tSize = MAX(tSize, packDeltaSize(addr, t->addr)); oSize = MAX(oSize, packSize(t->out)); anyOuts = anyOuts || (t->out != 0); @@ -272,21 +263,19 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil fstStateSetStateNtrans(&st, (uint8_t)sz); if (anyOuts) { - if (FST_BUILDER_NODE_IS_FINAL(node)) { - fstCountingWriterPackUintIn(w, node->finalOutput, oSize); - } + if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); } } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); fstPackDeltaIn(w, addr, t->addr, tSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstCountingWriterWrite(w, (char *)&t->inp, 1); + FstTransition* t = taosArrayGet(node->trans, i); + fstCountingWriterWrite(w, (char*)&t->inp, 1); // fstPackDeltaIn(w, addr, t->addr, tSize); } if (sz > TRANS_INDEX_THRESHOLD) { @@ -294,20 +283,20 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; ///} for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); index[t->inp] = i; // fstPackDeltaIn(w, addr, t->addr, tSize); } - fstCountingWriterWrite(w, (char *)index, 256); + fstCountingWriterWrite(w, (char*)index, 256); free(index); } - fstCountingWriterWrite(w, (char *)&packSizes, 1); + fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; fstStateStateNtrans(&st, &null); if (null == true) { @@ -316,17 +305,17 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // encoded in the state byte. uint8_t v = 1; if (sz == 256) { - fstCountingWriterWrite(w, (char *)&v, 1); + fstCountingWriterWrite(w, (char*)&v, 1); } else { - fstCountingWriterWrite(w, (char *)&sz, 1); + fstCountingWriterWrite(w, (char*)&sz, 1); } } - fstCountingWriterWrite(w, (char *)(&(st.val)), 1); + fstCountingWriterWrite(w, (char*)(&(st.val)), 1); return; } // set_comm_input -void fstStateSetCommInput(FstState *s, uint8_t inp) { +void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; @@ -335,7 +324,7 @@ void fstStateSetCommInput(FstState *s, uint8_t inp) { } // comm_input -uint8_t fstStateCommInput(FstState *s, bool *null) { +uint8_t fstStateCommInput(FstState* s, bool* null) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; if (v == 0) { @@ -348,7 +337,7 @@ uint8_t fstStateCommInput(FstState *s, bool *null) { // input_len -uint64_t fstStateInputLen(FstState *s) { +uint64_t fstStateInputLen(FstState* s) { assert(s->state == OneTransNext || s->state == OneTrans); bool null = false; fstStateCommInput(s, &null); @@ -356,17 +345,16 @@ uint64_t fstStateInputLen(FstState *s) { } // end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState *s, FstSlice *data) { +uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { assert(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } -uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { +uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { assert(s->state == OneTrans); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } -uint64_t fstStateEndAddrForAnyTrans( - FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size @@ -374,29 +362,29 @@ uint64_t fstStateEndAddrForAnyTrans( - finalOsize; // final output } // input -uint8_t fstStateInput(FstState *s, FstNode *node) { +uint8_t fstStateInput(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); - uint8_t * data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return null == false ? inp : data[-1]; } -uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return data[at]; } // trans_addr -CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { +CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (s->state == OneTransNext) { return (CompiledAddr)(node->end) - 1; } else { @@ -406,23 +394,23 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { - tSizes; // refactor error logic - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + i, tSizes, node->end); } } -CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - - node->nTrans - (i * tSizes) - tSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans - + (i * tSizes) - tSizes; + uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + at, tSizes, node->end); } // sizes -PackSizes fstStateSizes(FstState *s, FstSlice *slice) { +PackSizes fstStateSizes(FstState* s, FstSlice* slice) { assert(s->state == OneTrans || s->state == AnyTrans); uint64_t i; if (s->state == OneTrans) { @@ -431,33 +419,29 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return (PackSizes)(*(data + i)); } // Output -Output fstStateOutput(FstState *s, FstNode *node) { +Output fstStateOutput(FstState* s, FstNode* node) { assert(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackUint64(data + i, oSizes); } -Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; - uint8_t * data = fstSliceData(slice, NULL); + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; + uint8_t* data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; @@ -466,27 +450,23 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { // anyTrans specify function -void fstStateSetFinalState(FstState *s, bool yes) { +void fstStateSetFinalState(FstState* s, bool yes) { assert(s->state == AnyTrans); - if (yes) { - s->val |= 0b01000000; - } + if (yes) { s->val |= 0b01000000; } return; } -bool fstStateIsFinalState(FstState *s) { +bool fstStateIsFinalState(FstState* s) { assert(s->state == AnyTrans); return (s->val & 0b01000000) == 0b01000000; } -void fstStateSetStateNtrans(FstState *s, uint8_t n) { +void fstStateSetStateNtrans(FstState* s, uint8_t n) { assert(s->state == AnyTrans); - if (n <= 0b00111111) { - s->val = (s->val & 0b11000000) | n; - } + if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } return; } // state_ntrans -uint8_t fstStateStateNtrans(FstState *s, bool *null) { +uint8_t fstStateStateNtrans(FstState* s, bool* null) { assert(s->state == AnyTrans); *null = false; uint8_t n = s->val & 0b00111111; @@ -496,58 +476,52 @@ uint8_t fstStateStateNtrans(FstState *s, bool *null) { } return n; } -uint64_t fstStateTotalTransSize(FstState *s, uint64_t version, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { assert(s->state == AnyTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } -uint64_t fstStateTransIndexSize(FstState *s, uint64_t version, uint64_t nTrans) { +uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { assert(s->state == AnyTrans); return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } -uint64_t fstStateNtransLen(FstState *s) { +uint64_t fstStateNtransLen(FstState* s) { assert(s->state == AnyTrans); bool null = false; fstStateStateNtrans(s, &null); return null == true ? 1 : 0; } -uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { +uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { bool null = false; uint8_t n = fstStateStateNtrans(s, &null); - if (null != true) { - return n; - } + if (null != true) { return n; } int32_t len; - uint8_t *data = fstSliceData(slice, &len); + uint8_t* data = fstSliceData(slice, &len); n = data[len - 2]; // n = data[slice->end - 1]; // data[data.len() - 2] return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, // then it is encoded in the state byte } -Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { +Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { - return 0; - } + if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackUint64(data + at, (uint8_t)oSizes); } -uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { +uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); int32_t dlen = 0; - uint8_t *data = fstSliceData(slice, &dlen); + uint8_t* data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; // uint64_t i = slice->data[slice->start + at + b]; - if (i >= node->nTrans) { - *null = true; - } + if (i >= node->nTrans) { *null = true; } return i; } else { uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size @@ -555,7 +529,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint64_t end = start + node->nTrans; FstSlice t = fstSliceCopy(slice, start, end - 1); int32_t len = 0; - uint8_t *data = fstSliceData(&t, &len); + uint8_t* data = fstSliceData(&t, &len); int i = 0; for (; i < len; i++) { uint8_t v = data[i]; @@ -564,20 +538,16 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { return node->nTrans - i - 1; // bug } } - if (i == len) { - *null = true; - } + if (i == len) { *null = true; } fstSliceDestroy(&t); } } // fst node function -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { - FstNode *n = (FstNode *)malloc(sizeof(FstNode)); - if (n == NULL) { - return NULL; - } +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { + FstNode* n = (FstNode*)malloc(sizeof(FstNode)); + if (n == NULL) { return NULL; } FstState st = fstStateCreateFrom(slice, addr); @@ -625,27 +595,24 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->nTrans = nTrans; n->sizes = sz; - n->finalOutput = - fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); } return n; } // debug state transition -static const char *fstNodeState(FstNode *node) { - FstState *st = &node->state; +static const char* fstNodeState(FstNode* node) { + FstState* st = &node->state; return fstStateStr[st->state]; } -void fstNodeDestroy(FstNode *node) { +void fstNodeDestroy(FstNode* node) { fstSliceDestroy(&node->data); free(node); } -FstTransitions *fstNodeTransitions(FstNode *node) { - FstTransitions *t = malloc(sizeof(FstTransitions)); - if (NULL == t) { - return NULL; - } +FstTransitions* fstNodeTransitions(FstNode* node) { + FstTransitions* t = malloc(sizeof(FstTransitions)); + if (NULL == t) { return NULL; } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; t->node = node; @@ -653,9 +620,9 @@ FstTransitions *fstNodeTransitions(FstNode *node) { } // Returns the transition at index `i`. -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { trn->inp = fstStateInput(st, node); trn->out = 0; @@ -675,9 +642,9 @@ bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { } // Returns the transition address of the `i`th transition -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { assert(i == 0); fstStateTransAddr(st, node); @@ -696,9 +663,9 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { // Finds the `i`th transition corresponding to the given input byte. // If no transition for this byte exists, then `false` is returned. -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { if (fstStateInput(st, node) == b) { *res = 0; @@ -723,7 +690,7 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { return s; } -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode) { +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { size_t sz = taosArrayGetSize(builderNode->trans); assert(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { @@ -732,7 +699,7 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr fstStateCompileForAnyTrans(w, addr, builderNode); // AnyTrans->Compile(w, addr, node); } else { - FstTransition *tran = taosArrayGet(builderNode->trans, 0); + FstTransition* tran = taosArrayGet(builderNode->trans, 0); if (tran->addr == lastAddr && tran->out == 0) { fstStateCompileForOneTransNext(w, addr, tran->inp); // OneTransNext::compile(w, lastAddr, tran->inp); @@ -746,15 +713,13 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr return true; } -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); } -FstBuilder *fstBuilderCreate(void *w, FstType ty) { - FstBuilder *b = malloc(sizeof(FstBuilder)); - if (NULL == b) { - return b; - } +FstBuilder* fstBuilderCreate(void* w, FstType ty) { + FstBuilder* b = malloc(sizeof(FstBuilder)); + if (NULL == b) { return b; } b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); @@ -764,7 +729,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { b->len = 0; char buf64[8] = {0}; - void *pBuf64 = buf64; + void* pBuf64 = buf64; taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); @@ -775,10 +740,8 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { return b; } -void fstBuilderDestroy(FstBuilder *b) { - if (b == NULL) { - return; - } +void fstBuilderDestroy(FstBuilder* b) { + if (b == NULL) { return; } fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); @@ -787,19 +750,19 @@ void fstBuilderDestroy(FstBuilder *b) { free(b); } -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { OrderType t = fstBuilderCheckLastKey(b, bs, true); if (t == Ordered) { // add log info fstBuilderInsertOutput(b, bs, in); return true; } - indexInfo("key must be ordered"); + indexInfo("fst write key must be ordered"); return false; } -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { - FstSlice *s = &bs; +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { + FstSlice* s = &bs; if (fstSliceIsEmpty(s)) { b->len = 1; fstUnFinishedNodesSetRootOutput(b->unfinished, in); @@ -828,8 +791,8 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { return; } -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { - FstSlice *input = &bs; +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { + FstSlice* input = &bs; if (fstSliceIsEmpty(&b->last)) { fstSliceDestroy(&b->last); // deep copy or not @@ -847,10 +810,10 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } return Ordered; } -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { - FstBuilderNode *bn = NULL; + FstBuilderNode* bn = NULL; if (addr == NONE_ADDRESS) { bn = fstUnFinishedNodesPopEmpty(b->unfinished); } else { @@ -865,11 +828,11 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; } -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { return EMPTY_ADDRESS; } - FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); + FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); if (entry->state == FOUND) { CompiledAddr ret = entry->addr; fstRegistryEntryDestroy(entry); @@ -879,23 +842,21 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); - if (entry->state == NOTFOUND) { - FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); - } + if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } fstRegistryEntryDestroy(entry); return b->lastAddr; } -void *fstBuilderInsertInner(FstBuilder *b) { +void* fstBuilderInsertInner(FstBuilder* b) { fstBuilderCompileFrom(b, 0); - FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); + FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); fstBuilderNodeDestroy(rootNode); char buf64[8] = {0}; - void *pBuf64 = buf64; + void* pBuf64 = buf64; taosEncodeFixedU64(&pBuf64, b->len); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); @@ -904,7 +865,7 @@ void *fstBuilderInsertInner(FstBuilder *b) { fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); char buf32[4] = {0}; - void * pBuf32 = buf32; + void* pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); taosEncodeFixedU32(&pBuf32, sum); fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); @@ -914,31 +875,31 @@ void *fstBuilderInsertInner(FstBuilder *b) { // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); } +void fstBuilderFinish(FstBuilder* b) { + fstBuilderInsertInner(b); +} -FstSlice fstNodeAsSlice(FstNode *node) { - FstSlice *slice = &node->data; +FstSlice fstNodeAsSlice(FstNode* node) { + FstSlice* slice = &node->data; FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); return s; } -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - if (trn == NULL) { - return NULL; - } +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { + FstLastTransition* trn = malloc(sizeof(FstLastTransition)); + if (trn == NULL) { return NULL; } trn->inp = inp; trn->out = out; return trn; } -void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); } -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { - FstLastTransition *trn = unNode->last; - if (trn == NULL) { - return; - } +void fstLastTransitionDestroy(FstLastTransition* trn) { + free(trn); +} +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { + FstLastTransition* trn = unNode->last; + if (trn == NULL) { return; } FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; taosArrayPush(unNode->node->trans, &t); fstLastTransitionDestroy(trn); @@ -946,36 +907,28 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp return; } -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, Output out) { - if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { - unNode->node->finalOutput += out; - } +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { + if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(unNode->node->trans, i); + FstTransition* trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } - if (unNode->last) { - unNode->last->out += out; - } + if (unNode->last) { unNode->last->out += out; } return; } -Fst *fstCreate(FstSlice *slice) { +Fst* fstCreate(FstSlice* slice) { int32_t slen; - char * buf = fstSliceData(slice, &slen); - if (slen < 36) { - return NULL; - } + char* buf = fstSliceData(slice, &slen); + if (slen < 36) { return NULL; } uint64_t len = slen; uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); skip += sizeof(version); - if (version == 0 || version > VERSION) { - return NULL; - } + if (version == 0 || version > VERSION) { return NULL; } uint64_t type; taosDecodeFixedU64(buf + skip, &type); @@ -993,15 +946,11 @@ Fst *fstCreate(FstSlice *slice) { len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) - Fst *fst = (Fst *)calloc(1, sizeof(Fst)); - if (fst == NULL) { - return NULL; - } + Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + if (fst == NULL) { return NULL; } - fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { - goto FST_CREAT_FAILED; - } + fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + if (NULL == fst->meta) { goto FST_CREAT_FAILED; } fst->meta->version = version; fst->meta->rootAddr = rootAddr; @@ -1009,7 +958,7 @@ Fst *fstCreate(FstSlice *slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice *s = calloc(1, sizeof(FstSlice)); + FstSlice* s = calloc(1, sizeof(FstSlice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); fst->data = s; @@ -1019,7 +968,7 @@ FST_CREAT_FAILED: free(fst->meta); free(fst); } -void fstDestroy(Fst *fst) { +void fstDestroy(Fst* fst) { if (fst) { free(fst->meta); fstSliceDestroy(fst->data); @@ -1028,20 +977,18 @@ void fstDestroy(Fst *fst) { free(fst); } -bool fstGet(Fst *fst, FstSlice *b, Output *out) { - FstNode *root = fstGetRoot(fst); +bool fstGet(Fst* fst, FstSlice* b, Output* out) { + FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; - uint8_t *data = fstSliceData(b, &len); + uint8_t* data = fstSliceData(b, &len); - SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; - if (false == fstNodeFindInput(root, inp, &res)) { - return false; - } + if (false == fstNodeFindInput(root, inp, &res)) { return false; } FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); @@ -1056,7 +1003,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); @@ -1066,30 +1013,36 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { return true; } -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } -StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { - if (sb == NULL) { - return NULL; - } +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { + return fstStreamBuilderCreate(fst, ctx); +} +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { + if (sb == NULL) { return NULL; } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { + return fstStreamBuilderCreate(fst, ctx); +} -FstNode *fstGetRoot(Fst *fst) { - if (fst->root != NULL) { - return fst->root; - } +FstNode* fstGetRoot(Fst* fst) { + if (fst->root != NULL) { return fst->root; } CompiledAddr rAddr = fstGetRootAddr(fst); fst->root = fstGetNode(fst, rAddr); return fst->root; } -FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); } -FstType fstGetType(Fst *fst) { return fst->meta->ty; } -CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; } +FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { + return fstNodeCreate(fst->meta->version, addr, fst->data); +} +FstType fstGetType(Fst* fst) { + return fst->meta->ty; +} +CompiledAddr fstGetRootAddr(Fst* fst) { + return fst->meta->rootAddr; +} -Output fstEmptyFinalOutput(Fst *fst, bool *null) { +Output fstEmptyFinalOutput(Fst* fst, bool* null) { Output res = 0; - FstNode *node = fstGetRoot(fst); + FstNode* node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; res = FST_NODE_FINAL_OUTPUT(node); @@ -1099,23 +1052,19 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { return res; } -bool fstVerify(Fst *fst) { +bool fstVerify(Fst* fst) { uint32_t checkSum = fst->meta->checkSum; int32_t len; - uint8_t *data = fstSliceData(fst->data, &len); + uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data, len)) { - return false; - } + if (!taosCheckChecksumWhole(data, len)) { return false; } return true; } // data bound function -FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { - FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); - if (b == NULL) { - return NULL; - } +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { + FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); + if (b == NULL) { return NULL; } if (data != NULL) { b->data = fstSliceCopy(data, data->start, data->end); @@ -1127,7 +1076,7 @@ FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { return b; } -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { return comp > 0 ? true : false; @@ -1137,7 +1086,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { return true; } } -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { if (bound->type == Unbounded) { return true; } else { @@ -1145,34 +1094,33 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { } } -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; } +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { + return bound->type == Excluded ? false : true; +} -void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } +void fstBoundDestroy(FstBoundWithData* bound) { + free(bound); +} -StreamWithState *streamWithStateCreate( - Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { - StreamWithState *sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { - return NULL; - } +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { + StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + if (sws == NULL) { return NULL; } sws->fst = fst; sws->aut = automation; - sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); + sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); sws->emptyOutput.null = false; sws->emptyOutput.out = 0; - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); sws->endAt = max; streamWithStateSeekMin(sws, min); return sws; } -void streamWithStateDestroy(StreamWithState *sws) { - if (sws == NULL) { - return; - } +void streamWithStateDestroy(StreamWithState* sws) { + if (sws == NULL) { return; } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1180,22 +1128,19 @@ void streamWithStateDestroy(StreamWithState *sws) { free(sws); } -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { - AutomationCtx *aut = sws->aut; +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { + AutomationCtx* aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { - if (fstBoundWithDataIsIncluded(min)) { - sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); - } + if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); } StreamState s = {.node = fstGetRoot(sws->fst), - .trans = 0, - .out = {.null = false, .out = 0}, - .autState = automFuncs[aut->type].start(aut)}; // auto.start callback + .trans = 0, + .out = {.null = false, .out = 0}, + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; } - FstSlice *key = NULL; + FstSlice* key = NULL; bool inclusize = false; - ; if (min->type == Included) { key = &min->data; @@ -1206,13 +1151,13 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { return false; } - FstNode *node = fstGetRoot(sws->fst); + FstNode* node = fstGetRoot(sws->fst); Output out = 0; // void* autState = sws->aut->start(); - void *autState = automFuncs[aut->type].start(aut); + void* autState = automFuncs[aut->type].start(aut); int32_t len; - uint8_t *data = fstSliceData(key, &len); + uint8_t* data = fstSliceData(key, &len); for (uint32_t i = 0; i < len; i++) { uint8_t b = data[i]; uint64_t res = 0; @@ -1220,7 +1165,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { if (null == false) { FstTransition trn; fstNodeGetTransitionAt(node, res, &trn); - void *preState = autState; + void* preState = autState; // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); @@ -1235,13 +1180,11 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { // Since this is a minimum bound, we need to find the // first transition in this node that proceeds the current // input byte. - FstTransitions *trans = fstNodeTransitions(node); + FstTransitions* trans = fstNodeTransitions(node); uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; - if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { - break; - } + if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } } StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; @@ -1251,17 +1194,16 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { - StreamState *s = taosArrayGet(sws->stack, sz - 1); + StreamState* s = taosArrayGet(sws->stack, sz - 1); if (inclusize) { s->trans -= 1; taosArrayPop(sws->inp); } else { - FstNode * n = s->node; + FstNode* n = s->node; uint64_t trans = s->trans; FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = { - .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; + StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); return true; } @@ -1269,48 +1211,44 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } } -StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { - AutomationCtx *aut = sws->aut; +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { + AutomationCtx* aut = sws->aut; FstOutput output = sws->emptyOutput; if (output.null == false) { FstSlice emptySlice = fstSliceCreate(NULL, 0); if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); return NULL; } - void *start = automFuncs[aut->type].start(aut); + void* start = automFuncs[aut->type].start(aut); if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); return swsResultCreate(&s, output, callback(start)); } } - SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); + SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); while (taosArrayGetSize(sws->stack) > 0) { - StreamState *p = (StreamState *)taosArrayPop(sws->stack); + StreamState* p = (StreamState*)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { - if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { - taosArrayPop(sws->inp); - } + if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } streamStateDestroy(p); continue; } FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); Output out = p->out.out + trn.out; - void * nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); - void * tState = callback(nextState); + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); + void* tState = callback(nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - FstNode *nextNode = fstGetNode(sws->fst, trn.addr); + FstNode* nextNode = fstGetNode(sws->fst, trn.addr); taosArrayPush(nodes, &nextNode); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { // void *eofState = sws->aut->acceptEof(nextState); - void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); - if (eofState != NULL) { - isMatch = automFuncs[aut->type].isMatch(aut, eofState); - } + void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); + if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } } StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); @@ -1319,21 +1257,21 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb taosArrayPush(sws->stack, &s2); size_t isz = taosArrayGetSize(sws->inp); - uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { - buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); + buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); return result; @@ -1342,18 +1280,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); return NULL; } -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { - StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); - if (result == NULL) { - return NULL; - } +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { + StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); + if (result == NULL) { return NULL; } result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->out = fOut; @@ -1361,31 +1297,25 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta return result; } -void swsResultDestroy(StreamWithStateResult *result) { - if (NULL == result) { - return; - } +void swsResultDestroy(StreamWithStateResult* result) { + if (NULL == result) { return; } fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); free(result); } -void streamStateDestroy(void *s) { - if (NULL == s) { - return; - } - StreamState *ss = (StreamState *)s; +void streamStateDestroy(void* s) { + if (NULL == s) { return; } + StreamState* ss = (StreamState*)s; fstNodeDestroy(ss->node); // free(s->autoState); } -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { - FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); - if (NULL == b) { - return NULL; - } +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { + FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); + if (NULL == b) { return NULL; } b->fst = fst; b->aut = aut; @@ -1393,17 +1323,15 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { b->max = fstBoundStateCreate(Unbounded, NULL); return b; } -void fstStreamBuilderDestroy(FstStreamBuilder *b) { +void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); tfree(b->min); fstSliceDestroy(&b->max->data); tfree(b->max); free(b); } -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { - if (b == NULL) { - return NULL; - } +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { + if (b == NULL) { return NULL; } if (type == GE) { b->min->type = Included; diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index cf3165709cdb961190d32dd646a54468c6d6ae95..eb4c479c9406d4be367df458235c47f7b87ff9f8 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -15,19 +15,17 @@ #include "index_fst_automation.h" -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } nsv->kind = kind; nsv->type = ty; if (ty == FST_INT) { - nsv->val = *(int *)val; + nsv->val = *(int*)val; } else if (ty == FST_CHAR) { - size_t len = strlen((char *)val); - nsv->ptr = (char *)calloc(1, len + 1); + size_t len = strlen((char*)val); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp } return nsv; } -void startWithStateValueDestroy(void *val) { - StartWithStateValue *sv = (StartWithStateValue *)val; - if (sv == NULL) { - return; - } +void startWithStateValueDestroy(void* val) { + StartWithStateValue* sv = (StartWithStateValue*)val; + if (sv == NULL) { return; } if (sv->type == FST_INT) { // @@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) { } free(sv); } -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } nsv->kind = sv->kind; nsv->type = sv->type; @@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char *)calloc(1, len + 1); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { + // } return nsv; } // prefix query, impl later -static void *prefixStart(AutomationCtx *ctx) { - StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); +static void* prefixStart(AutomationCtx* ctx) { + StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata); return startWithStateValueDump(data); }; -static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue *ssv = (StartWithStateValue *)sv; +static bool prefixIsMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val == strlen(ctx->data); } -static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue *ssv = (StartWithStateValue *)sv; +static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val >= 0; } -static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } -static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { - StartWithStateValue *ssv = (StartWithStateValue *)state; - if (ssv == NULL || ctx == NULL) { - return NULL; - } +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { + return true; +} +static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { + StartWithStateValue* ssv = (StartWithStateValue*)state; + if (ssv == NULL || ctx == NULL) { return NULL; } - char *data = ctx->data; - if (ssv->kind == Done) { - return startWithStateValueCreate(Done, FST_INT, &ssv->val); - } + char* data = ctx->data; + if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; - StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; } else { @@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { } return NULL; } -static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { + return NULL; +} // pattern query, impl later -static void *patternStart(AutomationCtx *ctx) { return NULL; } -static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; } -static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; } -static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } +static void* patternStart(AutomationCtx* ctx) { + return NULL; +} +static bool patternIsMatch(AutomationCtx* ctx, void* data) { + return true; +} +static bool patternCanMatch(AutomationCtx* ctx, void* data) { + return true; +} +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { + return true; +} -static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; } +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { + return NULL; +} -static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { + return NULL; +} AutomationFunc automFuncs[] = { {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, @@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = { // add more search type }; -AutomationCtx *automCtxCreate(void *data, AutomationType atype) { - AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { - return NULL; - } +AutomationCtx* automCtxCreate(void* data, AutomationType atype) { + AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); + if (ctx == NULL) { return NULL; } - StartWithStateValue *sv = NULL; + StartWithStateValue* sv = NULL; if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void *)sv; + ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type } - char * src = (char *)data; + char* src = (char*)data; size_t len = strlen(src); - char * dst = (char *)malloc(len * sizeof(char) + 1); + char* dst = (char*)malloc(len * sizeof(char) + 1); memcpy(dst, src, len); dst[len] = 0; ctx->data = dst; ctx->type = atype; - ctx->stdata = (void *)sv; + ctx->stdata = (void*)sv; return ctx; } -void automCtxDestroy(AutomationCtx *ctx) { +void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); free(ctx->data); free(ctx); diff --git a/source/libs/index/src/index_fst_common.c b/source/libs/index/src/index_fst_common.c index ee3b07713da13008c29c863e164e72f9b56a3cda..2d7586e8d939c8ef6dc80e998e97f3cf0b08f921 100644 --- a/source/libs/index/src/index_fst_common.c +++ b/source/libs/index/src/index_fst_common.c @@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = { }; const char COMMON_INPUTS_INV[] = { - 't', - 'e', - '/', - 'o', - 'a', - 's', - 'r', - 'i', - 'p', - 'c', - 'n', - 'w', - '.', - 'h', - 'l', - 'm', - '-', - 'd', - 'u', - '0', - '1', - '2', - 'g', - '=', - ':', - 'b', - 'f', - '3', - 'y', - '5', - '&', - '_', - '4', - 'v', - '9', - '6', - '7', - '8', - 'k', - '%', - '?', - 'x', - 'C', - 'D', - 'A', - 'S', - 'F', - 'I', - 'B', - 'E', - 'j', - 'P', - 'T', - 'z', - 'R', - 'N', - 'M', - '+', - 'L', - 'O', - 'q', - 'H', - 'G', - 'W', - 'U', - 'V', - ',', - 'Y', - 'K', - 'J', - 'Z', - 'X', - 'Q', - ';', - ')', - '(', - '~', - '[', - ']', - '$', - '!', - '\'', - '*', - '@', - '\x00', - '\x01', - '\x02', - '\x03', - '\x04', - '\x05', - '\x06', - '\x07', - '\x08', - '\t', - '\n', - '\x0b', - '\x0c', - '\r', - '\x0e', - '\x0f', - '\x10', - '\x11', - '\x12', - '\x13', - '\x14', - '\x15', - '\x16', - '\x17', - '\x18', - '\x19', - '\x1a', - '\x1b', - '\x1c', - '\x1d', - '\x1e', - '\x1f', - ' ', - '"', - '#', - '<', - '>', - '\\', - '^', - '`', - '{', - '|', - '}', - '\x7f', - '\x80', - '\x81', - '\x82', - '\x83', - '\x84', - '\x85', - '\x86', - '\x87', - '\x88', - '\x89', - '\x8a', - '\x8b', - '\x8c', - '\x8d', - '\x8e', - '\x8f', - '\x90', - '\x91', - '\x92', - '\x93', - '\x94', - '\x95', - '\x96', - '\x97', - '\x98', - '\x99', - '\x9a', - '\x9b', - '\x9c', - '\x9d', - '\x9e', - '\x9f', - '\xa0', - '\xa1', - '\xa2', - '\xa3', - '\xa4', - '\xa5', - '\xa6', - '\xa7', - '\xa8', - '\xa9', - '\xaa', - '\xab', - '\xac', - '\xad', - '\xae', - '\xaf', - '\xb0', - '\xb1', - '\xb2', - '\xb3', - '\xb4', - '\xb5', - '\xb6', - '\xb7', - '\xb8', - '\xb9', - '\xba', - '\xbb', - '\xbc', - '\xbd', - '\xbe', - '\xbf', - '\xc0', - '\xc1', - '\xc2', - '\xc3', - '\xc4', - '\xc5', - '\xc6', - '\xc7', - '\xc8', - '\xc9', - '\xca', - '\xcb', - '\xcc', - '\xcd', - '\xce', - '\xcf', - '\xd0', - '\xd1', - '\xd2', - '\xd3', - '\xd4', - '\xd5', - '\xd6', - '\xd7', - '\xd8', - '\xd9', - '\xda', - '\xdb', - '\xdc', - '\xdd', - '\xde', - '\xdf', - '\xe0', - '\xe1', - '\xe2', - '\xe3', - '\xe4', - '\xe5', - '\xe6', - '\xe7', - '\xe8', - '\xe9', - '\xea', - '\xeb', - '\xec', - '\xed', - '\xee', - '\xef', - '\xf0', - '\xf1', - '\xf2', - '\xf3', - '\xf4', - '\xf5', - '\xf6', - '\xf7', - '\xf8', - '\xf9', - '\xfa', - '\xfb', - '\xfc', - '\xfd', - '\xfe', - '\xff', + 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', '.', 'h', 'l', 'm', + '-', 'd', 'u', '0', '1', '2', 'g', '=', ':', 'b', 'f', '3', 'y', '5', '&', '_', + '4', 'v', '9', '6', '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', + 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'q', 'H', 'G', 'W', + 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 'Q', ';', ')', '(', '~', '[', ']', '$', + '!', '\'', '*', '@', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b', + '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', + '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', '#', '<', '>', '\\', '^', '`', '{', '|', '}', '\x7f', + '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', + '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', + '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', + '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', + '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', + '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', + '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', + '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', }; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 824021c9e379fa0bc3239e1bad5b2acee525ffae..962973bb5cd6fa47c0635ac5a4e5d8fa708dbfa5 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -17,10 +17,8 @@ #include "index_fst_util.h" #include "tutil.h" -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { - if (ctx->offset + len > ctx->limit) { - return -1; - } +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { + if (ctx->offset + len > ctx->limit) { return -1; } if (ctx->type == TFile) { assert(len == tfWrite(ctx->file.fd, buf, len)); @@ -30,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { ctx->offset += len; return len; } -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { int nRead = 0; if (ctx->type == TFile) { nRead = tfRead(ctx->file.fd, buf, len); @@ -41,7 +39,18 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { return nRead; } -static int writeCtxDoFlush(WriterCtx *ctx) { +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { + int nRead = 0; + if (ctx->type == TFile) { + tfLseek(ctx->file.fd, offset, 0); + nRead = tfRead(ctx->file.fd, buf, len); + } else { + // refactor later + assert(0); + } + return nRead; +} +static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { // tfFsync(ctx->fd); // tfFlush(ctx->file.fd); @@ -51,11 +60,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) { return 1; } -WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { - WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); - if (ctx == NULL) { - return NULL; - } +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { + WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); + if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { @@ -67,8 +74,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ctx->file.fd = tfOpenReadWrite(tmpFile); } if (ctx->file.fd < 0) { - goto END; indexError("open file error %d", errno); + goto END; } } else if (ctx->type == TMemory) { ctx->mem.buf = calloc(1, sizeof(char) * capacity); @@ -77,18 +84,17 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ctx->write = writeCtxDoWrite; ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; + ctx->readFrom = writeCtxDoReadFrom; ctx->offset = 0; ctx->limit = capacity; return ctx; END: - if (ctx->type == TMemory) { - free(ctx->mem.buf); - } + if (ctx->type == TMemory) { free(ctx->mem.buf); } free(ctx); } -void writerCtxDestroy(WriterCtx *ctx) { +void writerCtxDestroy(WriterCtx* ctx) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { @@ -97,57 +103,53 @@ void writerCtxDestroy(WriterCtx *ctx) { free(ctx); } -FstCountingWriter *fstCountingWriterCreate(void *wrt) { - FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); - if (cw == NULL) { - return NULL; - } +FstCountingWriter* fstCountingWriterCreate(void* wrt) { + FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); + if (cw == NULL) { return NULL; } cw->wrt = wrt; //(void *)(writerCtxCreate(TFile, readOnly)); return cw; } -void fstCountingWriterDestroy(FstCountingWriter *cw) { +void fstCountingWriterDestroy(FstCountingWriter* cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); // writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { - return 0; - } +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { + if (write == NULL) { return 0; } // update checksum // write data to file/socket or mem - WriterCtx *ctx = write->wrt; + WriterCtx* ctx = write->wrt; int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; return len; } -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { - return 0; - } - WriterCtx *ctx = write->wrt; +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { + if (write == NULL) { return 0; } + WriterCtx* ctx = write->wrt; int nRead = ctx->read(ctx, buf, len); // assert(nRead == len); return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } -int fstCountingWriterFlush(FstCountingWriter *write) { - WriterCtx *ctx = write->wrt; +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { + return 0; +} +int fstCountingWriterFlush(FstCountingWriter* write) { + WriterCtx* ctx = write->wrt; ctx->flush(ctx); // write->wtr->flush return 1; } -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t *buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = calloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; @@ -157,7 +159,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t return; } -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) { uint8_t nBytes = packSize(n); fstCountingWriterPackUintIn(writer, n, nBytes); return nBytes; diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 084f280bc3d3b8d224824005b0bb9b3a9d4ec894..5cdf6adb31a421e38fb22d30fe34599f1489924c 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -14,60 +14,46 @@ */ #include "index_fst_node.h" -FstBuilderNode *fstBuilderNodeDefault() { - FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); +FstBuilderNode* fstBuilderNodeDefault() { + FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } -void fstBuilderNodeDestroy(FstBuilderNode *node) { - if (node == NULL) { - return; - } +void fstBuilderNodeDestroy(FstBuilderNode* node) { + if (node == NULL) { return; } taosArrayDestroy(node->trans); free(node); } -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { - if (n1 == n2) { - return true; - } - if (n1 == NULL || n2 == NULL) { - return false; - } +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { + if (n1 == n2) { return true; } + if (n1 == NULL || n2 == NULL) { return false; } - if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { - return false; - } + if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; } size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; - if (s1 != s2) { - return false; - } + if (s1 != s2) { return false; } for (size_t i = 0; i < s1; i++) { - FstTransition *t1 = taosArrayGet(n1->trans, i); - FstTransition *t2 = taosArrayGet(n2->trans, i); - if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { - return false; - } + FstTransition* t1 = taosArrayGet(n1->trans, i); + FstTransition* t2 = taosArrayGet(n2->trans, i); + if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; } } return true; } -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); - if (node == NULL) { - return NULL; - } +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + if (node == NULL) { return NULL; } // size_t sz = taosArrayGetSize(src->trans); - SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); + SArray* trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *tran = taosArrayGet(src->trans, i); + FstTransition* tran = taosArrayGet(src->trans, i); taosArrayPush(trans, tran); } @@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { return node; } // not destroy src, User's bussiness -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { - if (dst == NULL || src == NULL) { - return; - } +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { + if (dst == NULL || src == NULL) { return; } dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; @@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { size_t sz = taosArrayGetSize(src->trans); dst->trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(src->trans, i); + FstTransition* trn = taosArrayGet(src->trans, i); taosArrayPush(dst->trans, trn); } } diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 7bb2e722307a0b676fd6eb7485a3502508e9bbd9..07afc52ecf12bf515e32afca7fc50a44e0a422b3 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -15,7 +15,7 @@ #include "index_fst_registry.h" -uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { +uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { // TODO(yihaoDeng): refactor later const uint64_t FNV_PRIME = 1099511628211; uint64_t h = 14695981039346656037u; @@ -25,21 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); for (uint32_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(bNode->trans, i); + FstTransition* trn = taosArrayGet(bNode->trans, i); h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; } return h % (registry->tableSize); } -static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { +static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) { size_t sz = taosArrayGetSize(arr); - if (a >= sz || b >= sz) { - return; - } + if (a >= sz || b >= sz) { return; } - FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); - FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); + FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a); + FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b); FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; @@ -51,11 +49,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { return; } -static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { +static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { size_t sz = taosArrayGetSize(arr); - if (start >= sz && end >= sz) { - return; - } + if (start >= sz && end >= sz) { return; } assert(start >= end); @@ -67,14 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { } } -FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry *registry = malloc(sizeof(FstRegistry)); - if (registry == NULL) { - return NULL; - } +FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { + FstRegistry* registry = malloc(sizeof(FstRegistry)); + if (registry == NULL) { return NULL; } uint64_t nCells = tableSize * mruSize; - SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); + SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); if (NULL == tb) { free(registry); return NULL; @@ -91,32 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { return registry; } -void fstRegistryDestroy(FstRegistry *registry) { - if (registry == NULL) { - return; - } +void fstRegistryDestroy(FstRegistry* registry) { + if (registry == NULL) { return; } - SArray *tb = registry->table; + SArray* tb = registry->table; size_t sz = taosArrayGetSize(tb); for (size_t i = 0; i < sz; i++) { - FstRegistryCell *cell = taosArrayGet(tb, i); + FstRegistryCell* cell = taosArrayGet(tb, i); fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); free(registry); } -FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { - if (taosArrayGetSize(registry->table) <= 0) { - return NULL; - } +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { + if (taosArrayGetSize(registry->table) <= 0) { return NULL; } uint64_t bucket = fstRegistryHash(registry, bNode); uint64_t start = registry->mruSize * bucket; uint64_t end = start + registry->mruSize; - FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); + FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); if (end - start == 1) { - FstRegistryCell *cell = taosArrayGet(registry->table, start); + FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; @@ -128,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo entry->cell = cell; // copy or not } } else if (end - start == 2) { - FstRegistryCell *cell1 = taosArrayGet(registry->table, start); + FstRegistryCell* cell1 = taosArrayGet(registry->table, start); if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { entry->state = FOUND; entry->addr = cell1->addr; return entry; } - FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); + FstRegistryCell* cell2 = taosArrayGet(registry->table, start + 1); if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { entry->state = FOUND; entry->addr = cell2->addr; @@ -146,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo fstBuilderNodeCloneFrom(cell2->node, bNode); fstRegistryCellSwap(registry->table, start, start + 1); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); + FstRegistryCell* cCell = taosArrayGet(registry->table, start); entry->state = NOTFOUND; entry->cell = cCell; } else { uint32_t i = start; for (; i < end; i++) { - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, i); if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr; @@ -162,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo } if (i >= end) { uint64_t last = end - 1; - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last); // clone from bNode, refactor later fstBuilderNodeCloneFrom(cell->node, bNode); fstRegistryCellPromote(registry->table, last, start); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); + FstRegistryCell* cCell = taosArrayGet(registry->table, start); entry->state = NOTFOUND; entry->cell = cCell; } } return entry; } -void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); } +void fstRegistryEntryDestroy(FstRegistryEntry* entry) { + free(entry); +} diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 597f7cc61ade3bedd5e2814c166e86e1433c71f6..da1e177a18b3563dd2388e536cc301bb7b8117d0 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -61,9 +61,10 @@ uint8_t packSize(uint64_t n) { } } -uint64_t unpackUint64(uint8_t *ch, uint8_t sz) { +uint64_t unpackUint64(uint8_t* ch, uint8_t sz) { uint64_t n = 0; for (uint8_t i = 0; i < sz; i++) { + // n = n | (ch[i] << (8 * i)); } return n; @@ -75,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { return packSize(nodeAddr - transAddr); } } -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { uint64_t delta = unpackUint64(data, len); // delta_add = u64_to_usize if (delta == EMPTY_ADDRESS) { @@ -88,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { // fst slice func // -FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { - FstString *str = (FstString *)malloc(sizeof(FstString)); +FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { + FstString* str = (FstString*)malloc(sizeof(FstString)); str->ref = 1; str->len = len; str->data = malloc(len * sizeof(uint8_t)); @@ -99,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { return s; } // just shallow copy -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { - FstString *str = s->str; +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { + FstString* str = s->str; str->ref++; // uint8_t *buf = fstSliceData(s, &alen); // start = buf + start - (buf - s->start); @@ -109,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { int32_t tlen = end - start + 1; int32_t slen; - uint8_t *data = fstSliceData(s, &slen); + uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = malloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - FstString *str = malloc(sizeof(FstString)); + FstString* str = malloc(sizeof(FstString)); str->data = buf; str->len = tlen; str->ref = 1; @@ -129,17 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { ans.end = tlen - 1; return ans; } -bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } +bool fstSliceIsEmpty(FstSlice* s) { + return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; +} -uint8_t *fstSliceData(FstSlice *s, int32_t *size) { - FstString *str = s->str; - if (size != NULL) { - *size = s->end - s->start + 1; - } +uint8_t* fstSliceData(FstSlice* s, int32_t* size) { + FstString* str = s->str; + if (size != NULL) { *size = s->end - s->start + 1; } return str->data + s->start; } -void fstSliceDestroy(FstSlice *s) { - FstString *str = s->str; +void fstSliceDestroy(FstSlice* s) { + FstString* str = s->str; str->ref--; if (str->ref <= 0) { free(str->data); @@ -148,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) { } } -int fstSliceCompare(FstSlice *a, FstSlice *b) { +int fstSliceCompare(FstSlice* a, FstSlice* b) { int32_t alen, blen; - uint8_t *aBuf = fstSliceData(a, &alen); - uint8_t *bBuf = fstSliceData(b, &blen); + uint8_t* aBuf = fstSliceData(a, &alen); + uint8_t* bBuf = fstSliceData(b, &blen); uint32_t i, j; for (i = 0, j = 0; i < alen && j < blen; i++, j++) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 19e8ff67506ba0bc963ab910eaffa2f5754f6b60..ecacdcd13c463ef4ffc689534d13737d69f15e1b 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -21,107 +21,83 @@ #include "index_fst_counting_writer.h" #include "index_util.h" #include "taosdef.h" +#include "tcompare.h" -static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) { - // TODO simple tfile header later - char buf[TFILE_HADER_PRE_SIZE]; - char * p = buf; - TFileReadHeader *header = &reader->header; - int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); - assert(nread == TFILE_HADER_PRE_SIZE); +#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) - memcpy(&header->suid, p, sizeof(header->suid)); - p += sizeof(header->suid); +typedef struct TFileValue { + char* colVal; // null terminated + SArray* tableId; + int32_t offset; +} TFileValue; - memcpy(&header->version, p, sizeof(header->version)); - p += sizeof(header->version); +static int tfileValueCompare(const void* a, const void* b, const void* param); +static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); - int32_t colLen = 0; - memcpy(&colLen, p, sizeof(colLen)); - assert(colLen < sizeof(header->colName)); - nread = reader->ctx->read(reader->ctx, header->colName, colLen); - assert(nread == colLen); +static int tfileWriteHeader(TFileWriter* writer); +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); +static int tfileWriteData(TFileWriter* write, TFileValue* tval); - nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType)); - return 0; -}; -static int tfileGetFileList(const char *path, SArray *result) { - DIR *dir = opendir(path); - if (NULL == dir) { - return -1; - } +static int tfileReadLoadHeader(TFileReader* reader); +static int tfileReadLoadFst(TFileReader* reader); +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static void tfileReadRef(TFileReader* reader); +static void tfileReadUnRef(TFileReader* reader); - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { - size_t len = strlen(entry->d_name); - char * buf = calloc(1, len + 1); - memcpy(buf, entry->d_name, len); - taosArrayPush(result, &buf); - } - closedir(dir); - return 0; -} -static void tfileDestroyFileName(void *elem) { - char *p = *(char **)elem; - free(p); -} -static int tfileCompare(const void *a, const void *b) { - const char *aName = *(char **)a; - const char *bName = *(char **)b; - size_t aLen = strlen(aName); - size_t bLen = strlen(bName); - return strncmp(aName, bName, aLen > bLen ? aLen : bLen); -} -// tfile name suid-colId-version.tindex -static int tfileParseFileName(const char *filename, uint64_t *suid, int *colId, int *version) { - if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { - // read suid & colid & version success - return 0; - } - return -1; -} -static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { - SERIALIZE_MEM_TO_BUF(buf, key, suid); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, colType); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, version); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -} +static int tfileGetFileList(const char* path, SArray* result); +static int tfileRmExpireFile(SArray* result); +static void tfileDestroyFileName(void* elem); +static int tfileCompare(const void* a, const void* b); +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); -TFileCache *tfileCacheCreate(const char *path) { - TFileCache *tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } +TFileCache* tfileCacheCreate(const char* path) { + TFileCache* tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; - SArray *files = taosArrayInit(4, sizeof(void *)); + SArray* files = taosArrayInit(4, sizeof(void*)); tfileGetFileList(path, files); taosArraySort(files, tfileCompare); + tfileRmExpireFile(files); + + uint64_t suid; + int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { - char * file = taosArrayGetP(files, i); - uint64_t suid; - int colId, version; - if (0 != tfileParseFileName(file, &suid, &colId, &version)) { - goto End; + char* file = taosArrayGetP(files, i); + if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) { + indexInfo("try parse invalid file: %s, skip it", file); continue; } - WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); if (wc == NULL) { indexError("failed to open index: %s", file); goto End; } - TFileReader *reader = tfileReaderCreate(wc); + + TFileReader* reader = tfileReaderCreate(wc); if (0 != tfileReadLoadHeader(reader)) { - TFileReaderDestroy(reader); - indexError("failed to load index header, index Id: %s", file); + tfileReaderDestroy(reader); + indexError("failed to load index header, index file: %s", file); + goto End; + } + + if (0 != tfileReadLoadFst(reader)) { + tfileReaderDestroy(reader); + indexError("failed to load index fst, index file: %s", file); goto End; } + tfileReadRef(reader); + // loader fst and validate it + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; + + char buf[128] = {0}; + tfileSerialCacheKey(&key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); return tcache; @@ -130,78 +106,356 @@ End: taosArrayDestroyEx(files, tfileDestroyFileName); return NULL; } -void tfileCacheDestroy(TFileCache *tcache) { - if (tcache == NULL) { - return; - } +void tfileCacheDestroy(TFileCache* tcache) { + if (tcache == NULL) { return; } // free table cache - TFileReader **reader = taosHashIterate(tcache->tableCache, NULL); + TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { - TFileReader *p = *reader; - indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, - p->header.colType); - TFileReaderDestroy(p); + TFileReader* p = *reader; + indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); + + tfileReadUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); free(tcache); } -TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); - TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + + TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + tfileReadRef(reader); + return reader; } -void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); + // remove last version index reader + TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); + if (*p != NULL) { + TFileReader* oldReader = *p; + taosHashRemove(tcache->tableCache, buf, strlen(buf)); + tfileReadUnRef(oldReader); + } + + tfileReadRef(reader); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } -TFileReader *tfileReaderCreate(WriterCtx *ctx) { - TFileReader *reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } - reader->ctx = ctx; +TFileReader* tfileReaderCreate(WriterCtx* ctx) { + TFileReader* reader = calloc(1, sizeof(TFileReader)); + if (reader == NULL) { return NULL; } + // T_REF_INC(reader); + reader->ctx = ctx; return reader; } -void TFileReaderDestroy(TFileReader *reader) { - if (reader == NULL) { - return; - } +void tfileReaderDestroy(TFileReader* reader) { + if (reader == NULL) { return; } // T_REF_INC(reader); + fstDestroy(reader->fst); writerCtxDestroy(reader->ctx); free(reader); } -TFileWriter *tfileWriterCreate(const char *suid, const char *colName); -void tfileWriterDestroy(TFileWriter *tw); +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; -IndexTFile *indexTFileCreate(const char *path) { - IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); - tfile->cache = tfileCacheCreate(path); + int ret = -1; + // refactor to callback later + if (qtype == QUERY_TERM) { + uint64_t offset; + FstSlice key = fstSliceCreate(term->colVal, term->nColVal); + if (fstGet(reader->fst, &key, &offset)) { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); + ret = tfileReadLoadTableIds(reader, offset, result); + } else { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal); + } + fstSliceDestroy(&key); + } else if (qtype == QUERY_PREFIX) { + // handle later + // + } else { + // handle later + } + tfileReadUnRef(reader); + return ret; +} +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { + // char pathBuf[128] = {0}; + // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); + // TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType}; + // memcpy(header.colName, ); + + // char buf[TFILE_HADER_PRE_SIZE]; + // int len = TFILE_HADER_PRE_SIZE; + // if (len != ctx->write(ctx, buf, len)) { + // indexError("index: %" PRIu64 " failed to write header info", header->suid); + // return NULL; + //} + TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + if (tw == NULL) { + indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); + return NULL; + } + tw->ctx = ctx; + tw->header = *header; + tfileWriteHeader(tw); + return tw; +} + +int tfileWriterPut(TFileWriter* tw, void* data) { + // sort by coltype and write to tindex + __compar_fn_t fn = getComparFunc(tw->header.colType, 0); + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); + + int32_t bufLimit = 4096, offset = 0; + char* buf = calloc(1, sizeof(bufLimit)); + char* p = buf; + int32_t sz = taosArrayGetSize((SArray*)data); + int32_t fstOffset = tw->offset; + + // ugly code, refactor later + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); + } + tfileWriteFstOffset(tw, fstOffset); + + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + // check buf has enough space or not + int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); + if (offset + ttsz > bufLimit) { + // batch write + tw->ctx->write(tw->ctx, buf, offset); + offset = 0; + memset(buf, 0, bufLimit); + p = buf; + } + + tfileSerialTableIdsToBuf(p, v->tableId); + offset += ttsz; + p = buf + offset; + // set up value offset + v->offset = tw->offset; + tw->offset += ttsz; + } + if (offset != 0) { + // write reversed data in buf to tindex + tw->ctx->write(tw->ctx, buf, offset); + } + tfree(buf); + + // write fst + for (size_t i = 0; i < sz; i++) { + // TODO, fst batch write later + TFileValue* v = taosArrayGetP((SArray*)data, i); + if (tfileWriteData(tw, v) == 0) { + // + } + } + return 0; +} +void tfileWriterDestroy(TFileWriter* tw) { + if (tw == NULL) { return; } + + writerCtxDestroy(tw->ctx); + free(tw); +} + +IndexTFile* indexTFileCreate(const char* path) { + IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + if (tfile == NULL) { return NULL; } + + tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } +void IndexTFileDestroy(IndexTFile* tfile) { + free(tfile); +} + +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { + int ret = -1; + if (tfile == NULL) { return ret; } + IndexTFile* pTfile = (IndexTFile*)tfile; + + SIndexTerm* term = query->term; + TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; + TFileReader* reader = tfileCacheGet(pTfile->cache, &key); + + return tfileReaderSearch(reader, query, result); +} +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { + // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = + // 1}; + + return 0; +} + +static int tfileValueCompare(const void* a, const void* b, const void* param) { + __compar_fn_t fn = *(__compar_fn_t*)param; + + TFileValue* av = (TFileValue*)a; + TFileValue* bv = (TFileValue*)b; + + return fn(av->colVal, bv->colVal); +} +static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { + int sz = taosArrayGetSize(ids); + SERIALIZE_VAR_TO_BUF(buf, sz, int32_t); + for (size_t i = 0; i < sz; i++) { + uint64_t* v = taosArrayGet(ids, i); + SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t); + } +} + +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { + int32_t fstOffset = offset + sizeof(tw->header.fstOffset); + tw->header.fstOffset = fstOffset; + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + return 0; +} +static int tfileWriteHeader(TFileWriter* writer) { + char buf[TFILE_HEADER_NO_FST] = {0}; + + TFileHeader* header = &writer->header; + memcpy(buf, (char*)header, sizeof(buf)); + + int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); + if (sizeof(buf) != nwrite) { return -1; } + writer->offset = nwrite; + return 0; +} +static int tfileWriteData(TFileWriter* write, TFileValue* tval) { + TFileHeader* header = &write->header; + uint8_t colType = header->colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); + if (fstBuilderInsert(write->fb, key, tval->offset)) { + fstSliceDestroy(&key); + return 0; + } + fstSliceDestroy(&key); + return -1; + } else { + // handle other type later + } + return 0; +} +static int tfileReadLoadHeader(TFileReader* reader) { + // TODO simple tfile header later + char buf[TFILE_HEADER_SIZE] = {0}; + + int64_t nread = reader->ctx->read(reader->ctx, buf, sizeof(buf)); + assert(nread == sizeof(buf)); + memcpy(&reader->header, buf, sizeof(buf)); + return 0; +} +static int tfileReadLoadFst(TFileReader* reader) { + // current load fst into memory, refactor it later + static int FST_MAX_SIZE = 16 * 1024; + + char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); + if (buf == NULL) { return -1; } + + WriterCtx* ctx = reader->ctx; + int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); + // we assuse fst size less than FST_MAX_SIZE + assert(nread > 0 && nread < FST_MAX_SIZE); + + FstSlice st = fstSliceCreate((uint8_t*)buf, nread); + reader->fst = fstCreate(&st); + free(buf); + fstSliceDestroy(&st); + + return reader->fst == NULL ? 0 : -1; +} +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { + int32_t nid; + WriterCtx* ctx = reader->ctx; + + int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); + assert(sizeof(nid) == nread); + + int32_t total = sizeof(uint64_t) * nid; + char* buf = calloc(1, total); + if (buf == NULL) { return -1; } -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { - IndexTFile *pTfile = (IndexTFile *)tfile; + nread = ctx->read(ctx, buf, total); + assert(total == nread); - SIndexTerm * term = query->term; - TFileCacheKey key = { - .suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; - TFileReader *reader = tfileCacheGet(pTfile->cache, &key); + for (int32_t i = 0; i < nid; i++) { + taosArrayPush(result, (uint64_t*)buf + i); + } + free(buf); return 0; } -int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { - TFileWriterOpt wOpt = { - .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1}; +static void tfileReadRef(TFileReader* reader) { + int ref = T_REF_INC(reader); + UNUSED(ref); +} + +static void tfileReadUnRef(TFileReader* reader) { + int ref = T_REF_DEC(reader); + if (ref == 0) { tfileReaderDestroy(reader); } +} + +static int tfileGetFileList(const char* path, SArray* result) { + DIR* dir = opendir(path); + if (NULL == dir) { return -1; } + struct dirent* entry; + while ((entry = readdir(dir)) != NULL) { + size_t len = strlen(entry->d_name); + char* buf = calloc(1, len + 1); + memcpy(buf, entry->d_name, len); + taosArrayPush(result, &buf); + } + closedir(dir); + return 0; +} +static int tfileRmExpireFile(SArray* result) { + // TODO(yihao): remove expire tindex after restart return 0; } +static void tfileDestroyFileName(void* elem) { + char* p = *(char**)elem; + free(p); +} +static int tfileCompare(const void* a, const void* b) { + const char* aName = *(char**)a; + const char* bName = *(char**)b; + + size_t aLen = strlen(aName); + size_t bLen = strlen(bName); + + return strncmp(aName, bName, aLen > bLen ? aLen : bLen); +} +// tfile name suid-colId-version.tindex +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) { + if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { + // read suid & colid & version success + return 0; + } + return -1; +} +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +} diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 63328cb263aed721c21859ac2cb57c561f6b4ff8..400b533bf794bdbcfe66438de9fc8538e87d5acf 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -9,6 +9,6 @@ SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); -SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf); +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index 7d2e6e3aec336c2e1158501c7197ebdf790fa79b..69dad4d9f4b792ed573628f05ddb95f301f8cc08 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -78,7 +78,7 @@ typedef struct { typedef struct STableDataBlocks { int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id + int32_t vgId; // virtual group id int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending int32_t numOfTables; // number of tables in current submit block int32_t rowSize; // row size for current table @@ -170,10 +170,11 @@ int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +void destroyBlockArrayList(SArray* pDataBlockList); int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index bf9e53e8d4d7ad5f99681be3165eacda45fff8e1..93df023fbaedb0de568249557dd887e3250f9b4a 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -68,7 +68,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ * @param type * @return */ -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 780ff8b0e8858b241615c58031fdee11d25ab5b5..a9272f39df9cbbfece0ea7d1fe4d97107a272bea 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -221,7 +221,7 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32 return pCreateMsg; } -int32_t createSName(SName* pName, SToken* pTableName, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf) { +int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "acctId too long"; @@ -260,7 +260,7 @@ int32_t createSName(SName* pName, SToken* pTableName, SParseCtx* pParseCtx, SMsg return code; } -SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SSchema* pSchema; int32_t numOfCols = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pColumns); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 3b9a33af0259b5f9569abd69ae4cd1066f80e29a..33efeb469d7ad50fd51eebeacbe62a3e1689d58c 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4308,7 +4308,7 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 3138f0c1d8c8fe0bdca6b835e1c0afe9bb0fe407..2f3d5aab77ecaea9bf30ab2113f6636bcee7e231 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -123,7 +123,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - //dataBuf->pData = calloc(1, dataBuf->nAllocSize); dataBuf->pData = malloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tfree(dataBuf); @@ -145,8 +144,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; - // tNameAssign(&dataBuf->tableName, name); - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -269,19 +266,17 @@ void destroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock); } -void* destroyBlockArrayList(SArray* pDataBlockList) { +void destroyBlockArrayList(SArray* pDataBlockList) { if (pDataBlockList == NULL) { - return NULL; + return; } size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - void* d = taosArrayGetP(pDataBlockList, i); - destroyDataBlock(d); + destroyDataBlock(taosArrayGetP(pDataBlockList, i)); } taosArrayDestroy(pDataBlockList); - return NULL; } // data block is disordered, sort it in ascending order @@ -298,6 +293,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { int32_t i = 0; int32_t j = 1; + // delete rows with timestamp conflicts while (j < pBlocks->numOfRows) { TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); @@ -430,7 +426,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->dataLen = 0; - int32_t numOfRows = htons(pBlock->numOfRows); + int32_t numOfRows = pBlock->numOfRows; if (isRawPayload) { for (int32_t i = 0; i < numOfRows; ++i) { @@ -467,18 +463,10 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } } - int32_t len = pBlock->dataLen + pBlock->schemaLen; - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - - return len; -} - -static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { - // todo + return pBlock->dataLen + pBlock->schemaLen; } -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -537,24 +525,13 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; - - pBlocks->numOfRows = 0; } p = taosHashIterate(pHashObj, p); @@ -565,12 +542,10 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t pOneTableBlock = *p; } - extractTableNameList(pHashObj, freeBlockMap); - // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); tfree(blkKeyInfo.pKeyTuple); - + *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 877ecb2c85457c9aff9de9454306c9fffc559bf8..68aa7554833c34feee541d4c20a9ebd6b1e41873 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -31,33 +31,16 @@ pSql += index; \ } while (0) -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - return terrno; \ - } \ - } while (0) - -#define CHECK_CODE_1(expr, destroy) \ +#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy; \ - terrno = code; \ - return terrno; \ - } \ + sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE_2(expr, destroy1, destroy2) \ +#define CHECK_CODE(expr) \ do { \ int32_t code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy1; \ - (void)destroy2; \ - terrno = code; \ - return terrno; \ + return code; \ } \ } while (0) @@ -67,12 +50,16 @@ enum { }; typedef struct SInsertParseContext { - SParseContext* pComCxt; - const char* pSql; - SMsgBuf msg; - struct SCatalog* pCatalog; - STableMeta* pTableMeta; - SHashObj* pTableBlockHashObj; // data block for each table. need release + SParseContext* pComCxt; // input + const char* pSql; // input + SMsgBuf msg; // input + STableMeta* pTableMeta; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SArray* pTableDataBlocks; // global + SArray* pVgDataBlocks; // global int32_t totalNum; SInsertStmtInfo* pOutput; } SInsertParseContext; @@ -151,14 +138,6 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b return ret; } -static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { - *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (NULL == *pInsertInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; -} - static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -179,14 +158,14 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); if (NULL != p) { // db.table - strcpy(fullDbName, pCxt->pComCxt->pAcctId); - fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0]; - strncpy(fullDbName, pStname->z, p - pStname->z); + int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId); + strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); } else { - snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname); + snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db); strncpy(tableName, pStname->z, pStname->n); } + return TSDB_CODE_SUCCESS; } @@ -194,11 +173,13 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0}; CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); return TSDB_CODE_SUCCESS; } -// todo speedup by using hash list static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { @@ -209,6 +190,51 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS return -1; } +static void buildMsgHeader(SVgDataBlocks* blocks) { + SMsgDesc* desc = (SMsgDesc*)blocks->pData; + desc->numOfVnodes = htonl(1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + submit->header.vgId = htonl(blocks->vg.vgId); + submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); + submit->length = submit->header.contLen; + submit->numOfBlocks = htonl(blocks->numOfTables); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + int32_t numOfBlocks = blocks->numOfTables; + while (numOfBlocks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->tid = htonl(blk->tid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } +} + +static int32_t buildOutput(SInsertParseContext* pCxt) { + size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks); + pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pCxt->pOutput->pDataBlocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for (size_t i = 0; i < numOfVg; ++i) { + STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); + SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + dst->numOfTables = src->numOfTables; + dst->size = src->size; + SWAP(dst->pData, src->pData, char*); + buildMsgHeader(dst); + taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { @@ -219,16 +245,14 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) if (k == INT64_MIN) { if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; } else { if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; // client time/server time can not be mixed - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_CLI_TS; } if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { @@ -525,7 +549,7 @@ static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pTo } case TSDB_DATA_TYPE_BINARY: { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); } return func(pToken->z, pToken->n, param); @@ -608,26 +632,22 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) { - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - destroyBoundColumnInfo(pSpd); +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision) { + if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.builder = &kvRowBuilder}; + SKvParam param = {.builder = &pCxt->tagsBuilder}; SToken sToken; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - for (int i = 0; i < pSpd->numOfBound; ++i) { + for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; param.schema = pSchema; - CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); + CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m)); } - destroyBoundColumnInfo(pSpd); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); + SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } @@ -650,13 +670,12 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) } SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); - SParsedDataColInfo spd = {0}; - setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); if (TK_LP == sToken.type) { - CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd)); + CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -668,7 +687,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); return TSDB_CODE_SUCCESS; } @@ -732,10 +751,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" SToken sToken; while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); if (TK_LP != sToken.type) { break; } + pCxt->pSql += index; if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; @@ -779,13 +800,30 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } +static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { + tfree(pCxt->pTableMeta); + destroyBoundColumnInfo(&pCxt->tags); + tdDestroyKVRowBuilder(&pCxt->tagsBuilder); +} + +static void destroyInsertParseContext(SInsertParseContext* pCxt) { + destroyInsertParseContextForTable(pCxt); + taosHashCleanup(pCxt->pVgroupsHashObj); + taosHashCleanup(pCxt->pTableBlockHashObj); + destroyBlockArrayList(pCxt->pTableDataBlocks); + destroyBlockArrayList(pCxt->pVgDataBlocks); +} + // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { + // for each table while (1) { + destroyInsertParseContextForTable(pCxt); + SToken sToken; // pSql -> tb_name ... NEXT_TOKEN(pCxt->pSql, sToken); @@ -815,7 +853,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -842,9 +880,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } - return TSDB_CODE_SUCCESS; + return buildOutput(pCxt); } // INSERT INTO @@ -854,26 +892,31 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { - CHECK_CODE(createInsertStmtInfo(pInfo)); - SInsertParseContext context = { .pComCxt = pContext, .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pCatalog = NULL, .pTableMeta = NULL, + .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, - .pOutput = *pInfo + .pOutput = calloc(1, sizeof(SInsertStmtInfo)) }; - if (NULL == context.pTableBlockHashObj) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } - CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog)); - CHECK_CODE(skipInsertInto(&context)); - CHECK_CODE(parseInsertBody(&context)); + *pInfo = context.pOutput; + context.pOutput->schemaAttache = pContext->schemaAttached; + context.pOutput->payloadType = PAYLOAD_TYPE_KV; - return TSDB_CODE_SUCCESS; + int32_t code = skipInsertInto(&context); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(&context); + } + destroyInsertParseContext(&context); + terrno = code; + return (TSDB_CODE_SUCCESS == code ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 1e61be15dd2fade5a1e7e00411c280e65179c0e0..3490580f155f2f583c487852f3803091ef49c48d 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,7 +31,7 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, SParseCtx* pParseCtx, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { +int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 3abdaa10364cc6933f1ba53ac2a6382d21eddc9e..307062ef155d952e7c95dcdcfe6b1b480d26fdff 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1167,7 +1167,7 @@ void cleanupTagCond(STagCond* pTagCond) { * @param tableIndex denote the table index for join query, where more than one table exists * @return */ -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { assert(pQueryInfo != NULL); if (pQueryInfo->pTableMetaInfo == NULL) { assert(pQueryInfo->numOfTables == 0); diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 502946675b40738c8ec7d3e35287669ce04dab71..1ae0d9211a2925b27b415becfab1fc385d242965 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -354,7 +354,7 @@ bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) { return false; } -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo) { +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo) { int32_t n = 10; int32_t level = 0; diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index feae00827310a93e9117361cc5d7c682e3582a76..e722b873b6882f625fbc81e9347dcd90437a3337 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,5 +17,3 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common parser catalog transport gtest function planner qcom ) - -TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9e681c7ccb8b8d5412892c3b41413c0f455cf69b --- /dev/null +++ b/source/libs/parser/test/insertParserTest.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "insertParser.h" +// #include "mockCatalog.h" + +using namespace std; +using namespace testing; + +namespace { + string toString(int32_t code) { + return tstrerror(code); + } +} + +// syntax: +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +class InsertTest : public Test { +protected: + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.ctx.acctId = atoi(acctId_.c_str()); + cxt_.ctx.db = (char*) db_.c_str(); + strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + + } + + int32_t run() { + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; + } + return code_; + } + + SInsertStmtInfo* reslut() { + return res_; + } + + void dumpReslut() { + size_t num = taosArrayGetSize(res_->pDataBlocks); + cout << "schemaAttache:" << (int32_t)res_->schemaAttache << ", payloadType:" << (int32_t)res_->payloadType << ", insertType:" << res_->insertType << ", numOfVgs:" << num << endl; + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + cout << "numOfVnodes:" << ntohl(desc->numOfVnodes) << endl; + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + cout << "length:" << ntohl(submit->length) << ", numOfBlocks:" << ntohl(submit->numOfBlocks) << endl; + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + cout << "Block:" << i << endl; + cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << ntohl(blk->tid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) + << ", dataLen:" << ntohl(blk->dataLen) << ", schemaLen:" << ntohl(blk->schemaLen) << ", numOfRows:" << ntohs(blk->numOfRows) << endl; + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + + void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { + ASSERT_EQ(res_->schemaAttache, 0); + ASSERT_EQ(res_->payloadType, PAYLOAD_TYPE_KV); + ASSERT_EQ(res_->insertType, TSDB_QUERY_TYPE_INSERT); + size_t num = taosArrayGetSize(res_->pDataBlocks); + ASSERT_GE(num, 0); + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + ASSERT_EQ(vg->numOfTables, numOfTables); + ASSERT_GE(vg->size, 0); + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + ASSERT_EQ(ntohl(desc->numOfVnodes), 1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + ASSERT_GE(ntohl(submit->length), 0); + ASSERT_GE(ntohl(submit->numOfBlocks), 0); + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + ASSERT_EQ(ntohs(blk->numOfRows), (0 == i ? numOfRows1 : (numOfRows2 > 0 ? numOfRows2 : numOfRows1))); + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; + } + + string acctId_; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; +}; + +// INSERT INTO tb_name VALUES (field1_value, ...) +TEST_F(InsertTest, singleTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 1); +} + +// INSERT INTO tb_name VALUES (field1_value, ...)(field1_value, ...) +TEST_F(InsertTest, singleTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\") st1s2 values (now, 10, \"131028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 1); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")" + " st1s2 values (now, 10, \"131028\")(now+1s, 20, \"132028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 3, 2); +} + +TEST_F(InsertTest, toleranceTest) { + setDatabase("root", "test"); + + bind("insert into"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); + bind("insert into t"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp deleted file mode 100644 index 85db46d7bf1cfa29576926a0dafc7ef52310a2a8..0000000000000000000000000000000000000000 --- a/source/libs/parser/test/insertTest.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -#include "insertParser.h" -#include "mockCatalog.h" - -using namespace std; -using namespace testing; - -namespace { - string toString(int32_t code) { - return tstrerror(code); - } -} - -extern "C" { - -#include - -void *__real_malloc(size_t); - -void *__wrap_malloc(size_t c) { - // printf("My MALLOC called: %d\n", c); - // void *array[32]; - // int size = backtrace(array, 32); - // char **symbols = backtrace_symbols(array, size); - // for (int i = 0; i < size; ++i) { - // cout << symbols[i] << endl; - // } - // free(symbols); - - return __real_malloc(c); -} - -} - -// syntax: -// INSERT INTO -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -class InsertTest : public Test { -protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.pAcctId = acctId_.c_str(); - cxt_.pDbname = db_.c_str(); - strcpy(sqlBuf_, sql); - cxt_.sqlLen = strlen(sql); - sqlBuf_[cxt_.sqlLen] = '\0'; - cxt_.pSql = sqlBuf_; - - } - - int32_t run() { - code_ = parseInsertSql(&cxt_, &res_); - if (code_ != TSDB_CODE_SUCCESS) { - cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; - } - return code_; - } - - SInsertStmtInfo* reslut() { - return res_; - } - -private: - static const int max_err_len = 1024; - static const int max_sql_len = 1024 * 1024; - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - code_ = TSDB_CODE_SUCCESS; - res_ = nullptr; - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - char sqlBuf_[max_sql_len]; - SParseContext cxt_; - int32_t code_; - SInsertStmtInfo* res_; -}; - -// INSERT INTO tb_name VALUES (field1_value, ...) -TEST_F(InsertTest, simpleTest) { - setDatabase("root", "test"); - - bind("insert into t1 values (now, 1, \"wxy\")"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); - SInsertStmtInfo* res = reslut(); - // todo check - ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); - // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); -} - -TEST_F(InsertTest, toleranceTest) { - setDatabase("root", "test"); - - bind("insert into"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); - bind("insert into t"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); -} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index dc341f6af7eb65a5197dedac76437668984d1d83..6d8973366803dc30f5cd29f5baed40f0563facf3 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -25,15 +25,15 @@ namespace { void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); } void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); mcs->createSubTable("root.test", "st1", "st1s1", 1); mcs->createSubTable("root.test", "st1", "st1s2", 2); @@ -42,11 +42,15 @@ void generateTestST1(MockCatalogService* mcs) { } int32_t __catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { - return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); + return 0; } int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { - return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); + return mockCatalogService->catalogGetTableMeta(pDBName, pTableName, pTableMeta); +} + +int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) { + return mockCatalogService->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); } void initMetaDataEnv() { @@ -55,6 +59,7 @@ void initMetaDataEnv() { static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); { AddrAny any("libcatalog.so"); std::map result; @@ -71,6 +76,14 @@ void initMetaDataEnv() { stub.set(f.second, __catalogGetTableMeta); } } + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetTableHashVgroup$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetTableHashVgroup); + } + } } void generateMetaData() { diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index 35eb1ddcd9ddfdd96c749db35b1271bcf6c531ff..f8e80a48aa5dacc7dc6d90d1a9db4623bfc73278 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -22,8 +22,4 @@ void initMetaDataEnv(); void generateMetaData(); void destroyMetaDataEnv(); -// mock -// int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); -// int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); - #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index e7271c3cb2a2901f339500369030d3cb95ccf9e4..e234f82da93aad8612e57673fd89f64025d52685 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,6 +33,7 @@ public: col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); + rowsize_ += bytes; return *this; } @@ -89,11 +90,11 @@ public: MockCatalogServiceImpl() : id_(1) { } - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + int32_t catalogGetHandle() const { return 0; } - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { @@ -103,6 +104,11 @@ public: return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + // todo + return 0; + } + TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); @@ -147,11 +153,11 @@ public: for (const auto& db : meta_) { std::cout << "Databse:" << db.first << std::endl; - std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << IH("RowSize") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { const auto& schema = table.second->schema; - std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << IF(schema->tableInfo.rowSize) << std::endl; } std::cout << std::endl; } @@ -269,10 +275,10 @@ std::shared_ptr MockCatalogService::getTableMeta(const std::strin return impl_->getTableMeta(db, tbname); } -int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { - return impl_->catalogGetHandle(clusterId, catalogHandle); +int32_t MockCatalogService::catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pDBName, pTableName, pTableMeta); } -int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { - return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); -} +int32_t MockCatalogService::catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + return impl_->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); +} \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index cd01db09cce7ff4c5949671ff44af041a47d80cb..889a45439715792d09a33d4c9073b4175a275dca 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -57,9 +57,8 @@ public: void showTables() const; std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; - // mock interface - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 7e289d51586a742a1e88c7977ad560705a990142..be68149e09a7d57f1315a3d7f6a56dcfdbcdeb18 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -718,7 +718,7 @@ TEST(testCase, show_user_Test) { int32_t type = 0; int32_t len = 0; - SParseCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); @@ -742,7 +742,7 @@ TEST(testCase, create_user_Test) { int32_t type = 0; int32_t len = 0; - SParseCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index a2078defdaf1acbe5b9e0824f08a934371345d6c..f9b172210627cede4727f8e1a3eeaa2eb9ee3ae1 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -93,7 +93,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(ret, 0); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); @@ -156,7 +156,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index b1c4643f154f56be96a096d09161e1f0aa74540a..1bee95b8e56965f424488e38c3fe2da6a66de677 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -40,6 +40,7 @@ extern "C" { #define QNODE_SESSIONWINDOW 12 #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 +#define QNODE_INSERT 15 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not @@ -82,7 +83,7 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode); * @param pQueryNode * @return */ -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode); +int32_t createQueryPlan(const SQueryNode* pNode, struct SQueryPlanNode** pQueryPlan); /** * Convert the query plan to string, in order to display it in the shell. diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index ce923314bd9530b6425fff926cb3b9662f57b813..136073aa6030945962b6740bfd3f74d55b30fc78 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -29,7 +29,7 @@ typedef struct SJoinCond { SColumn *colCond[2]; } SJoinCond; -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); @@ -37,13 +37,37 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); +int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) { + *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); + SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + if (NULL == *pQueryPlan || NULL == blocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + (*pQueryPlan)->info.type = QNODE_INSERT; + taosArrayAddAll(blocks, pInsert->pDataBlocks); + (*pQueryPlan)->pExtInfo = blocks; + return TSDB_CODE_SUCCESS; +} + +int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryPlan) { + SArray* upstream = createQueryPlanImpl(pSelect); assert(taosArrayGetSize(upstream) == 1); + *pQueryPlan = taosArrayGetP(upstream, 0); + taosArrayDestroy(upstream); + return TSDB_CODE_SUCCESS; +} - *pQueryNode = taosArrayGetP(upstream, 0); +int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { + switch (nodeType(pNode)) { + case TSDB_SQL_SELECT: { + return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); + } + case TSDB_SQL_INSERT: + return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan); + default: + return TSDB_CODE_FAILED; + } - taosArrayDestroy(upstream); return TSDB_CODE_SUCCESS; } @@ -62,7 +86,7 @@ void destroyQueryPlan(SQueryPlanNode* pQueryNode) { //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SExprInfo** pExpr, int32_t numOfOutput, const void* pExtInfo) { SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); pNode->info.type = type; @@ -123,7 +147,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla } case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; + pNode->pExtInfo = (void*)pExtInfo; break; } @@ -145,7 +169,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla return pNode; } -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, +static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { int32_t num = (int32_t) taosArrayGetSize(pExprs); @@ -186,7 +210,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { // group by column not by tag size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); @@ -239,7 +263,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p->pData, taosArrayGetSize(p), pInfo); } if (pQueryInfo->order != NULL) { @@ -254,7 +278,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, +static SQueryPlanNode* doCreateQueryPlanForSingleTable(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, SArray* tableCols) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); @@ -286,7 +310,7 @@ static bool isAllAggExpr(SArray* pList) { return true; } -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { +SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* upstream = NULL; if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 1d29e48e308e1b60ee60e35ea6e6c3dcf489e643..9db9e059b33df0bfe8f6e70492b8b6151bb0a395 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,6 +14,7 @@ */ #include "plannerInt.h" +#include "exception.h" #include "parser.h" #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan @@ -33,6 +34,13 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; +static void* vailidPointer(void* p) { + if (NULL == p) { + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + return p; +} + const char* opTypeToOpName(int32_t type) { return gOpName[type]; } @@ -46,6 +54,25 @@ int32_t opNameToOpType(const char* name) { return OP_Unknown; } +static SDataSink* initDataSink(int32_t type, int32_t size) { + SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); + sink->info.type = type; + return sink; +} + +static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher)); + return (SDataSink*)dispatcher; +} + +static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks) { + SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter)); + inserter->numOfTables = pBlocks->numOfTables; + inserter->size = pBlocks->size; + SWAP(inserter->pData, pBlocks->pData, char*); + return (SDataSink*)inserter; +} + static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); @@ -72,10 +99,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) { } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { - SPhyNode* node = (SPhyNode*)calloc(1, size); - if (NULL == node) { - return NULL; - } + SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { @@ -138,7 +162,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { - SSubplan* subplan = calloc(1, sizeof(SSubplan)); + SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan))); subplan->id = pCxt->nextId; ++(pCxt->nextId.subplanId); subplan->type = type; @@ -146,15 +170,15 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL != pCxt->pCurrentSubplan) { subplan->level = pCxt->pCurrentSubplan->level + 1; if (NULL == pCxt->pCurrentSubplan->pChildern) { - pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); } taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); - subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } SArray* currentLevel; if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { - currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); } else { currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); @@ -164,7 +188,17 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { return subplan; } -static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { +static void vgroupInfoToEpSet(const SVgroupInfo* vg, SEpSet* epSet) { + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; +} + +static void vgroupMsgToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { epSet->inUse = 0; // todo epSet->numOfEps = vg->numOfEps; for (int8_t i = 0; i < vg->numOfEps; ++i) { @@ -179,8 +213,9 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); - vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); + vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } return pCxt->nextId.templateId++; @@ -214,6 +249,9 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_INSERT: + // Insert is not an operator in a physical plan. + break; default: assert(false); } @@ -221,7 +259,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { - SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); + SPhyNode* child = createPhyNode(pCxt, taosArrayGetP(pPlanNode->pChildren, i)); child->pParent = node; taosArrayPush(node->pChildren, &child); } @@ -229,26 +267,46 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { return node; } +static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SArray* vgs = (SArray*)pPlanNode->pExtInfo; + size_t numOfVg = taosArrayGetSize(vgs); + for (int32_t i = 0; i < numOfVg; ++i) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); + SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i); + vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet); + subplan->pNode = NULL; + subplan->pDataSink = createDataInserter(pCxt, blocks); + } +} + static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { - SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); - ++(pCxt->nextId.templateId); - subplan->pNode = createPhyNode(pCxt, pRoot); + if (QNODE_INSERT == pRoot->info.type) { + splitInsertSubplan(pCxt, pRoot); + } else { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); + subplan->pNode = createPhyNode(pCxt, pRoot); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot); + } // todo deal subquery } int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) { - SPlanContext context = { - .pCatalog = pCatalog, - .pDag = calloc(1, sizeof(SQueryDag)), - .pCurrentSubplan = NULL, - .nextId = {0} // todo queryid - }; - if (NULL == context.pDag) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - createSubplanByLevel(&context, pQueryNode); - *pDag = context.pDag; + TRY(TSDB_MAX_TAG_CONDITIONS) { + SPlanContext context = { + .pCatalog = pCatalog, + .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))), + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid + }; + *pDag = context.pDag; + context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + createSubplanByLevel(&context, pQueryNode); + } CATCH(code) { + CLEANUP_EXECUTE(); + terrno = code; + return TSDB_CODE_FAILED; + } END_TRY return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 2510797158978e3ba8fc4f0a6732a2f0a7c40ac6..15c0e632a7d7ccf1e2fa32f89e66f85f86e0259f 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -651,7 +651,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "Schema"; +static const char* jkPnodeSchema = "InputSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e9a4591d4afe8b4b0efed0f322c9be6ba5c4ee40..7722a7d3639ffcfef62ef9610de5d1ff90daa4c4 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -24,9 +24,9 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { // todo } -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SEpSet* pEpSet, struct SQueryDag** pDag) { SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + int32_t code = createQueryPlan(pNode, &logicPlan); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(logicPlan); return code; diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4eafa50bdc200615391cfc2f364fc9c9c20430bd --- /dev/null +++ b/source/libs/qworker/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QWORKER_SRC) +add_library(qworker ${QWORKER_SRC}) +target_include_directories( + qworker + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + qworker + PRIVATE os util transport planner qcom +) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h new file mode 100644 index 0000000000000000000000000000000000000000..1adc09def41234a26d3557916eb40917e6e1b57e --- /dev/null +++ b/source/libs/qworker/inc/qworkerInt.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_INT_H_ +#define _TD_QWORKER_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 +#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000 +#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 + +enum { + QW_READY_NOT_RECEIVED = 0, + QW_READY_RECEIVED, + QW_READY_RESPONSED, +}; + +typedef struct SQWorkerTaskStatus { + int8_t status; + int8_t ready; +} SQWorkerTaskStatus; + +typedef struct SQWorkerResCache { + void *data; +} SQWorkerResCache; + +typedef struct SQWorkerSchTaskStatus { + int32_t lastAccessTs; // timestamp in second + SHashObj *taskStatus; // key:queryId+taskId, value: SQWorkerTaskStatus +} SQWorkerSchTaskStatus; + +// Qnode/Vnode level task management +typedef struct SQWorkerMgmt { + SQWorkerCfg cfg; + SHashObj *scheduleHash; //key: schedulerId, value: SQWorkerSchTaskStatus + SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache +} SQWorkerMgmt; + +#define QW_TASK_DONE(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == status == JOB_TASK_STATUS_CANCELLED) +#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0) +#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0) + +#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_INT_H_*/ diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c new file mode 100644 index 0000000000000000000000000000000000000000..82bfd75b6a1d2ffb22a4c11b6f286b185d6a643a --- /dev/null +++ b/source/libs/qworker/src/qworker.c @@ -0,0 +1,298 @@ +#include "taosmsg.h" +#include "query.h" +#include "qworker.h" +#include "qworkerInt.h" +#include "planner.h" + +int32_t qwAddTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t taskStatus) { + SQWorkerTaskStatus tStatus = {0}; + tStatus.status = taskStatus; + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + SQWorkerSchTaskStatus newSchStatus = {0}; + newSchStatus.taskStatus = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == newSchStatus.taskStatus) { + qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (0 != taosHashPut(newSchStatus.taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + newSchStatus.lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId), &newSchStatus, sizeof(newSchStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"] to scheduleHash failed", schedulerId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(schStatus->taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SQWorkerTaskStatus **taskStatus) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerTaskStatus *tStatus = taosHashGet(schStatus->taskStatus, id, sizeof(id)); + if (NULL == tStatus) { + qError("no task status for schedulerId[%"PRIx64"] queryId[%"PRIx64"] taskId[%"PRIx64"]", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *taskStatus = tStatus; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void *data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache resCache = {0}; + resCache.data = data; + + if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwGetTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void **data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache *resCache = taosHashGet(mgmt->resHash, id, sizeof(id)); + if (NULL == resCache) { + qError("no task res for queryId[%"PRIx64"] taskId[%"PRIx64"]", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *data = resCache->data; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t schedulerId) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SSchedulerStatusRsp **rsp) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + int32_t i = 0; + int32_t taskNum = taosHashGetSize(schStatus->taskStatus); + int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; + *rsp = calloc(1, size); + if (NULL == *rsp) { + qError("calloc %d failed", size); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *key = NULL; + size_t keyLen = 0; + void *pIter = taosHashIterate(schStatus->taskStatus, NULL); + while (pIter) { + SQWorkerTaskStatus *taskStatus = (SQWorkerTaskStatus *)pIter; + taosHashGetKey(pIter, &key, &keyLen); + + QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); + (*rsp)->status[i].status = taskStatus->status; + + pIter = taosHashIterate(schStatus->taskStatus, pIter); + } + + (*rsp)->num = taskNum; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwBuildRspMsg(void *data, int32_t msgType); + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + if (NULL == mgmt) { + qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (cfg) { + mgmt->cfg = *cfg; + } else { + mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER; + mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + } + + mgmt->scheduleHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->scheduleHash) { + tfree(mgmt); + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum); + } + + mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->resHash) { + taosHashCleanup(mgmt->scheduleHash); + mgmt->scheduleHash = NULL; + tfree(mgmt); + + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum); + } + + *qWorkerMgmt = mgmt; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSubplan *plan = NULL; + SQWorkerTaskStatus *tStatus = NULL; + + int32_t code = qStringToSubplan(msg->msg, &plan); + if (TSDB_CODE_SUCCESS != code) { + qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->schedulerId, msg->queryId, msg->taskId, code); + return code; + } + + //TODO call executer to init subquery + + QW_ERR_JRET(qwAddTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING)); + + QW_ERR_JRET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_QUERY_RSP)); + + //TODO call executer to execute subquery + code = 0; + void *data = NULL; + //TODO call executer to execute subquery + + QW_ERR_JRET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + tStatus->status = (code) ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_SUCCEED; + + QW_ERR_JRET(qwAddTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, data)); + +_return: + + if (tStatus && QW_TASK_DONE(tStatus->status) && QW_READY_RECEIVED == tStatus->ready) { + QW_ERR_RET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_RES_READY_RSP)); + } + + qDestroySubplan(plan); + + return code; +} + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp){ + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SQWorkerTaskStatus *tStatus = NULL; + + QW_ERR_RET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + if (QW_TASK_DONE(tStatus->status)) { + QW_ERR_RET(qwBuildRspMsg(tStatus, TSDB_MSG_TYPE_RES_READY_RSP)); + } else { + tStatus->ready = QW_READY_RECEIVED; + + return TSDB_CODE_SUCCESS; + } + + tStatus->ready = QW_READY_RESPONSED; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSchedulerStatusRsp *sStatus = NULL; + + QW_ERR_RET(qwGetSchTasksStatus(qWorkerMgmt, msg->schedulerId, &sStatus)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->schedulerId)); + + void *data = NULL; + + QW_ERR_RET(qwGetTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, &data)); + + QW_ERR_RET(qwBuildRspMsg(data, TSDB_MSG_TYPE_FETCH_RSP)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt) { + if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { + return; + } + + SQWorkerMgmt *mgmt = *qWorkerMgmt; + + //TODO STOP ALL QUERY + + //TODO FREE ALL + + tfree(*qWorkerMgmt); +} + + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 73e7c4d24e1c2693d174dcb7acad4112939778c3..3fab91edacd77096fbacceee5df3f762452ad16d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -31,17 +31,9 @@ extern "C" { #define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA -enum { - SCH_STATUS_NOT_START = 1, - SCH_STATUS_EXECUTING, - SCH_STATUS_SUCCEED, - SCH_STATUS_FAILED, - SCH_STATUS_CANCELLING, - SCH_STATUS_CANCELLED -}; - typedef struct SSchedulerMgmt { - uint64_t taskId; + uint64_t taskId; + uint64_t schedulerId; SSchedulerCfg cfg; SHashObj *Jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6014ff9ab6282248f52bfe1aba0535e5b4d4cb71..99a9b06fe441832d42fbc5bc86ad24bcc3ebc263 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -161,7 +161,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { SArray *levelPlans = NULL; int32_t levelPlanNum = 0; - level.status = SCH_STATUS_NOT_START; + level.status = JOB_TASK_STATUS_NOT_START; for (int32_t i = 0; i < levelNum; ++i) { level.level = i; @@ -191,7 +191,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); task.plan = plan; - task.status = SCH_STATUS_NOT_START; + task.status = JOB_TASK_STATUS_NOT_START; void *p = taosArrayPush(level.subTasks, &task); if (NULL == p) { @@ -304,13 +304,15 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerQueryMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; - pMsg->contentLen = len; + + pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + pMsg->contentLen = htonl(len); memcpy(pMsg->msg, task->msg, len); break; } - case TSDB_MSG_TYPE_RSP_READY: { + case TSDB_MSG_TYPE_RES_READY: { msgSize = sizeof(SSchedulerReadyMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -319,8 +321,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerReadyMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } case TSDB_MSG_TYPE_FETCH: { @@ -332,8 +334,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerFetchMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } default: @@ -376,7 +378,7 @@ _return: int32_t schProcessOnJobSuccess(SQueryJob *job) { - job->status = SCH_STATUS_SUCCEED; + job->status = JOB_TASK_STATUS_SUCCEED; if (job->userFetch) { SCH_ERR_RET(schFetchFromRemote(job)); @@ -386,7 +388,7 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) { } int32_t schProcessOnJobFailure(SQueryJob *job) { - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); @@ -413,7 +415,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { return TSDB_CODE_SUCCESS; } - task->status = SCH_STATUS_SUCCEED; + task->status = JOB_TASK_STATUS_SUCCEED; int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); if (parentNum == 0) { @@ -459,7 +461,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod if (!needRetry) { SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; SCH_ERR_RET(schProcessOnJobFailure(job)); return TSDB_CODE_SUCCESS; @@ -478,13 +480,13 @@ int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32 if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { - code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RSP_READY); + code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RES_READY); if (code) { goto _task_error; } } break; - case TSDB_MSG_TYPE_RSP_READY: + case TSDB_MSG_TYPE_RES_READY: if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { @@ -534,7 +536,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(schPushTaskToExecList(job, task)); - task->status = SCH_STATUS_EXECUTING; + task->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } @@ -546,22 +548,26 @@ int32_t schLaunchJob(SQueryJob *job) { SCH_ERR_RET(schLaunchTask(job, task)); } - job->status = SCH_STATUS_EXECUTING; + job->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } int32_t schedulerInit(SSchedulerCfg *cfg) { - schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == schMgmt.Jobs) { - SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); - } - if (cfg) { schMgmt.cfg = *cfg; + } else { + schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } + schMgmt.Jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == schMgmt.Jobs) { + SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum); + } + + schMgmt.schedulerId = 1; //TODO GENERATE A UUID + return TSDB_CODE_SUCCESS; } @@ -605,7 +611,7 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - job->status = SCH_STATUS_NOT_START; + job->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(job)); @@ -634,7 +640,7 @@ int32_t scheduleFetchRows(void *pJob, void **data) { return TSDB_CODE_QRY_APP_ERROR; } - if (job->status == SCH_STATUS_SUCCEED) { + if (job->status == JOB_TASK_STATUS_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(job)); } @@ -668,7 +674,7 @@ void scheduleFreeJob(void *pJob) { return; } - if (job->status == SCH_STATUS_EXECUTING) { + if (job->status == JOB_TASK_STATUS_EXECUTING) { scheduleCancelJob(pJob); } } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index bfc6b47fa6abd80245b021954ee908156e9eebeb..c4f6f78106e1ee6703e0429e705f62ab83db1881 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -776,9 +776,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } -FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { +FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { + if (NULL == data || NULL == key) { + return -1; + } + SHashNode * node = GET_HASH_PNODE(data); - return GET_HASH_NODE_KEY(node); + *key = GET_HASH_NODE_KEY(node); + *keyLen = node->keyLen; + + return 0; } FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { diff --git a/tests/script/general/user/basic1.sim b/tests/script/general/user/basic1.sim index d4a663c096f99cfcde87399bfe57b66f424e2410..33c4dc96d41c76e9ebe2661d806cb1d0ac6b304a 100644 --- a/tests/script/general/user/basic1.sim +++ b/tests/script/general/user/basic1.sim @@ -26,7 +26,7 @@ if $rows != 2 then endi print $data00 $data01 $data02 -print $data10 $data11 $data22 +print $data10 $data11 $data12 print $data20 $data11 $data22 print $data30 $data31 $data32 @@ -38,7 +38,7 @@ if $rows != 3 then endi print $data00 $data01 $data02 -print $data10 $data11 $data22 +print $data10 $data11 $data12 print $data20 $data11 $data22 print $data30 $data31 $data32 print $data40 $data41 $data42 diff --git a/tests/script/general/user/pass_len.sim b/tests/script/general/user/pass_len.sim index 5eb200b51f9ac28b19ea095be6530dbada9be894..649b3efa485b07df486b87195397ae8b4d25ef18 100644 --- a/tests/script/general/user/pass_len.sim +++ b/tests/script/general/user/pass_len.sim @@ -3,8 +3,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect $i = 0 @@ -29,7 +27,7 @@ step12: sql create user $user PASS 'taosdata' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -40,7 +38,7 @@ sql drop user $user -x step2 step2: sql create user $user PASS '1' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -52,7 +50,7 @@ step3: sql create user $user PASS 'abc0123456789' sql show users -if $rows != 6 then +if $rows != 3 then return -1 endi @@ -63,7 +61,7 @@ sql create user $user PASS 'abcd012345678901234567891234567890' -x step4 return -1 step4: sql show users -if $rows != 6 then +if $rows != 4 then return -1 endi @@ -75,7 +73,7 @@ while $i < 3 endw sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi