diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 591ffb2701cc0a5e229ebf711bb3a668fef407ab..2a4512ef48af15e1b4ea71708064030f20375616 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -51,7 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" ) // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -1081,6 +1081,7 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSchedulerQueryMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; uint32_t contentLen; @@ -1088,15 +1089,39 @@ typedef struct SSchedulerQueryMsg { } SSchedulerQueryMsg; typedef struct SSchedulerReadyMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerReadyMsg; typedef struct SSchedulerFetchMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerFetchMsg; +typedef struct SSchedulerStatusMsg { + uint64_t schedulerId; +} SSchedulerStatusMsg; + +typedef struct STaskStatus { + uint64_t queryId; + uint64_t taskId; + int8_t status; +} STaskStatus; + +typedef struct SSchedulerStatusRsp { + uint32_t num; + STaskStatus status[]; +} SSchedulerStatusRsp; + + +typedef struct SSchedulerCancelMsg { + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; +} SSchedulerCancelMsg; + #pragma pack(pop) diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 65779b209938bc9766f2d0d11881c2354d0c5092..aa4c3af392dfb2e94198d0062801fe405624752c 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif - +#include "trpc.h" typedef struct { uint64_t numOfStartTask; @@ -32,48 +32,6 @@ typedef struct { uint64_t numOfErrors; } SQnodeStat; -/* start Task msg */ -typedef struct { - uint32_t schedulerIp; - uint16_t schedulerPort; - int64_t taskId; - int64_t queryId; - uint32_t srcIp; - uint16_t srcPort; -} SQnodeStartTaskMsg; - -/* stop Task msg */ -typedef struct { - int64_t taskId; -} SQnodeStopTaskMsg; - -/* start/stop Task msg response */ -typedef struct { - int64_t taskId; - int32_t code; -} SQnodeTaskRespMsg; - -/* Task status msg */ -typedef struct { - int64_t taskId; - int32_t status; - int64_t queryId; -} SQnodeTaskStatusMsg; - -/* Qnode/Scheduler heartbeat msg */ -typedef struct { - int32_t status; - int32_t load; - -} SQnodeHeartbeatMsg; - -/* Qnode sent/received msg */ -typedef struct { - int8_t msgType; - int32_t msgLen; - char msg[]; -} SQnodeMsg; - /** * Start one Qnode in Dnode. diff --git a/include/libs/index/index.h b/include/libs/index/index.h index f4d45477c11af7161221e493f8c15118d358d252..f93c46da0c412e3529dd0c016ef7c42dc8de1686 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -44,41 +44,46 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = * @param: oper * */ -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); /* * @param: * @param: */ -int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); -void indexClose(SIndex *index); -int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); -int indexDelete(SIndex *index, SIndexMultiTermQuery *query); -int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); -int indexRebuild(SIndex *index, SIndexOpts *opt); +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +void indexClose(SIndex* index); +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +int indexRebuild(SIndex* index, SIndexOpts* opt); /* * @param * @param */ -SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +SIndexMultiTerm* indexMultiTermCreate(); +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term); +void indexMultiTermDestroy(SIndexMultiTerm* terms); /* * @param: * @param: */ -SIndexOpts *indexOptsCreate(); -void indexOptsDestroy(SIndexOpts *opts); +SIndexOpts* indexOptsCreate(); +void indexOptsDestroy(SIndexOpts* opts); /* * @param: * @param: */ -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName, - int32_t nColName, const char *colVal, int32_t nColVal); -void indexTermDestroy(SIndexTerm *p); +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn operType, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal); +void indexTermDestroy(SIndexTerm* p); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 8e26fa98c945973bc6ef90aa262f2b9be20fe2b8..df2170423f899b5ba1a8f2d9472e1fea4ccb72a3 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -136,6 +136,7 @@ typedef struct SParseContext { const char* pDbname; void *pRpc; const char* pClusterId; + struct SCatalog* pCatalog; const SEpSet* pEpSet; int64_t id; // query id, generated by uuid generator int8_t schemaAttached; // denote if submit block is built with table schema or not diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 254d572149b6c01a439cd9e23470ff90c5a65936..4fcbc1c5282a7bf7f1339381ed17be8399a76ad4 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -24,6 +24,15 @@ extern "C" { #include "thash.h" #include "tlog.h" +enum { + JOB_TASK_STATUS_NOT_START = 1, + JOB_TASK_STATUS_EXECUTING, + JOB_TASK_STATUS_SUCCEED, + JOB_TASK_STATUS_FAILED, + JOB_TASK_STATUS_CANCELLING, + JOB_TASK_STATUS_CANCELLED +}; + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h new file mode 100644 index 0000000000000000000000000000000000000000..63a6b6f89bee7100abd91426b979163a99cfd641 --- /dev/null +++ b/include/libs/qworker/qworker.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_H_ +#define _TD_QWORKER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "trpc.h" + +typedef struct SQWorkerCfg { + uint32_t maxSchedulerNum; + uint32_t maxResCacheNum; + uint32_t maxSchTaskNum; +} SQWorkerCfg; + +typedef struct { + uint64_t numOfStartTask; + uint64_t numOfStopTask; + uint64_t numOfRecvedFetch; + uint64_t numOfSentHb; + uint64_t numOfSentFetch; + uint64_t numOfTaskInQueue; + uint64_t numOfFetchInQueue; + uint64_t numOfErrors; +} SQWorkerStat; + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt); + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_H_*/ diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 2cbf26f8772927f67c3a882465cd66414be20a93..975b10353871f269abecb70d4974836d3e4a44b2 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -25,6 +25,7 @@ extern "C" { typedef struct SSchedulerCfg { int32_t clusterType; + int32_t maxJobNum; } SSchedulerCfg; typedef struct SQueryProfileSummary { diff --git a/include/util/thash.h b/include/util/thash.h index d5038802c079f3ed3cddcef6c30c219459fc9917..ebdc91f0546a7ef9de3801f4eabf13f18c2cdf4c 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -193,11 +193,10 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); /** * Get the corresponding key information for a given data in hash table - * @param pHashObj * @param data * @return */ -int32_t taosHashGetKey(SHashObj *pHashObj, void *data, void** key, size_t* keyLen); +int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); #ifdef __cplusplus } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 636cf0a9a84453458ec0d06b84231f926a4637b5..027532bbb1d1514604fc8277862e764cec13ec51 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -11,3 +11,4 @@ add_subdirectory(executor) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) +add_subdirectory(qworker) diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c index 48662f34436d05c9c2ef873bb52d6e1aa515f4f3..97dccb5c7b69405f91934362262e339985a5060d 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/executor/src/tfilter.c @@ -1175,7 +1175,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g void* key = NULL; len = 0; - taosHashGetKey((SHashObj *)data, p, &key, &len); + taosHashGetKey(p, &key, &len); void *fdata = NULL; if (IS_VAR_DATA_TYPE(type)) { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 72415a971674790f1b8ce5d9f21261c82eb16b89..2584a847ff2343db27a2f2acaf00a3f68c7aad68 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -42,11 +42,11 @@ typedef struct SIndexStat { struct SIndex { #ifdef USE_LUCENE - index_t *index; + index_t* index; #endif - void * cache; - void * tindex; - SHashObj *colObj; // < field name, field id> + void* cache; + void* tindex; + SHashObj* colObj; // < field name, field id> int64_t suid; // current super table id, -1 is normal table int colId; // field id allocated to cache @@ -58,7 +58,7 @@ struct SIndex { struct SIndexOpts { #ifdef USE_LUCENE - void *opts; + void* opts; #endif #ifdef USE_INVERTED_INDEX @@ -69,7 +69,7 @@ struct SIndexOpts { struct SIndexMultiTermQuery { EIndexOperatorType opera; - SArray * query; + SArray* query; }; // field and key; @@ -77,14 +77,14 @@ typedef struct SIndexTerm { int64_t suid; SIndexOperOnColumn operType; // oper type, add/del/update uint8_t colType; // term data type, str/interger/json - char * colName; + char* colName; int32_t nColName; - char * colVal; + char* colVal; int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { - SIndexTerm * term; + SIndexTerm* term; EIndexQueryType qType; } SIndexTermQuery; diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 9d5083d23d14c3e2490f3228b28672cdf8417f08..fb4f478ae99fdc7654536a98d67d5e3d8ac407fa 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -34,19 +34,18 @@ extern "C" { typedef struct IndexCache { T_REF_DECLARE() - SSkipList *skiplist; + SSkipList* skiplist; } IndexCache; // -IndexCache *indexCacheCreate(); +IndexCache* indexCacheCreate(); -void indexCacheDestroy(void *cache); +void indexCacheDestroy(void* cache); -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch( - void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 67438c092b3a99df876e445e2abf71d1b1a126c1..3c572787fc4b4fa17f7e870b5af44aa95b3ef115 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -40,10 +40,10 @@ typedef struct FstBoundWithData { } FstBoundWithData; typedef struct FstStreamBuilder { - Fst * fst; - AutomationCtx * aut; - FstBoundWithData *min; - FstBoundWithData *max; + Fst* fst; + AutomationCtx* aut; + FstBoundWithData* min; + FstBoundWithData* max; } FstStreamBuilder, FstStreamWithStateBuilder; typedef struct FstRange { @@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; -FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data); -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data); +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice); +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound); +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound); typedef struct FstOutput { bool null; @@ -71,46 +71,46 @@ typedef struct FstOutput { * TODO: simple function name */ typedef struct FstUnFinishedNodes { - SArray *stack; // } FstUnFinishedNodes; + SArray* stack; // } FstUnFinishedNodes; } FstUnFinishedNodes; #define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) -FstUnFinishedNodes *fstUnFinishedNodesCreate(); -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); -FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); -FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); -FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); +FstUnFinishedNodes* fstUnFinishedNodesCreate(); +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* node); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* node, Output out); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* node, CompiledAddr addr); +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* node, FstSlice bs, Output out); +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs); +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr); +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes); -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out); typedef struct FstBuilder { - FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`. - FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry * registry; // A map of finished nodes. + FstCountingWriter* wrt; // The FST raw data is written directly to `wtr`. + FstUnFinishedNodes* unfinished; // The stack of unfinished nodes + FstRegistry* registry; // A map of finished nodes. FstSlice last; // The last word added CompiledAddr lastAddr; // The address of the last compiled node uint64_t len; // num of keys added } FstBuilder; -FstBuilder *fstBuilderCreate(void *w, FstType ty); +FstBuilder* fstBuilderCreate(void* w, FstType ty); -void fstBuilderDestroy(FstBuilder *b); -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); -void * fstBuilerIntoInner(FstBuilder *b); -void fstBuilderFinish(FstBuilder *b); -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); +void fstBuilderDestroy(FstBuilder* b); +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in); +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate); +void* fstBuilerIntoInner(FstBuilder* b); +void fstBuilderFinish(FstBuilder* b); +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup); +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn); typedef struct FstTransitions { - FstNode *node; + FstNode* node; FstRange range; } FstTransitions; @@ -121,56 +121,55 @@ typedef struct FstState { uint8_t val; } FstState; -FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr); +FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr); FstState fstStateCreate(State state); // compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp); +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn); +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node); // set_comm_input -void fstStateSetCommInput(FstState *state, uint8_t inp); +void fstStateSetCommInput(FstState* state, uint8_t inp); // comm_input -uint8_t fstStateCommInput(FstState *state, bool *null); +uint8_t fstStateCommInput(FstState* state, bool* null); // input_len -uint64_t fstStateInputLen(FstState *state); +uint64_t fstStateInputLen(FstState* state); // end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data); -uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans( - FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); +uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); // input -uint8_t fstStateInput(FstState *state, FstNode *node); -uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +uint8_t fstStateInput(FstState* state, FstNode* node); +uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // trans_addr -CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); -CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); +CompiledAddr fstStateTransAddr(FstState* state, FstNode* node); +CompiledAddr fstStateTransAddrForAnyTrans(FstState* state, FstNode* node, uint64_t i); // sizes -PackSizes fstStateSizes(FstState *state, FstSlice *data); +PackSizes fstStateSizes(FstState* state, FstSlice* data); // Output -Output fstStateOutput(FstState *state, FstNode *node); -Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +Output fstStateOutput(FstState* state, FstNode* node); +Output fstStateOutputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // anyTrans specify function -void fstStateSetFinalState(FstState *state, bool yes); -bool fstStateIsFinalState(FstState *state); -void fstStateSetStateNtrans(FstState *state, uint8_t n); +void fstStateSetFinalState(FstState* state, bool yes); +bool fstStateIsFinalState(FstState* state); +void fstStateSetStateNtrans(FstState* state, uint8_t n); // state_ntrans -uint8_t fstStateStateNtrans(FstState *state, bool *null); -uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); -uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); -uint64_t fstStateNtransLen(FstState *state); -uint64_t fstStateNtrans(FstState *state, FstSlice *slice); -Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); -uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); +uint8_t fstStateStateNtrans(FstState* state, bool* null); +uint64_t fstStateTotalTransSize(FstState* state, uint64_t version, PackSizes size, uint64_t nTrans); +uint64_t fstStateTransIndexSize(FstState* state, uint64_t version, uint64_t nTrans); +uint64_t fstStateNtransLen(FstState* state); +uint64_t fstStateNtrans(FstState* state, FstSlice* slice); +Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) @@ -187,13 +186,13 @@ typedef struct FstLastTransition { * TODO: simple function name */ typedef struct FstBuilderNodeUnfinished { - FstBuilderNode * node; - FstLastTransition *last; + FstBuilderNode* node; + FstLastTransition* last; } FstBuilderNodeUnfinished; -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* node, CompiledAddr addr); -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* node, Output out); /* * FstNode and helper function @@ -224,18 +223,18 @@ typedef struct FstNode { // Return the address of this node. #define FST_NODE_ADDR(node) node->start -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); -void fstNodeDestroy(FstNode *fstNode); +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* data); +void fstNodeDestroy(FstNode* fstNode); -FstTransitions fstNodeTransitionIter(FstNode *node); -FstTransitions *fstNodeTransitions(FstNode *node); -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); +FstTransitions fstNodeTransitionIter(FstNode* node); +FstTransitions* fstNodeTransitions(FstNode* node); +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* res); +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res); +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res); -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode); -FstSlice fstNodeAsSlice(FstNode *node); +FstSlice fstNodeAsSlice(FstNode* node); // ops @@ -244,8 +243,8 @@ typedef struct FstIndexedValue { uint64_t value; } FstIndexedValue; -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); -void fstLastTransitionDestroy(FstLastTransition *trn); +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out); +void fstLastTransitionDestroy(FstLastTransition* trn); typedef struct FstMeta { uint64_t version; @@ -256,75 +255,74 @@ typedef struct FstMeta { } FstMeta; typedef struct Fst { - FstMeta * meta; - FstSlice *data; // - FstNode * root; // + FstMeta* meta; + FstSlice* data; // + FstNode* root; // } Fst; // refactor simple function -Fst *fstCreate(FstSlice *data); -void fstDestroy(Fst *fst); +Fst* fstCreate(FstSlice* data); +void fstDestroy(Fst* fst); -bool fstGet(Fst *fst, FstSlice *b, Output *out); -FstNode * fstGetNode(Fst *fst, CompiledAddr); -FstNode * fstGetRoot(Fst *fst); -FstType fstGetType(Fst *fst); -CompiledAddr fstGetRootAddr(Fst *fst); -Output fstEmptyFinalOutput(Fst *fst, bool *null); -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); +bool fstGet(Fst* fst, FstSlice* b, Output* out); +FstNode* fstGetNode(Fst* fst, CompiledAddr); +FstNode* fstGetRoot(Fst* fst); +FstType fstGetType(Fst* fst); +CompiledAddr fstGetRootAddr(Fst* fst); +Output fstEmptyFinalOutput(Fst* fst, bool* null); +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx); -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx); // into stream to expand later -StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb); +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb); -bool fstVerify(Fst *fst); +bool fstVerify(Fst* fst); // refactor this function -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr); typedef struct StreamState { - FstNode * node; + FstNode* node; uint64_t trans; FstOutput out; - void * autState; + void* autState; } StreamState; -void streamStateDestroy(void *s); +void streamStateDestroy(void* s); typedef struct StreamWithState { - Fst * fst; - AutomationCtx * aut; - SArray * inp; + Fst* fst; + AutomationCtx* aut; + SArray* inp; FstOutput emptyOutput; - SArray * stack; // - FstBoundWithData *endAt; + SArray* stack; // + FstBoundWithData* endAt; } StreamWithState; typedef struct StreamWithStateResult { FstSlice data; FstOutput out; - void * state; + void* state; } StreamWithStateResult; -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); -void swsResultDestroy(StreamWithStateResult *result); +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state); +void swsResultDestroy(StreamWithStateResult* result); -typedef void *(*StreamCallback)(void *); -StreamWithState *streamWithStateCreate( - Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max); +typedef void* (*StreamCallback)(void*); +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max); -void streamWithStateDestroy(StreamWithState *sws); +void streamWithStateDestroy(StreamWithState* sws); -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min); -StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback); -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut); // set up bound range // refator, simple code by marco -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 4e5309bf07abfb653a1ff69d0ef789b5bfce6739..c7269eda0f30a845c5b9a1810a5a4aea4baa56eb 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx; typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef struct StartWith { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } StartWith; typedef struct Complement { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } Complement; // automation typedef struct AutomationCtx { AutomationType type; - void * stdata; - char * data; + void* stdata; + char* data; } AutomationCtx; typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; @@ -48,27 +48,27 @@ typedef struct StartWithStateValue { ValueType type; union { int val; - char * ptr; - SArray *arr; + char* ptr; + SArray* arr; // add more type }; } StartWithStateValue; -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); -void startWithStateValueDestroy(void *sv); +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val); +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv); +void startWithStateValueDestroy(void* sv); typedef struct AutomationFunc { - void *(*start)(AutomationCtx *ctx); - bool (*isMatch)(AutomationCtx *ctx, void *); - bool (*canMatch)(AutomationCtx *ctx, void *data); - bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); - void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte); - void *(*acceptEof)(AutomationCtx *ct, void *state); + void* (*start)(AutomationCtx* ctx); + bool (*isMatch)(AutomationCtx* ctx, void*); + bool (*canMatch)(AutomationCtx* ctx, void* data); + bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state); + void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte); + void* (*acceptEof)(AutomationCtx* ct, void* state); } AutomationFunc; -AutomationCtx *automCtxCreate(void *data, AutomationType atype); -void automCtxDestroy(AutomationCtx *ctx); +AutomationCtx* automCtxCreate(void* data, AutomationType atype); +void automCtxDestroy(AutomationCtx* ctx); extern AutomationFunc automFuncs[]; #ifdef __cplusplus diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index b8a0384fb87584bbd8ad7da9193b19cf7165b772..ba0bd52e10280cd0139a6dc852b61c7f75b91a77 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -28,9 +28,10 @@ static char tmpFile[] = "./index"; typedef enum WriterType { TMemory, TFile } WriterType; typedef struct WriterCtx { - int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*flush)(struct WriterCtx *ctx); + int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*flush)(struct WriterCtx* ctx); + int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); WriterType type; union { struct { @@ -39,41 +40,42 @@ typedef struct WriterCtx { } file; struct { int32_t capa; - char * buf; + char* buf; } mem; }; int32_t offset; int32_t limit; } WriterCtx; -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoFlush(WriterCtx *ctx); +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); +static int writeCtxDoFlush(WriterCtx* ctx); -WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); -void writerCtxDestroy(WriterCtx *w); +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); +void writerCtxDestroy(WriterCtx* w); typedef uint32_t CheckSummer; typedef struct FstCountingWriter { - void * wrt; // wrap any writer that counts and checksum bytes written + void* wrt; // wrap any writer that counts and checksum bytes written uint64_t count; CheckSummer summer; } FstCountingWriter; -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len); -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len); -int fstCountingWriterFlush(FstCountingWriter *write); +int fstCountingWriterFlush(FstCountingWriter* write); -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write); -FstCountingWriter *fstCountingWriterCreate(void *wtr); -void fstCountingWriterDestroy(FstCountingWriter *w); +FstCountingWriter* fstCountingWriterCreate(void* wtr); +void fstCountingWriterDestroy(FstCountingWriter* w); -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes); +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n); #define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index a2041fad408745dbc502bd1c206dcaff29f64de7..32de7ea6211d4ab449fc6ef0d0fe8355ff380299 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -36,20 +36,20 @@ typedef struct FstTransition { typedef struct FstBuilderNode { bool isFinal; Output finalOutput; - SArray *trans; // + SArray* trans; // } FstBuilderNode; -FstBuilderNode *fstBuilderNodeDefault(); +FstBuilderNode* fstBuilderNodeDefault(); -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src); -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src); // bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, // CompiledAddr lastAddr, CompiledAddr startAddr); -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2); -void fstBuilderNodeDestroy(FstBuilderNode *node); +void fstBuilderNodeDestroy(FstBuilderNode* node); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 1b0922c7241cfa1c5f92863f459b5cf999baac99..f94a21d8dd7fd6887f531793af74f2d5023f66b7 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -25,7 +25,7 @@ extern "C" { typedef struct FstRegistryCell { CompiledAddr addr; - FstBuilderNode *node; + FstBuilderNode* node; } FstRegistryCell; #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) @@ -45,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState; typedef struct FstRegistryEntry { FstRegistryEntryState state; CompiledAddr addr; - FstRegistryCell * cell; + FstRegistryCell* cell; } FstRegistryEntry; // Registry relation function typedef struct FstRegistry { - SArray * table; // + SArray* table; // uint64_t tableSize; // num of rows uint64_t mruSize; // num of columns } FstRegistry; // -FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); -void fstRegistryDestroy(FstRegistry *registry); +FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); +void fstRegistryDestroy(FstRegistry* registry); -FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); -void fstRegistryEntryDestroy(FstRegistryEntry *entry); +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode); +void fstRegistryEntryDestroy(FstRegistryEntry* entry); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index c41977e6377f97a007c221cc63bf28182cae40ba..71a378584d401e966d7b2966ff215916c40c8232 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD; // uint8_t commonIdx(uint8_t v, uint8_t max); uint8_t packSize(uint64_t n); -uint64_t unpackUint64(uint8_t *ch, uint8_t sz); +uint64_t unpackUint64(uint8_t* ch, uint8_t sz); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr); typedef struct FstString { - uint8_t *data; + uint8_t* data; uint32_t len; int32_t ref; } FstString; typedef struct FstSlice { - FstString *str; + FstString* str; int32_t start; int32_t end; } FstSlice; -FstSlice fstSliceCreate(uint8_t *data, uint64_t len); -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); -bool fstSliceIsEmpty(FstSlice *s); -int fstSliceCompare(FstSlice *s1, FstSlice *s2); -void fstSliceDestroy(FstSlice *s); -uint8_t *fstSliceData(FstSlice *s, int32_t *sz); +FstSlice fstSliceCreate(uint8_t* data, uint64_t len); +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end); +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end); +bool fstSliceIsEmpty(FstSlice* s); +int fstSliceCompare(FstSlice* s1, FstSlice* s2); +void fstSliceDestroy(FstSlice* s); +uint8_t* fstSliceData(FstSlice* s, int32_t* sz); #define FST_SLICE_LEN(s) (s->end - s->start + 1) diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 7c6261aacf7ad14d58c883443cfadc16d50269d4..d711da18077a580643fcc3a2b84e257e4a4dd06c 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -26,84 +26,93 @@ extern "C" { #endif -// tfile header -// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| -// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| +// tfile header content +// |<---suid--->|<---version--->|<-------colName------>|<---type-->|<--fstOffset->| +// |<-uint64_t->|<---int32_t--->|<--TSDB_COL_NAME_LEN-->|<-uint8_t->|<---int32_t-->| -typedef struct TFileReadHeader { +#pragma pack(push, 1) +typedef struct TFileHeader { uint64_t suid; int32_t version; - char colName[128]; // + char colName[TSDB_COL_NAME_LEN]; // uint8_t colType; -} TFileReadHeader; + int32_t fstOffset; +} TFileHeader; +#pragma pack(pop) -#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); -#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) +#define TFILE_HEADER_SIZE (sizeof(TFileHeader)) +#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t)) +//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) typedef struct TFileCacheKey { - uint64_t suid; - uint8_t colType; - int32_t version; - const char *colName; - int32_t nColName; + uint64_t suid; + uint8_t colType; + int32_t version; + char* colName; + int32_t nColName; } TFileCacheKey; // table cache // refactor to LRU cache later typedef struct TFileCache { - SHashObj *tableCache; + SHashObj* tableCache; int16_t capacity; // add more param } TFileCache; typedef struct TFileWriter { - FstBuilder *fb; - WriterCtx * ctx; + FstBuilder* fb; + WriterCtx* ctx; + TFileHeader header; + uint32_t offset; } TFileWriter; typedef struct TFileReader { T_REF_DECLARE() - Fst * fst; - WriterCtx * ctx; - TFileReadHeader header; + Fst* fst; + WriterCtx* ctx; + TFileHeader header; } TFileReader; typedef struct IndexTFile { - char * path; - TFileCache * cache; - TFileWriter *tw; + char* path; + TFileCache* cache; + TFileWriter* tw; } IndexTFile; typedef struct TFileWriterOpt { uint64_t suid; int8_t colType; - char * colName; + char* colName; int32_t nColName; int32_t version; } TFileWriterOpt; typedef struct TFileReaderOpt { uint64_t suid; - char * colName; + char* colName; int32_t nColName; } TFileReaderOpt; // tfile cache, manage tindex reader -TFileCache * tfileCacheCreate(const char *path); -void tfileCacheDestroy(TFileCache *tcache); -TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); -void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); +TFileCache* tfileCacheCreate(const char* path); +void tfileCacheDestroy(TFileCache* tcache); +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); -TFileReader *tfileReaderCreate(); -void TFileReaderDestroy(TFileReader *reader); +TFileReader* tfileReaderCreate(WriterCtx* ctx); +void tfileReaderDestroy(TFileReader* reader); +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); -TFileWriter *tfileWriterCreate(const char *suid, const char *colName); -void tfileWriterDestroy(TFileWriter *tw); +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); +void tfileWriterDestroy(TFileWriter* tw); +int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterFinish(TFileWriter* tw); // -IndexTFile *indexTFileCreate(const char *path); -int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); +IndexTFile* indexTFileCreate(const char* path); +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index f708e71f57364a673d3f1c7327f932464702350c..21c5ca155b3bb252429be2dd6c30a344694b168a 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -19,29 +19,29 @@ extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void *)buf, (void *)key->mem, len); \ + memcpy((void*)buf, (void*)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(var) == sizeof(type)); \ - memcpy((void *)buf, (void *)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type)); \ + memcpy((void*)buf, (void*)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void *)buf, (void *)var, len); \ + memcpy((void*)buf, (void*)var, len); \ buf += len; \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index cc68324017f44960d14537c782806a8351c65f89..e88cecfd623c86d39ae28c9906c976f60c27927f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -23,9 +23,9 @@ #include "lucene++/Lucene_c.h" #endif -static int uidCompare(const void *a, const void *b) { - uint64_t u1 = *(uint64_t *)a; - uint64_t u2 = *(uint64_t *)b; +static int uidCompare(const void* a, const void* b) { + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; if (u1 == u2) { return 0; } else { @@ -40,25 +40,26 @@ typedef struct SIdxColInfo { static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); -static int indexMergeCacheIntoTindex(SIndex *sIdx); +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); +static int indexMergeCacheIntoTindex(SIndex* sIdx); -static void indexInterResultsDestroy(SArray *results); -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); +static void indexInterResultsDestroy(SArray* results); +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); -int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { +int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); - SIndex *sIdx = calloc(1, sizeof(SIndex)); + SIndex* sIdx = calloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE - index_t *index = index_open(path); + index_t* index = index_open(path); sIdx->index = index; #endif - sIdx->cache = (void *)indexCacheCreate(); +#ifdef USE_INVERTED_INDEX + sIdx->cache = (void*)indexCacheCreate(); sIdx->tindex = NULL; sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colId = 1; @@ -67,9 +68,13 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { *index = sIdx; return 0; +#endif + + *index = NULL; + return -1; } -void indexClose(SIndex *sIdx) { +void indexClose(SIndex* sIdx) { #ifdef USE_LUCENE index_close(sIdex->index); sIdx->index = NULL; @@ -84,16 +89,16 @@ void indexClose(SIndex *sIdx) { return; } -int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { +int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { #ifdef USE_LUCENE - index_document_t *doc = index_document_create(); + index_document_t* doc = index_document_create(); char buf[16] = {0}; sprintf(buf, "%d", uid); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); + SIndexTerm* p = taosArrayGetP(fVals, i); + index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1); } index_document_add(doc, NULL, 0, buf, strlen(buf), 0); @@ -106,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { // TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm * p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); if (fi == NULL) { SIdxColInfo tfi = {.colId = index->colId}; index->cVersion++; @@ -120,8 +125,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm * p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); assert(fi != NULL); int32_t colId = fi->colId; int32_t version = index->cVersion; @@ -134,18 +139,18 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { return 0; } -int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { +int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { #ifdef USE_LUCENE EIndexOperatorType opera = multiQuerys->opera; int nQuery = taosArrayGetSize(multiQuerys->query); - char **fields = malloc(sizeof(char *) * nQuery); - char **keys = malloc(sizeof(char *) * nQuery); - int * types = malloc(sizeof(int) * nQuery); + char** fields = malloc(sizeof(char*) * nQuery); + char** keys = malloc(sizeof(char*) * nQuery); + int* types = malloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { - SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); - SIndexTerm * term = p->field_value; + SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); + SIndexTerm* term = p->field_value; fields[i] = calloc(1, term->nKey + 1); keys[i] = calloc(1, term->nVal + 1); @@ -154,9 +159,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result memcpy(keys[i], term->val, term->nVal); types[i] = (int)(p->type); } - int *tResult = NULL; + int* tResult = NULL; int tsz = 0; - index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); + index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); @@ -174,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result #ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys - SArray *interResults = taosArrayInit(4, POINTER_BYTES); + SArray* interResults = taosArrayInit(4, POINTER_BYTES); int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); - SArray * tResult = NULL; + SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); + SArray* tResult = NULL; indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void *)&tResult); + taosArrayPush(interResults, (void*)&tResult); } indexMergeFinalResults(interResults, opera, result); indexInterResultsDestroy(interResults); @@ -189,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result return 1; } -int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { +int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { #ifdef USE_INVERTED_INDEX #endif return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts){ +int indexRebuild(SIndex* index, SIndexOpts* opts){ #ifdef USE_INVERTED_INDEX #endif } -SIndexOpts *indexOptsCreate() { +SIndexOpts* indexOptsCreate() { #ifdef USE_LUCENE #endif return NULL; } -void indexOptsDestroy(SIndexOpts *opts){ +void indexOptsDestroy(SIndexOpts* opts){ #ifdef USE_LUCENE #endif } /* @@ -214,8 +219,8 @@ void indexOptsDestroy(SIndexOpts *opts){ * */ -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } @@ -223,23 +228,28 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { - SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); + SIndexTermQuery* p = (SIndexTermQuery*)taosArrayGet(pQuery->query, i); indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) { +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, - int32_t nColName, const char *colVal, int32_t nColVal) { - SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn oper, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal) { + SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); if (t == NULL) { return NULL; } @@ -248,30 +258,32 @@ SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy t->operType = oper; t->colType = colType; - t->colName = (char *)calloc(1, nColName + 1); + t->colName = (char*)calloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char *)calloc(1, nColVal + 1); + t->colVal = (char*)calloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } -void indexTermDestroy(SIndexTerm *p) { +void indexTermDestroy(SIndexTerm* p) { free(p->colName); free(p->colVal); free(p); } -SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } +SIndexMultiTerm* indexMultiTermCreate() { + return taosArrayInit(4, sizeof(SIndexTerm*)); +} -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { taosArrayPush(terms, &term); return 0; } -void indexMultiTermDestroy(SIndexMultiTerm *terms) { +void indexMultiTermDestroy(SIndexMultiTerm* terms) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { - SIndexTerm *p = taosArrayGetP(terms, i); + SIndexTerm* p = taosArrayGetP(terms, i); indexTermDestroy(p); } taosArrayDestroy(terms); @@ -280,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) { void indexInit() { // do nothing } -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) { int32_t version = -1; int16_t colId = -1; - SIdxColInfo *colInfo = NULL; + SIdxColInfo* colInfo = NULL; - SIndexTerm *term = query->term; - const char *colName = term->colName; + SIndexTerm* term = query->term; + const char* colName = term->colName; int32_t nColName = term->nColName; pthread_mutex_lock(&sIdx->mtx); @@ -319,23 +331,24 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result } return 0; } -static void indexInterResultsDestroy(SArray *results) { +static void indexInterResultsDestroy(SArray* results) { if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { - SArray *p = taosArrayGetP(results, i); + SArray* p = taosArrayGetP(results, i); taosArrayDestroy(p); } taosArrayDestroy(results); } -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray *first = taosArrayGetP(interResults, 0); + SArray* first = taosArrayGetP(interResults, 0); taosArraySort(first, uidCompare); taosArrayRemoveDuplicate(first, uidCompare, NULL); + if (oType == MUST) { // just one column index, enhance later taosArrayAddAll(fResults, first); @@ -350,7 +363,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType } return 0; } -static int indexMergeCacheIntoTindex(SIndex *sIdx) { +static int indexMergeCacheIntoTindex(SIndex* sIdx) { if (sIdx == NULL) { return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index dd8a8bcbb6e31031429e7181793fafc65cc004c8..6170642707434b4daaf22bc14c9f8058f841e611 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,14 +20,15 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later // ref index_cache.h:22 -#define CACHE_KEY_LEN(p) \ - (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \ - sizeof(p->operType)) +#define CACHE_KEY_LEN(p) \ + (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) -static char * getIndexKey(const void *pData) { return NULL; } -static int32_t compareKey(const void *l, const void *r) { - char *lp = (char *)l; - char *rp = (char *)r; +static char* getIndexKey(const void* pData) { + return NULL; +} +static int32_t compareKey(const void* l, const void* r) { + char* lp = (char*)l; + char* rp = (char*)r; // skip total len, not compare int32_t ll, rl; // len @@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) { int16_t lf, rf; // field id memcpy(&lf, lp, sizeof(lf)); memcpy(&rf, rp, sizeof(rf)); - if (lf != rf) { - return lf < rf ? -1 : 1; - } + if (lf != rf) { return lf < rf ? -1 : 1; } lp += sizeof(lf); rp += sizeof(rf); @@ -89,41 +88,41 @@ static int32_t compareKey(const void *l, const void *r) { int32_t lv, rv; memcpy(&lv, lp, sizeof(lv)); memcpy(&rv, rp, sizeof(rv)); - if (lv != rv) { - return lv > rv ? -1 : 1; - } + if (lv != rv) { return lv > rv ? -1 : 1; } + lp += sizeof(lv); rp += sizeof(rv); // not care item type return 0; } -IndexCache *indexCacheCreate() { - IndexCache *cache = calloc(1, sizeof(IndexCache)); - cache->skiplist = tSkipListCreate( - MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); +IndexCache* indexCacheCreate() { + IndexCache* cache = calloc(1, sizeof(IndexCache)); + if (cache == NULL) { + indexError("failed to create index cache"); + return NULL; + } + cache->skiplist = + tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); return cache; } -void indexCacheDestroy(void *cache) { - IndexCache *pCache = cache; - if (pCache == NULL) { - return; - } +void indexCacheDestroy(void* cache) { + IndexCache* pCache = cache; + if (pCache == NULL) { return; } tSkipListDestroy(pCache->skiplist); free(pCache); } -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { - if (cache == NULL) { - return -1; - } +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { + if (cache == NULL) { return -1; } - IndexCache *pCache = cache; + IndexCache* pCache = cache; // encode data int32_t total = CACHE_KEY_LEN(term); - char * buf = calloc(1, total); - char * p = buf; + + char* buf = calloc(1, total); + char* p = buf; SERIALIZE_VAR_TO_BUF(p, total, int32_t); SERIALIZE_VAR_TO_BUF(p, colId, int16_t); @@ -137,30 +136,31 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, SERIALIZE_MEM_TO_BUF(p, term, operType); - tSkipListPut(pCache->skiplist, (void *)buf); + tSkipListPut(pCache->skiplist, (void*)buf); return 0; // encode end } -int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { - IndexCache *pCache = cache; +int indexCacheDel(void* cache, int32_t fieldId, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + IndexCache* pCache = cache; return 0; } -int indexCacheSearch( - void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { - if (cache == NULL) { - return -1; - } - IndexCache * pCache = cache; - SIndexTerm * term = query->term; +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; + SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; int32_t keyLen = CACHE_KEY_LEN(term); - char *buf = calloc(1, keyLen); + char* buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { + // } else if (qtype == QUERY_PREFIX) { + // } else if (qtype == QUERY_SUFFIX) { + // } else if (qtype == QUERY_REGEX) { + // } return 0; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index adb002c2b74301403e599a71935a66c7d5a6759b..18514cd0d5498eba9a31d8d28b197557bdf23bd8 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -19,43 +19,39 @@ #include "tchecksum.h" #include "tcoding.h" -static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); } -static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); return nBytes; } -FstUnFinishedNodes *fstUnFinishedNodesCreate() { - FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); - if (nodes == NULL) { - return NULL; - } +FstUnFinishedNodes* fstUnFinishedNodesCreate() { + FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); + if (nodes == NULL) { return NULL; } - nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); + nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -void unFinishedNodeDestroyElem(void *elem) { - FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished *)elem; +void unFinishedNodeDestroyElem(void* elem) { + FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); free(b->last); b->last = NULL; } -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { - if (nodes == NULL) { - return; - } +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { + if (nodes == NULL) { return; } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); } -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -63,56 +59,53 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); } -FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { assert(taosArrayGetSize(nodes->stack) == 1); - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); assert(un->last == NULL); return un->node; } -FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); // free(un->last); // TODO add func FstLastTransitionFree() // un->last = NULL; return un->node; } -FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); assert(un->last == NULL); return un->node; } -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); un->node->isFinal = true; un->node->finalOutput = out; // un->node->trans = NULL; } -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); fstBuilderNodeUnfinishedLastCompiled(un, addr); } -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { - FstSlice *s = &bs; - if (fstSliceIsEmpty(s)) { - return; - } +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { + FstSlice* s = &bs; + if (fstSliceIsEmpty(s)) { return; } size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; // trn->out = out; int32_t len = 0; - uint8_t *data = fstSliceData(s, &len); + uint8_t* data = fstSliceData(s, &len); un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -120,7 +113,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // trn->inp = s->data[i]; // trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); + FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; taosArrayPush(nodes->stack, &un); @@ -128,15 +121,15 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output fstUnFinishedNodesPushEmpty(nodes, true); } -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { - FstSlice *s = &bs; +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { + FstSlice* s = &bs; size_t ssz = taosArrayGetSize(node->stack); // stack size uint64_t count = 0; int32_t lsz; // data len - uint8_t *data = fstSliceData(s, &lsz); + uint8_t* data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); if (un->last->inp == data[i]) { count++; } else { @@ -145,19 +138,19 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) } return count; } -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) { - FstSlice *s = &bs; +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { + FstSlice* s = &bs; size_t lsz = (size_t)(s->end - s->start + 1); // data len size_t ssz = taosArrayGetSize(node->stack); // stack size *out = in; uint64_t i = 0; for (i = 0; i < lsz && i < ssz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); - FstLastTransition *t = un->last; + FstLastTransition* t = un->last; uint64_t addPrefix = 0; - uint8_t * data = fstSliceData(s, NULL); + uint8_t* data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { uint64_t commPrefix = MIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; @@ -169,7 +162,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, } if (addPrefix != 0) { if (i + 1 < ssz) { - FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); + FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); } } @@ -177,13 +170,11 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, return i; } -FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { +FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; - if (addr == EMPTY_ADDRESS) { - return fs; - } + if (addr == EMPTY_ADDRESS) { return fs; } - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); uint8_t v = data[addr]; uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { @@ -197,17 +188,19 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { return fs; } -static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000}, - {.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}}; +static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, + {.state = OneTrans, .val = 0b10000000}, + {.state = AnyTrans, .val = 0b00000000}, + {.state = EmptyFinal, .val = 0b00000000}}; // debug -static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; FstState fstStateCreate(State state) { uint8_t idx = (uint8_t)state; return fstStateDict[idx]; } // compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp) { +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { FstState s = fstStateCreate(OneTransNext); fstStateSetCommInput(&s, inp); @@ -221,7 +214,7 @@ void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uin // w->write_all(&[s.val]) return; } -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn) { +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { Output out = trn->out; uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); @@ -229,20 +222,18 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); - fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); + fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); bool null = false; uint8_t inp = fstStateCommInput(&st, &null); - if (null == true) { - fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp)); - } - fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val)); + if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } + fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); return; } -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { size_t sz = taosArrayGetSize(node->trans); assert(sz <= 256); @@ -252,7 +243,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // finalOutput.is_zero() bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); tSize = MAX(tSize, packDeltaSize(addr, t->addr)); oSize = MAX(oSize, packSize(t->out)); anyOuts = anyOuts || (t->out != 0); @@ -272,21 +263,19 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil fstStateSetStateNtrans(&st, (uint8_t)sz); if (anyOuts) { - if (FST_BUILDER_NODE_IS_FINAL(node)) { - fstCountingWriterPackUintIn(w, node->finalOutput, oSize); - } + if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); } } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); fstPackDeltaIn(w, addr, t->addr, tSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstCountingWriterWrite(w, (char *)&t->inp, 1); + FstTransition* t = taosArrayGet(node->trans, i); + fstCountingWriterWrite(w, (char*)&t->inp, 1); // fstPackDeltaIn(w, addr, t->addr, tSize); } if (sz > TRANS_INDEX_THRESHOLD) { @@ -294,20 +283,20 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; ///} for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); index[t->inp] = i; // fstPackDeltaIn(w, addr, t->addr, tSize); } - fstCountingWriterWrite(w, (char *)index, 256); + fstCountingWriterWrite(w, (char*)index, 256); free(index); } - fstCountingWriterWrite(w, (char *)&packSizes, 1); + fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; fstStateStateNtrans(&st, &null); if (null == true) { @@ -316,17 +305,17 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // encoded in the state byte. uint8_t v = 1; if (sz == 256) { - fstCountingWriterWrite(w, (char *)&v, 1); + fstCountingWriterWrite(w, (char*)&v, 1); } else { - fstCountingWriterWrite(w, (char *)&sz, 1); + fstCountingWriterWrite(w, (char*)&sz, 1); } } - fstCountingWriterWrite(w, (char *)(&(st.val)), 1); + fstCountingWriterWrite(w, (char*)(&(st.val)), 1); return; } // set_comm_input -void fstStateSetCommInput(FstState *s, uint8_t inp) { +void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; @@ -335,7 +324,7 @@ void fstStateSetCommInput(FstState *s, uint8_t inp) { } // comm_input -uint8_t fstStateCommInput(FstState *s, bool *null) { +uint8_t fstStateCommInput(FstState* s, bool* null) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; if (v == 0) { @@ -348,7 +337,7 @@ uint8_t fstStateCommInput(FstState *s, bool *null) { // input_len -uint64_t fstStateInputLen(FstState *s) { +uint64_t fstStateInputLen(FstState* s) { assert(s->state == OneTransNext || s->state == OneTrans); bool null = false; fstStateCommInput(s, &null); @@ -356,17 +345,16 @@ uint64_t fstStateInputLen(FstState *s) { } // end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState *s, FstSlice *data) { +uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { assert(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } -uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { +uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { assert(s->state == OneTrans); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } -uint64_t fstStateEndAddrForAnyTrans( - FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size @@ -374,29 +362,29 @@ uint64_t fstStateEndAddrForAnyTrans( - finalOsize; // final output } // input -uint8_t fstStateInput(FstState *s, FstNode *node) { +uint8_t fstStateInput(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); - uint8_t * data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return null == false ? inp : data[-1]; } -uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return data[at]; } // trans_addr -CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { +CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (s->state == OneTransNext) { return (CompiledAddr)(node->end) - 1; } else { @@ -406,23 +394,23 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { - tSizes; // refactor error logic - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + i, tSizes, node->end); } } -CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - - node->nTrans - (i * tSizes) - tSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans - + (i * tSizes) - tSizes; + uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + at, tSizes, node->end); } // sizes -PackSizes fstStateSizes(FstState *s, FstSlice *slice) { +PackSizes fstStateSizes(FstState* s, FstSlice* slice) { assert(s->state == OneTrans || s->state == AnyTrans); uint64_t i; if (s->state == OneTrans) { @@ -431,33 +419,29 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return (PackSizes)(*(data + i)); } // Output -Output fstStateOutput(FstState *s, FstNode *node) { +Output fstStateOutput(FstState* s, FstNode* node) { assert(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackUint64(data + i, oSizes); } -Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; - uint8_t * data = fstSliceData(slice, NULL); + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; + uint8_t* data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; @@ -466,27 +450,23 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { // anyTrans specify function -void fstStateSetFinalState(FstState *s, bool yes) { +void fstStateSetFinalState(FstState* s, bool yes) { assert(s->state == AnyTrans); - if (yes) { - s->val |= 0b01000000; - } + if (yes) { s->val |= 0b01000000; } return; } -bool fstStateIsFinalState(FstState *s) { +bool fstStateIsFinalState(FstState* s) { assert(s->state == AnyTrans); return (s->val & 0b01000000) == 0b01000000; } -void fstStateSetStateNtrans(FstState *s, uint8_t n) { +void fstStateSetStateNtrans(FstState* s, uint8_t n) { assert(s->state == AnyTrans); - if (n <= 0b00111111) { - s->val = (s->val & 0b11000000) | n; - } + if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } return; } // state_ntrans -uint8_t fstStateStateNtrans(FstState *s, bool *null) { +uint8_t fstStateStateNtrans(FstState* s, bool* null) { assert(s->state == AnyTrans); *null = false; uint8_t n = s->val & 0b00111111; @@ -496,58 +476,52 @@ uint8_t fstStateStateNtrans(FstState *s, bool *null) { } return n; } -uint64_t fstStateTotalTransSize(FstState *s, uint64_t version, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { assert(s->state == AnyTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } -uint64_t fstStateTransIndexSize(FstState *s, uint64_t version, uint64_t nTrans) { +uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { assert(s->state == AnyTrans); return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } -uint64_t fstStateNtransLen(FstState *s) { +uint64_t fstStateNtransLen(FstState* s) { assert(s->state == AnyTrans); bool null = false; fstStateStateNtrans(s, &null); return null == true ? 1 : 0; } -uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { +uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { bool null = false; uint8_t n = fstStateStateNtrans(s, &null); - if (null != true) { - return n; - } + if (null != true) { return n; } int32_t len; - uint8_t *data = fstSliceData(slice, &len); + uint8_t* data = fstSliceData(slice, &len); n = data[len - 2]; // n = data[slice->end - 1]; // data[data.len() - 2] return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, // then it is encoded in the state byte } -Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { +Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { - return 0; - } + if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return unpackUint64(data + at, (uint8_t)oSizes); } -uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { +uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); int32_t dlen = 0; - uint8_t *data = fstSliceData(slice, &dlen); + uint8_t* data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; // uint64_t i = slice->data[slice->start + at + b]; - if (i >= node->nTrans) { - *null = true; - } + if (i >= node->nTrans) { *null = true; } return i; } else { uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size @@ -555,7 +529,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint64_t end = start + node->nTrans; FstSlice t = fstSliceCopy(slice, start, end - 1); int32_t len = 0; - uint8_t *data = fstSliceData(&t, &len); + uint8_t* data = fstSliceData(&t, &len); int i = 0; for (; i < len; i++) { uint8_t v = data[i]; @@ -564,20 +538,16 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { return node->nTrans - i - 1; // bug } } - if (i == len) { - *null = true; - } + if (i == len) { *null = true; } fstSliceDestroy(&t); } } // fst node function -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { - FstNode *n = (FstNode *)malloc(sizeof(FstNode)); - if (n == NULL) { - return NULL; - } +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { + FstNode* n = (FstNode*)malloc(sizeof(FstNode)); + if (n == NULL) { return NULL; } FstState st = fstStateCreateFrom(slice, addr); @@ -625,27 +595,24 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->nTrans = nTrans; n->sizes = sz; - n->finalOutput = - fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); } return n; } // debug state transition -static const char *fstNodeState(FstNode *node) { - FstState *st = &node->state; +static const char* fstNodeState(FstNode* node) { + FstState* st = &node->state; return fstStateStr[st->state]; } -void fstNodeDestroy(FstNode *node) { +void fstNodeDestroy(FstNode* node) { fstSliceDestroy(&node->data); free(node); } -FstTransitions *fstNodeTransitions(FstNode *node) { - FstTransitions *t = malloc(sizeof(FstTransitions)); - if (NULL == t) { - return NULL; - } +FstTransitions* fstNodeTransitions(FstNode* node) { + FstTransitions* t = malloc(sizeof(FstTransitions)); + if (NULL == t) { return NULL; } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; t->node = node; @@ -653,9 +620,9 @@ FstTransitions *fstNodeTransitions(FstNode *node) { } // Returns the transition at index `i`. -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { trn->inp = fstStateInput(st, node); trn->out = 0; @@ -675,9 +642,9 @@ bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { } // Returns the transition address of the `i`th transition -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { assert(i == 0); fstStateTransAddr(st, node); @@ -696,9 +663,9 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { // Finds the `i`th transition corresponding to the given input byte. // If no transition for this byte exists, then `false` is returned. -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { bool s = true; - FstState *st = &node->state; + FstState* st = &node->state; if (st->state == OneTransNext) { if (fstStateInput(st, node) == b) { *res = 0; @@ -723,7 +690,7 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { return s; } -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode) { +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { size_t sz = taosArrayGetSize(builderNode->trans); assert(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { @@ -732,7 +699,7 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr fstStateCompileForAnyTrans(w, addr, builderNode); // AnyTrans->Compile(w, addr, node); } else { - FstTransition *tran = taosArrayGet(builderNode->trans, 0); + FstTransition* tran = taosArrayGet(builderNode->trans, 0); if (tran->addr == lastAddr && tran->out == 0) { fstStateCompileForOneTransNext(w, addr, tran->inp); // OneTransNext::compile(w, lastAddr, tran->inp); @@ -746,15 +713,13 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr return true; } -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); } -FstBuilder *fstBuilderCreate(void *w, FstType ty) { - FstBuilder *b = malloc(sizeof(FstBuilder)); - if (NULL == b) { - return b; - } +FstBuilder* fstBuilderCreate(void* w, FstType ty) { + FstBuilder* b = malloc(sizeof(FstBuilder)); + if (NULL == b) { return b; } b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); @@ -764,7 +729,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { b->len = 0; char buf64[8] = {0}; - void *pBuf64 = buf64; + void* pBuf64 = buf64; taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); @@ -775,10 +740,8 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { return b; } -void fstBuilderDestroy(FstBuilder *b) { - if (b == NULL) { - return; - } +void fstBuilderDestroy(FstBuilder* b) { + if (b == NULL) { return; } fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); @@ -787,19 +750,19 @@ void fstBuilderDestroy(FstBuilder *b) { free(b); } -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { OrderType t = fstBuilderCheckLastKey(b, bs, true); if (t == Ordered) { // add log info fstBuilderInsertOutput(b, bs, in); return true; } - indexInfo("key must be ordered"); + indexInfo("fst write key must be ordered"); return false; } -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { - FstSlice *s = &bs; +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { + FstSlice* s = &bs; if (fstSliceIsEmpty(s)) { b->len = 1; fstUnFinishedNodesSetRootOutput(b->unfinished, in); @@ -828,8 +791,8 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { return; } -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { - FstSlice *input = &bs; +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { + FstSlice* input = &bs; if (fstSliceIsEmpty(&b->last)) { fstSliceDestroy(&b->last); // deep copy or not @@ -847,10 +810,10 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } return Ordered; } -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { - FstBuilderNode *bn = NULL; + FstBuilderNode* bn = NULL; if (addr == NONE_ADDRESS) { bn = fstUnFinishedNodesPopEmpty(b->unfinished); } else { @@ -865,11 +828,11 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; } -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { return EMPTY_ADDRESS; } - FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); + FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); if (entry->state == FOUND) { CompiledAddr ret = entry->addr; fstRegistryEntryDestroy(entry); @@ -879,23 +842,21 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); - if (entry->state == NOTFOUND) { - FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); - } + if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } fstRegistryEntryDestroy(entry); return b->lastAddr; } -void *fstBuilderInsertInner(FstBuilder *b) { +void* fstBuilderInsertInner(FstBuilder* b) { fstBuilderCompileFrom(b, 0); - FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); + FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); fstBuilderNodeDestroy(rootNode); char buf64[8] = {0}; - void *pBuf64 = buf64; + void* pBuf64 = buf64; taosEncodeFixedU64(&pBuf64, b->len); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); @@ -904,7 +865,7 @@ void *fstBuilderInsertInner(FstBuilder *b) { fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); char buf32[4] = {0}; - void * pBuf32 = buf32; + void* pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); taosEncodeFixedU32(&pBuf32, sum); fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); @@ -914,31 +875,31 @@ void *fstBuilderInsertInner(FstBuilder *b) { // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); } +void fstBuilderFinish(FstBuilder* b) { + fstBuilderInsertInner(b); +} -FstSlice fstNodeAsSlice(FstNode *node) { - FstSlice *slice = &node->data; +FstSlice fstNodeAsSlice(FstNode* node) { + FstSlice* slice = &node->data; FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); return s; } -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - if (trn == NULL) { - return NULL; - } +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { + FstLastTransition* trn = malloc(sizeof(FstLastTransition)); + if (trn == NULL) { return NULL; } trn->inp = inp; trn->out = out; return trn; } -void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); } -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { - FstLastTransition *trn = unNode->last; - if (trn == NULL) { - return; - } +void fstLastTransitionDestroy(FstLastTransition* trn) { + free(trn); +} +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { + FstLastTransition* trn = unNode->last; + if (trn == NULL) { return; } FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; taosArrayPush(unNode->node->trans, &t); fstLastTransitionDestroy(trn); @@ -946,36 +907,28 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp return; } -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, Output out) { - if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { - unNode->node->finalOutput += out; - } +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { + if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(unNode->node->trans, i); + FstTransition* trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } - if (unNode->last) { - unNode->last->out += out; - } + if (unNode->last) { unNode->last->out += out; } return; } -Fst *fstCreate(FstSlice *slice) { +Fst* fstCreate(FstSlice* slice) { int32_t slen; - char * buf = fstSliceData(slice, &slen); - if (slen < 36) { - return NULL; - } + char* buf = fstSliceData(slice, &slen); + if (slen < 36) { return NULL; } uint64_t len = slen; uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); skip += sizeof(version); - if (version == 0 || version > VERSION) { - return NULL; - } + if (version == 0 || version > VERSION) { return NULL; } uint64_t type; taosDecodeFixedU64(buf + skip, &type); @@ -993,15 +946,11 @@ Fst *fstCreate(FstSlice *slice) { len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) - Fst *fst = (Fst *)calloc(1, sizeof(Fst)); - if (fst == NULL) { - return NULL; - } + Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + if (fst == NULL) { return NULL; } - fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { - goto FST_CREAT_FAILED; - } + fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + if (NULL == fst->meta) { goto FST_CREAT_FAILED; } fst->meta->version = version; fst->meta->rootAddr = rootAddr; @@ -1009,7 +958,7 @@ Fst *fstCreate(FstSlice *slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice *s = calloc(1, sizeof(FstSlice)); + FstSlice* s = calloc(1, sizeof(FstSlice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); fst->data = s; @@ -1019,7 +968,7 @@ FST_CREAT_FAILED: free(fst->meta); free(fst); } -void fstDestroy(Fst *fst) { +void fstDestroy(Fst* fst) { if (fst) { free(fst->meta); fstSliceDestroy(fst->data); @@ -1028,20 +977,18 @@ void fstDestroy(Fst *fst) { free(fst); } -bool fstGet(Fst *fst, FstSlice *b, Output *out) { - FstNode *root = fstGetRoot(fst); +bool fstGet(Fst* fst, FstSlice* b, Output* out) { + FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; - uint8_t *data = fstSliceData(b, &len); + uint8_t* data = fstSliceData(b, &len); - SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; - if (false == fstNodeFindInput(root, inp, &res)) { - return false; - } + if (false == fstNodeFindInput(root, inp, &res)) { return false; } FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); @@ -1056,7 +1003,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); @@ -1066,30 +1013,36 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { return true; } -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } -StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { - if (sb == NULL) { - return NULL; - } +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { + return fstStreamBuilderCreate(fst, ctx); +} +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { + if (sb == NULL) { return NULL; } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { + return fstStreamBuilderCreate(fst, ctx); +} -FstNode *fstGetRoot(Fst *fst) { - if (fst->root != NULL) { - return fst->root; - } +FstNode* fstGetRoot(Fst* fst) { + if (fst->root != NULL) { return fst->root; } CompiledAddr rAddr = fstGetRootAddr(fst); fst->root = fstGetNode(fst, rAddr); return fst->root; } -FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); } -FstType fstGetType(Fst *fst) { return fst->meta->ty; } -CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; } +FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { + return fstNodeCreate(fst->meta->version, addr, fst->data); +} +FstType fstGetType(Fst* fst) { + return fst->meta->ty; +} +CompiledAddr fstGetRootAddr(Fst* fst) { + return fst->meta->rootAddr; +} -Output fstEmptyFinalOutput(Fst *fst, bool *null) { +Output fstEmptyFinalOutput(Fst* fst, bool* null) { Output res = 0; - FstNode *node = fstGetRoot(fst); + FstNode* node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; res = FST_NODE_FINAL_OUTPUT(node); @@ -1099,23 +1052,19 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { return res; } -bool fstVerify(Fst *fst) { +bool fstVerify(Fst* fst) { uint32_t checkSum = fst->meta->checkSum; int32_t len; - uint8_t *data = fstSliceData(fst->data, &len); + uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data, len)) { - return false; - } + if (!taosCheckChecksumWhole(data, len)) { return false; } return true; } // data bound function -FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { - FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); - if (b == NULL) { - return NULL; - } +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { + FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); + if (b == NULL) { return NULL; } if (data != NULL) { b->data = fstSliceCopy(data, data->start, data->end); @@ -1127,7 +1076,7 @@ FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { return b; } -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { return comp > 0 ? true : false; @@ -1137,7 +1086,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { return true; } } -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { if (bound->type == Unbounded) { return true; } else { @@ -1145,34 +1094,33 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { } } -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; } +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { + return bound->type == Excluded ? false : true; +} -void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } +void fstBoundDestroy(FstBoundWithData* bound) { + free(bound); +} -StreamWithState *streamWithStateCreate( - Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { - StreamWithState *sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { - return NULL; - } +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { + StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + if (sws == NULL) { return NULL; } sws->fst = fst; sws->aut = automation; - sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); + sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); sws->emptyOutput.null = false; sws->emptyOutput.out = 0; - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); sws->endAt = max; streamWithStateSeekMin(sws, min); return sws; } -void streamWithStateDestroy(StreamWithState *sws) { - if (sws == NULL) { - return; - } +void streamWithStateDestroy(StreamWithState* sws) { + if (sws == NULL) { return; } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1180,22 +1128,19 @@ void streamWithStateDestroy(StreamWithState *sws) { free(sws); } -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { - AutomationCtx *aut = sws->aut; +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { + AutomationCtx* aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { - if (fstBoundWithDataIsIncluded(min)) { - sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); - } + if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); } StreamState s = {.node = fstGetRoot(sws->fst), - .trans = 0, - .out = {.null = false, .out = 0}, - .autState = automFuncs[aut->type].start(aut)}; // auto.start callback + .trans = 0, + .out = {.null = false, .out = 0}, + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; } - FstSlice *key = NULL; + FstSlice* key = NULL; bool inclusize = false; - ; if (min->type == Included) { key = &min->data; @@ -1206,13 +1151,13 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { return false; } - FstNode *node = fstGetRoot(sws->fst); + FstNode* node = fstGetRoot(sws->fst); Output out = 0; // void* autState = sws->aut->start(); - void *autState = automFuncs[aut->type].start(aut); + void* autState = automFuncs[aut->type].start(aut); int32_t len; - uint8_t *data = fstSliceData(key, &len); + uint8_t* data = fstSliceData(key, &len); for (uint32_t i = 0; i < len; i++) { uint8_t b = data[i]; uint64_t res = 0; @@ -1220,7 +1165,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { if (null == false) { FstTransition trn; fstNodeGetTransitionAt(node, res, &trn); - void *preState = autState; + void* preState = autState; // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); @@ -1235,13 +1180,11 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { // Since this is a minimum bound, we need to find the // first transition in this node that proceeds the current // input byte. - FstTransitions *trans = fstNodeTransitions(node); + FstTransitions* trans = fstNodeTransitions(node); uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; - if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { - break; - } + if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } } StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; @@ -1251,17 +1194,16 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { - StreamState *s = taosArrayGet(sws->stack, sz - 1); + StreamState* s = taosArrayGet(sws->stack, sz - 1); if (inclusize) { s->trans -= 1; taosArrayPop(sws->inp); } else { - FstNode * n = s->node; + FstNode* n = s->node; uint64_t trans = s->trans; FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = { - .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; + StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); return true; } @@ -1269,48 +1211,44 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } } -StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { - AutomationCtx *aut = sws->aut; +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { + AutomationCtx* aut = sws->aut; FstOutput output = sws->emptyOutput; if (output.null == false) { FstSlice emptySlice = fstSliceCreate(NULL, 0); if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); return NULL; } - void *start = automFuncs[aut->type].start(aut); + void* start = automFuncs[aut->type].start(aut); if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); return swsResultCreate(&s, output, callback(start)); } } - SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); + SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); while (taosArrayGetSize(sws->stack) > 0) { - StreamState *p = (StreamState *)taosArrayPop(sws->stack); + StreamState* p = (StreamState*)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { - if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { - taosArrayPop(sws->inp); - } + if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } streamStateDestroy(p); continue; } FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); Output out = p->out.out + trn.out; - void * nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); - void * tState = callback(nextState); + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); + void* tState = callback(nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - FstNode *nextNode = fstGetNode(sws->fst, trn.addr); + FstNode* nextNode = fstGetNode(sws->fst, trn.addr); taosArrayPush(nodes, &nextNode); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { // void *eofState = sws->aut->acceptEof(nextState); - void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); - if (eofState != NULL) { - isMatch = automFuncs[aut->type].isMatch(aut, eofState); - } + void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); + if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } } StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); @@ -1319,21 +1257,21 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb taosArrayPush(sws->stack, &s2); size_t isz = taosArrayGetSize(sws->inp); - uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { - buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); + buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); return result; @@ -1342,18 +1280,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); return NULL; } -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { - StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); - if (result == NULL) { - return NULL; - } +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { + StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); + if (result == NULL) { return NULL; } result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->out = fOut; @@ -1361,31 +1297,25 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta return result; } -void swsResultDestroy(StreamWithStateResult *result) { - if (NULL == result) { - return; - } +void swsResultDestroy(StreamWithStateResult* result) { + if (NULL == result) { return; } fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); free(result); } -void streamStateDestroy(void *s) { - if (NULL == s) { - return; - } - StreamState *ss = (StreamState *)s; +void streamStateDestroy(void* s) { + if (NULL == s) { return; } + StreamState* ss = (StreamState*)s; fstNodeDestroy(ss->node); // free(s->autoState); } -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { - FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); - if (NULL == b) { - return NULL; - } +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { + FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); + if (NULL == b) { return NULL; } b->fst = fst; b->aut = aut; @@ -1393,17 +1323,15 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { b->max = fstBoundStateCreate(Unbounded, NULL); return b; } -void fstStreamBuilderDestroy(FstStreamBuilder *b) { +void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); tfree(b->min); fstSliceDestroy(&b->max->data); tfree(b->max); free(b); } -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { - if (b == NULL) { - return NULL; - } +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { + if (b == NULL) { return NULL; } if (type == GE) { b->min->type = Included; diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index cf3165709cdb961190d32dd646a54468c6d6ae95..eb4c479c9406d4be367df458235c47f7b87ff9f8 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -15,19 +15,17 @@ #include "index_fst_automation.h" -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } nsv->kind = kind; nsv->type = ty; if (ty == FST_INT) { - nsv->val = *(int *)val; + nsv->val = *(int*)val; } else if (ty == FST_CHAR) { - size_t len = strlen((char *)val); - nsv->ptr = (char *)calloc(1, len + 1); + size_t len = strlen((char*)val); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp } return nsv; } -void startWithStateValueDestroy(void *val) { - StartWithStateValue *sv = (StartWithStateValue *)val; - if (sv == NULL) { - return; - } +void startWithStateValueDestroy(void* val) { + StartWithStateValue* sv = (StartWithStateValue*)val; + if (sv == NULL) { return; } if (sv->type == FST_INT) { // @@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) { } free(sv); } -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } nsv->kind = sv->kind; nsv->type = sv->type; @@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char *)calloc(1, len + 1); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { + // } return nsv; } // prefix query, impl later -static void *prefixStart(AutomationCtx *ctx) { - StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); +static void* prefixStart(AutomationCtx* ctx) { + StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata); return startWithStateValueDump(data); }; -static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue *ssv = (StartWithStateValue *)sv; +static bool prefixIsMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val == strlen(ctx->data); } -static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue *ssv = (StartWithStateValue *)sv; +static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val >= 0; } -static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } -static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { - StartWithStateValue *ssv = (StartWithStateValue *)state; - if (ssv == NULL || ctx == NULL) { - return NULL; - } +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { + return true; +} +static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { + StartWithStateValue* ssv = (StartWithStateValue*)state; + if (ssv == NULL || ctx == NULL) { return NULL; } - char *data = ctx->data; - if (ssv->kind == Done) { - return startWithStateValueCreate(Done, FST_INT, &ssv->val); - } + char* data = ctx->data; + if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; - StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; } else { @@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { } return NULL; } -static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { + return NULL; +} // pattern query, impl later -static void *patternStart(AutomationCtx *ctx) { return NULL; } -static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; } -static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; } -static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } +static void* patternStart(AutomationCtx* ctx) { + return NULL; +} +static bool patternIsMatch(AutomationCtx* ctx, void* data) { + return true; +} +static bool patternCanMatch(AutomationCtx* ctx, void* data) { + return true; +} +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { + return true; +} -static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; } +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { + return NULL; +} -static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { + return NULL; +} AutomationFunc automFuncs[] = { {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, @@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = { // add more search type }; -AutomationCtx *automCtxCreate(void *data, AutomationType atype) { - AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { - return NULL; - } +AutomationCtx* automCtxCreate(void* data, AutomationType atype) { + AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); + if (ctx == NULL) { return NULL; } - StartWithStateValue *sv = NULL; + StartWithStateValue* sv = NULL; if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void *)sv; + ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type } - char * src = (char *)data; + char* src = (char*)data; size_t len = strlen(src); - char * dst = (char *)malloc(len * sizeof(char) + 1); + char* dst = (char*)malloc(len * sizeof(char) + 1); memcpy(dst, src, len); dst[len] = 0; ctx->data = dst; ctx->type = atype; - ctx->stdata = (void *)sv; + ctx->stdata = (void*)sv; return ctx; } -void automCtxDestroy(AutomationCtx *ctx) { +void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); free(ctx->data); free(ctx); diff --git a/source/libs/index/src/index_fst_common.c b/source/libs/index/src/index_fst_common.c index ee3b07713da13008c29c863e164e72f9b56a3cda..2d7586e8d939c8ef6dc80e998e97f3cf0b08f921 100644 --- a/source/libs/index/src/index_fst_common.c +++ b/source/libs/index/src/index_fst_common.c @@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = { }; const char COMMON_INPUTS_INV[] = { - 't', - 'e', - '/', - 'o', - 'a', - 's', - 'r', - 'i', - 'p', - 'c', - 'n', - 'w', - '.', - 'h', - 'l', - 'm', - '-', - 'd', - 'u', - '0', - '1', - '2', - 'g', - '=', - ':', - 'b', - 'f', - '3', - 'y', - '5', - '&', - '_', - '4', - 'v', - '9', - '6', - '7', - '8', - 'k', - '%', - '?', - 'x', - 'C', - 'D', - 'A', - 'S', - 'F', - 'I', - 'B', - 'E', - 'j', - 'P', - 'T', - 'z', - 'R', - 'N', - 'M', - '+', - 'L', - 'O', - 'q', - 'H', - 'G', - 'W', - 'U', - 'V', - ',', - 'Y', - 'K', - 'J', - 'Z', - 'X', - 'Q', - ';', - ')', - '(', - '~', - '[', - ']', - '$', - '!', - '\'', - '*', - '@', - '\x00', - '\x01', - '\x02', - '\x03', - '\x04', - '\x05', - '\x06', - '\x07', - '\x08', - '\t', - '\n', - '\x0b', - '\x0c', - '\r', - '\x0e', - '\x0f', - '\x10', - '\x11', - '\x12', - '\x13', - '\x14', - '\x15', - '\x16', - '\x17', - '\x18', - '\x19', - '\x1a', - '\x1b', - '\x1c', - '\x1d', - '\x1e', - '\x1f', - ' ', - '"', - '#', - '<', - '>', - '\\', - '^', - '`', - '{', - '|', - '}', - '\x7f', - '\x80', - '\x81', - '\x82', - '\x83', - '\x84', - '\x85', - '\x86', - '\x87', - '\x88', - '\x89', - '\x8a', - '\x8b', - '\x8c', - '\x8d', - '\x8e', - '\x8f', - '\x90', - '\x91', - '\x92', - '\x93', - '\x94', - '\x95', - '\x96', - '\x97', - '\x98', - '\x99', - '\x9a', - '\x9b', - '\x9c', - '\x9d', - '\x9e', - '\x9f', - '\xa0', - '\xa1', - '\xa2', - '\xa3', - '\xa4', - '\xa5', - '\xa6', - '\xa7', - '\xa8', - '\xa9', - '\xaa', - '\xab', - '\xac', - '\xad', - '\xae', - '\xaf', - '\xb0', - '\xb1', - '\xb2', - '\xb3', - '\xb4', - '\xb5', - '\xb6', - '\xb7', - '\xb8', - '\xb9', - '\xba', - '\xbb', - '\xbc', - '\xbd', - '\xbe', - '\xbf', - '\xc0', - '\xc1', - '\xc2', - '\xc3', - '\xc4', - '\xc5', - '\xc6', - '\xc7', - '\xc8', - '\xc9', - '\xca', - '\xcb', - '\xcc', - '\xcd', - '\xce', - '\xcf', - '\xd0', - '\xd1', - '\xd2', - '\xd3', - '\xd4', - '\xd5', - '\xd6', - '\xd7', - '\xd8', - '\xd9', - '\xda', - '\xdb', - '\xdc', - '\xdd', - '\xde', - '\xdf', - '\xe0', - '\xe1', - '\xe2', - '\xe3', - '\xe4', - '\xe5', - '\xe6', - '\xe7', - '\xe8', - '\xe9', - '\xea', - '\xeb', - '\xec', - '\xed', - '\xee', - '\xef', - '\xf0', - '\xf1', - '\xf2', - '\xf3', - '\xf4', - '\xf5', - '\xf6', - '\xf7', - '\xf8', - '\xf9', - '\xfa', - '\xfb', - '\xfc', - '\xfd', - '\xfe', - '\xff', + 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', '.', 'h', 'l', 'm', + '-', 'd', 'u', '0', '1', '2', 'g', '=', ':', 'b', 'f', '3', 'y', '5', '&', '_', + '4', 'v', '9', '6', '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', + 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'q', 'H', 'G', 'W', + 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 'Q', ';', ')', '(', '~', '[', ']', '$', + '!', '\'', '*', '@', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b', + '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', + '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', '#', '<', '>', '\\', '^', '`', '{', '|', '}', '\x7f', + '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', + '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', + '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', + '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', + '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', + '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', + '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', + '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', }; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 824021c9e379fa0bc3239e1bad5b2acee525ffae..962973bb5cd6fa47c0635ac5a4e5d8fa708dbfa5 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -17,10 +17,8 @@ #include "index_fst_util.h" #include "tutil.h" -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { - if (ctx->offset + len > ctx->limit) { - return -1; - } +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { + if (ctx->offset + len > ctx->limit) { return -1; } if (ctx->type == TFile) { assert(len == tfWrite(ctx->file.fd, buf, len)); @@ -30,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { ctx->offset += len; return len; } -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { int nRead = 0; if (ctx->type == TFile) { nRead = tfRead(ctx->file.fd, buf, len); @@ -41,7 +39,18 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { return nRead; } -static int writeCtxDoFlush(WriterCtx *ctx) { +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { + int nRead = 0; + if (ctx->type == TFile) { + tfLseek(ctx->file.fd, offset, 0); + nRead = tfRead(ctx->file.fd, buf, len); + } else { + // refactor later + assert(0); + } + return nRead; +} +static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { // tfFsync(ctx->fd); // tfFlush(ctx->file.fd); @@ -51,11 +60,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) { return 1; } -WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { - WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); - if (ctx == NULL) { - return NULL; - } +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { + WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); + if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { @@ -67,8 +74,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ctx->file.fd = tfOpenReadWrite(tmpFile); } if (ctx->file.fd < 0) { - goto END; indexError("open file error %d", errno); + goto END; } } else if (ctx->type == TMemory) { ctx->mem.buf = calloc(1, sizeof(char) * capacity); @@ -77,18 +84,17 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ctx->write = writeCtxDoWrite; ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; + ctx->readFrom = writeCtxDoReadFrom; ctx->offset = 0; ctx->limit = capacity; return ctx; END: - if (ctx->type == TMemory) { - free(ctx->mem.buf); - } + if (ctx->type == TMemory) { free(ctx->mem.buf); } free(ctx); } -void writerCtxDestroy(WriterCtx *ctx) { +void writerCtxDestroy(WriterCtx* ctx) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { @@ -97,57 +103,53 @@ void writerCtxDestroy(WriterCtx *ctx) { free(ctx); } -FstCountingWriter *fstCountingWriterCreate(void *wrt) { - FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); - if (cw == NULL) { - return NULL; - } +FstCountingWriter* fstCountingWriterCreate(void* wrt) { + FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); + if (cw == NULL) { return NULL; } cw->wrt = wrt; //(void *)(writerCtxCreate(TFile, readOnly)); return cw; } -void fstCountingWriterDestroy(FstCountingWriter *cw) { +void fstCountingWriterDestroy(FstCountingWriter* cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); // writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { - return 0; - } +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { + if (write == NULL) { return 0; } // update checksum // write data to file/socket or mem - WriterCtx *ctx = write->wrt; + WriterCtx* ctx = write->wrt; int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; return len; } -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { - return 0; - } - WriterCtx *ctx = write->wrt; +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { + if (write == NULL) { return 0; } + WriterCtx* ctx = write->wrt; int nRead = ctx->read(ctx, buf, len); // assert(nRead == len); return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } -int fstCountingWriterFlush(FstCountingWriter *write) { - WriterCtx *ctx = write->wrt; +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { + return 0; +} +int fstCountingWriterFlush(FstCountingWriter* write) { + WriterCtx* ctx = write->wrt; ctx->flush(ctx); // write->wtr->flush return 1; } -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t *buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = calloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; @@ -157,7 +159,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t return; } -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) { uint8_t nBytes = packSize(n); fstCountingWriterPackUintIn(writer, n, nBytes); return nBytes; diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 084f280bc3d3b8d224824005b0bb9b3a9d4ec894..5cdf6adb31a421e38fb22d30fe34599f1489924c 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -14,60 +14,46 @@ */ #include "index_fst_node.h" -FstBuilderNode *fstBuilderNodeDefault() { - FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); +FstBuilderNode* fstBuilderNodeDefault() { + FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } -void fstBuilderNodeDestroy(FstBuilderNode *node) { - if (node == NULL) { - return; - } +void fstBuilderNodeDestroy(FstBuilderNode* node) { + if (node == NULL) { return; } taosArrayDestroy(node->trans); free(node); } -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { - if (n1 == n2) { - return true; - } - if (n1 == NULL || n2 == NULL) { - return false; - } +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { + if (n1 == n2) { return true; } + if (n1 == NULL || n2 == NULL) { return false; } - if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { - return false; - } + if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; } size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; - if (s1 != s2) { - return false; - } + if (s1 != s2) { return false; } for (size_t i = 0; i < s1; i++) { - FstTransition *t1 = taosArrayGet(n1->trans, i); - FstTransition *t2 = taosArrayGet(n2->trans, i); - if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { - return false; - } + FstTransition* t1 = taosArrayGet(n1->trans, i); + FstTransition* t2 = taosArrayGet(n2->trans, i); + if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; } } return true; } -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); - if (node == NULL) { - return NULL; - } +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + if (node == NULL) { return NULL; } // size_t sz = taosArrayGetSize(src->trans); - SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); + SArray* trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *tran = taosArrayGet(src->trans, i); + FstTransition* tran = taosArrayGet(src->trans, i); taosArrayPush(trans, tran); } @@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { return node; } // not destroy src, User's bussiness -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { - if (dst == NULL || src == NULL) { - return; - } +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { + if (dst == NULL || src == NULL) { return; } dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; @@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { size_t sz = taosArrayGetSize(src->trans); dst->trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(src->trans, i); + FstTransition* trn = taosArrayGet(src->trans, i); taosArrayPush(dst->trans, trn); } } diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 7bb2e722307a0b676fd6eb7485a3502508e9bbd9..07afc52ecf12bf515e32afca7fc50a44e0a422b3 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -15,7 +15,7 @@ #include "index_fst_registry.h" -uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { +uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { // TODO(yihaoDeng): refactor later const uint64_t FNV_PRIME = 1099511628211; uint64_t h = 14695981039346656037u; @@ -25,21 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); for (uint32_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(bNode->trans, i); + FstTransition* trn = taosArrayGet(bNode->trans, i); h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; } return h % (registry->tableSize); } -static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { +static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) { size_t sz = taosArrayGetSize(arr); - if (a >= sz || b >= sz) { - return; - } + if (a >= sz || b >= sz) { return; } - FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); - FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); + FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a); + FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b); FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; @@ -51,11 +49,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { return; } -static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { +static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { size_t sz = taosArrayGetSize(arr); - if (start >= sz && end >= sz) { - return; - } + if (start >= sz && end >= sz) { return; } assert(start >= end); @@ -67,14 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { } } -FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry *registry = malloc(sizeof(FstRegistry)); - if (registry == NULL) { - return NULL; - } +FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { + FstRegistry* registry = malloc(sizeof(FstRegistry)); + if (registry == NULL) { return NULL; } uint64_t nCells = tableSize * mruSize; - SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); + SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); if (NULL == tb) { free(registry); return NULL; @@ -91,32 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { return registry; } -void fstRegistryDestroy(FstRegistry *registry) { - if (registry == NULL) { - return; - } +void fstRegistryDestroy(FstRegistry* registry) { + if (registry == NULL) { return; } - SArray *tb = registry->table; + SArray* tb = registry->table; size_t sz = taosArrayGetSize(tb); for (size_t i = 0; i < sz; i++) { - FstRegistryCell *cell = taosArrayGet(tb, i); + FstRegistryCell* cell = taosArrayGet(tb, i); fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); free(registry); } -FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { - if (taosArrayGetSize(registry->table) <= 0) { - return NULL; - } +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { + if (taosArrayGetSize(registry->table) <= 0) { return NULL; } uint64_t bucket = fstRegistryHash(registry, bNode); uint64_t start = registry->mruSize * bucket; uint64_t end = start + registry->mruSize; - FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); + FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); if (end - start == 1) { - FstRegistryCell *cell = taosArrayGet(registry->table, start); + FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; @@ -128,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo entry->cell = cell; // copy or not } } else if (end - start == 2) { - FstRegistryCell *cell1 = taosArrayGet(registry->table, start); + FstRegistryCell* cell1 = taosArrayGet(registry->table, start); if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { entry->state = FOUND; entry->addr = cell1->addr; return entry; } - FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); + FstRegistryCell* cell2 = taosArrayGet(registry->table, start + 1); if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { entry->state = FOUND; entry->addr = cell2->addr; @@ -146,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo fstBuilderNodeCloneFrom(cell2->node, bNode); fstRegistryCellSwap(registry->table, start, start + 1); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); + FstRegistryCell* cCell = taosArrayGet(registry->table, start); entry->state = NOTFOUND; entry->cell = cCell; } else { uint32_t i = start; for (; i < end; i++) { - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, i); if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr; @@ -162,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo } if (i >= end) { uint64_t last = end - 1; - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last); // clone from bNode, refactor later fstBuilderNodeCloneFrom(cell->node, bNode); fstRegistryCellPromote(registry->table, last, start); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); + FstRegistryCell* cCell = taosArrayGet(registry->table, start); entry->state = NOTFOUND; entry->cell = cCell; } } return entry; } -void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); } +void fstRegistryEntryDestroy(FstRegistryEntry* entry) { + free(entry); +} diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 597f7cc61ade3bedd5e2814c166e86e1433c71f6..da1e177a18b3563dd2388e536cc301bb7b8117d0 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -61,9 +61,10 @@ uint8_t packSize(uint64_t n) { } } -uint64_t unpackUint64(uint8_t *ch, uint8_t sz) { +uint64_t unpackUint64(uint8_t* ch, uint8_t sz) { uint64_t n = 0; for (uint8_t i = 0; i < sz; i++) { + // n = n | (ch[i] << (8 * i)); } return n; @@ -75,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { return packSize(nodeAddr - transAddr); } } -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { uint64_t delta = unpackUint64(data, len); // delta_add = u64_to_usize if (delta == EMPTY_ADDRESS) { @@ -88,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { // fst slice func // -FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { - FstString *str = (FstString *)malloc(sizeof(FstString)); +FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { + FstString* str = (FstString*)malloc(sizeof(FstString)); str->ref = 1; str->len = len; str->data = malloc(len * sizeof(uint8_t)); @@ -99,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { return s; } // just shallow copy -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { - FstString *str = s->str; +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { + FstString* str = s->str; str->ref++; // uint8_t *buf = fstSliceData(s, &alen); // start = buf + start - (buf - s->start); @@ -109,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { int32_t tlen = end - start + 1; int32_t slen; - uint8_t *data = fstSliceData(s, &slen); + uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = malloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - FstString *str = malloc(sizeof(FstString)); + FstString* str = malloc(sizeof(FstString)); str->data = buf; str->len = tlen; str->ref = 1; @@ -129,17 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { ans.end = tlen - 1; return ans; } -bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } +bool fstSliceIsEmpty(FstSlice* s) { + return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; +} -uint8_t *fstSliceData(FstSlice *s, int32_t *size) { - FstString *str = s->str; - if (size != NULL) { - *size = s->end - s->start + 1; - } +uint8_t* fstSliceData(FstSlice* s, int32_t* size) { + FstString* str = s->str; + if (size != NULL) { *size = s->end - s->start + 1; } return str->data + s->start; } -void fstSliceDestroy(FstSlice *s) { - FstString *str = s->str; +void fstSliceDestroy(FstSlice* s) { + FstString* str = s->str; str->ref--; if (str->ref <= 0) { free(str->data); @@ -148,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) { } } -int fstSliceCompare(FstSlice *a, FstSlice *b) { +int fstSliceCompare(FstSlice* a, FstSlice* b) { int32_t alen, blen; - uint8_t *aBuf = fstSliceData(a, &alen); - uint8_t *bBuf = fstSliceData(b, &blen); + uint8_t* aBuf = fstSliceData(a, &alen); + uint8_t* bBuf = fstSliceData(b, &blen); uint32_t i, j; for (i = 0, j = 0; i < alen && j < blen; i++, j++) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 19e8ff67506ba0bc963ab910eaffa2f5754f6b60..6ac1a893a6110a387e9cca6d559e859703a60d9c 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -21,107 +21,78 @@ #include "index_fst_counting_writer.h" #include "index_util.h" #include "taosdef.h" +#include "tcompare.h" -static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) { - // TODO simple tfile header later - char buf[TFILE_HADER_PRE_SIZE]; - char * p = buf; - TFileReadHeader *header = &reader->header; - int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); - assert(nread == TFILE_HADER_PRE_SIZE); +#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) - memcpy(&header->suid, p, sizeof(header->suid)); - p += sizeof(header->suid); +typedef struct TFileValue { + char* colVal; // null terminated + SArray* tableId; + int32_t offset; +} TFileValue; - memcpy(&header->version, p, sizeof(header->version)); - p += sizeof(header->version); +static int tfileValueCompare(const void* a, const void* b, const void* param); +static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); - int32_t colLen = 0; - memcpy(&colLen, p, sizeof(colLen)); - assert(colLen < sizeof(header->colName)); - nread = reader->ctx->read(reader->ctx, header->colName, colLen); - assert(nread == colLen); +static int tfileWriteHeader(TFileWriter* writer); +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); +static int tfileWriteData(TFileWriter* write, TFileValue* tval); - nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType)); - return 0; -}; -static int tfileGetFileList(const char *path, SArray *result) { - DIR *dir = opendir(path); - if (NULL == dir) { - return -1; - } +static int tfileReadLoadHeader(TFileReader* reader); +static int tfileReadLoadFst(TFileReader* reader); +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { - size_t len = strlen(entry->d_name); - char * buf = calloc(1, len + 1); - memcpy(buf, entry->d_name, len); - taosArrayPush(result, &buf); - } - closedir(dir); - return 0; -} -static void tfileDestroyFileName(void *elem) { - char *p = *(char **)elem; - free(p); -} -static int tfileCompare(const void *a, const void *b) { - const char *aName = *(char **)a; - const char *bName = *(char **)b; - size_t aLen = strlen(aName); - size_t bLen = strlen(bName); - return strncmp(aName, bName, aLen > bLen ? aLen : bLen); -} -// tfile name suid-colId-version.tindex -static int tfileParseFileName(const char *filename, uint64_t *suid, int *colId, int *version) { - if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { - // read suid & colid & version success - return 0; - } - return -1; -} -static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { - SERIALIZE_MEM_TO_BUF(buf, key, suid); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, colType); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, version); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -} +static int tfileGetFileList(const char* path, SArray* result); +static void tfileDestroyFileName(void* elem); +static int tfileCompare(const void* a, const void* b); +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); -TFileCache *tfileCacheCreate(const char *path) { - TFileCache *tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } +TFileCache* tfileCacheCreate(const char* path) { + TFileCache* tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; - SArray *files = taosArrayInit(4, sizeof(void *)); + SArray* files = taosArrayInit(4, sizeof(void*)); tfileGetFileList(path, files); taosArraySort(files, tfileCompare); + uint64_t suid; + int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { - char * file = taosArrayGetP(files, i); - uint64_t suid; - int colId, version; - if (0 != tfileParseFileName(file, &suid, &colId, &version)) { - goto End; + char* file = taosArrayGetP(files, i); + if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) { + indexInfo("try parse invalid file: %s, skip it", file); continue; } - - WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); if (wc == NULL) { indexError("failed to open index: %s", file); goto End; } - TFileReader *reader = tfileReaderCreate(wc); + TFileReader* reader = tfileReaderCreate(wc); if (0 != tfileReadLoadHeader(reader)) { - TFileReaderDestroy(reader); - indexError("failed to load index header, index Id: %s", file); + tfileReaderDestroy(reader); + indexError("failed to load index header, index file: %s", file); goto End; } + if (0 != tfileReadLoadFst(reader)) { + tfileReaderDestroy(reader); + indexError("failed to load index fst, index file: %s", file); + } + // loader fst and validate it + + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, + .version = header->version, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + + char buf[128] = {0}; + tfileSerialCacheKey(&key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); return tcache; @@ -130,78 +101,329 @@ End: taosArrayDestroyEx(files, tfileDestroyFileName); return NULL; } -void tfileCacheDestroy(TFileCache *tcache) { - if (tcache == NULL) { - return; - } +void tfileCacheDestroy(TFileCache* tcache) { + if (tcache == NULL) { return; } // free table cache - TFileReader **reader = taosHashIterate(tcache->tableCache, NULL); + TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { - TFileReader *p = *reader; - indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, - p->header.colType); - TFileReaderDestroy(p); + TFileReader* p = *reader; + indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); + + tfileReaderDestroy(p); reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); free(tcache); } -TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); - TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); return reader; } -void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } -TFileReader *tfileReaderCreate(WriterCtx *ctx) { - TFileReader *reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } - reader->ctx = ctx; +TFileReader* tfileReaderCreate(WriterCtx* ctx) { + TFileReader* reader = calloc(1, sizeof(TFileReader)); + if (reader == NULL) { return NULL; } + // T_REF_INC(reader); + reader->ctx = ctx; return reader; } -void TFileReaderDestroy(TFileReader *reader) { - if (reader == NULL) { - return; - } +void tfileReaderDestroy(TFileReader* reader) { + if (reader == NULL) { return; } // T_REF_INC(reader); + fstDestroy(reader->fst); writerCtxDestroy(reader->ctx); free(reader); } -TFileWriter *tfileWriterCreate(const char *suid, const char *colName); -void tfileWriterDestroy(TFileWriter *tw); +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { + SIndexTerm* term = query->term; -IndexTFile *indexTFileCreate(const char *path) { - IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); - tfile->cache = tfileCacheCreate(path); + // refactor to callback later + if (query->qType == QUERY_TERM) { + uint64_t offset; + FstSlice key = fstSliceCreate(term->colVal, term->nColVal); + if (fstGet(reader->fst, &key, &offset)) { + return tfileReadLoadTableIds(reader, offset, result); + } else { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found in tindex", term->suid, term->colName, term->colVal); + } + return 0; + } else if (query->qType == QUERY_PREFIX) { + // handle later + // + } else { + // handle later + } + return 0; +} + +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { + // char pathBuf[128] = {0}; + // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); + // TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType}; + // memcpy(header.colName, ); + + // char buf[TFILE_HADER_PRE_SIZE]; + // int len = TFILE_HADER_PRE_SIZE; + // if (len != ctx->write(ctx, buf, len)) { + // indexError("index: %" PRIu64 " failed to write header info", header->suid); + // return NULL; + //} + TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + if (tw == NULL) { + indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); + return NULL; + } + tw->ctx = ctx; + tw->header = *header; + tfileWriteHeader(tw); + return tw; +} + +int tfileWriterPut(TFileWriter* tw, void* data) { + // sort by coltype and write to tindex + __compar_fn_t fn = getComparFunc(tw->header.colType, 0); + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); + + int32_t bufLimit = 4096, offset = 0; + char* buf = calloc(1, sizeof(bufLimit)); + char* p = buf; + int32_t sz = taosArrayGetSize((SArray*)data); + int32_t fstOffset = tw->offset; + + // ugly code, refactor later + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); + } + // check result or not + tfileWriteFstOffset(tw, fstOffset); + + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + // check buf has enough space or not + int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); + if (offset + ttsz > bufLimit) { + // batch write + tw->ctx->write(tw->ctx, buf, offset); + offset = 0; + memset(buf, 0, bufLimit); + p = buf; + } + + tfileSerialTableIdsToBuf(p, v->tableId); + offset += ttsz; + p = buf + offset; + // set up value offset + v->offset = tw->offset; + tw->offset += ttsz; + } + if (offset != 0) { + // write reversed data in buf to tindex + tw->ctx->write(tw->ctx, buf, offset); + } + + // write fst + for (size_t i = 0; i < sz; i++) { + // TODO, fst batch write later + TFileValue* v = taosArrayGetP((SArray*)data, i); + if (tfileWriteData(tw, v) == 0) { + // + // + } + } + tfree(buf); + return 0; +} +void tfileWriterDestroy(TFileWriter* tw) { + if (tw == NULL) { return; } + + writerCtxDestroy(tw->ctx); + free(tw); +} + +IndexTFile* indexTFileCreate(const char* path) { + IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + if (tfile == NULL) { return NULL; } + + tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } +void IndexTFileDestroy(IndexTFile* tfile) { + free(tfile); +} + +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { + if (tfile == NULL) { return -1; } + IndexTFile* pTfile = (IndexTFile*)tfile; + + SIndexTerm* term = query->term; + TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; + + TFileReader* reader = tfileCacheGet(pTfile->cache, &key); + return tfileReaderSearch(reader, query, result); +} +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { + TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1}; + + return 0; +} + +static int tfileValueCompare(const void* a, const void* b, const void* param) { + __compar_fn_t fn = *(__compar_fn_t*)param; + + TFileValue* av = (TFileValue*)a; + TFileValue* bv = (TFileValue*)b; + + return fn(av->colVal, bv->colVal); +} +static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { + int sz = taosArrayGetSize(ids); + SERIALIZE_VAR_TO_BUF(buf, sz, int32_t); + for (size_t i = 0; i < sz; i++) { + uint64_t* v = taosArrayGet(ids, i); + SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t); + } +} -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { - IndexTFile *pTfile = (IndexTFile *)tfile; +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { + int32_t fstOffset = offset + sizeof(tw->header.fstOffset); + tw->header.fstOffset = fstOffset; + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + return 0; +} +static int tfileWriteHeader(TFileWriter* writer) { + char buf[TFILE_HEADER_NO_FST] = {0}; + char* p = buf; - SIndexTerm * term = query->term; - TFileCacheKey key = { - .suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; - TFileReader *reader = tfileCacheGet(pTfile->cache, &key); + TFileHeader* header = &writer->header; + memcpy(buf, (char*)header, sizeof(buf)); + + int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); + if (sizeof(buf) != nwrite) { return -1; } + writer->offset = nwrite; + return 0; +} +static int tfileWriteData(TFileWriter* write, TFileValue* tval) { + TFileHeader* header = &write->header; + uint8_t colType = header->colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); + if (fstBuilderInsert(write->fb, key, tval->offset)) { + fstSliceDestroy(&key); + return 0; + } + fstSliceDestroy(&key); + return -1; + } else { + // handle other type later + } return 0; } -int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { - TFileWriterOpt wOpt = { - .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1}; +static int tfileReadLoadHeader(TFileReader* reader) { + // TODO simple tfile header later + char buf[TFILE_HEADER_SIZE] = {0}; + char* p = buf; + int64_t nread = reader->ctx->read(reader->ctx, buf, sizeof(buf)); + assert(nread == sizeof(buf)); + memcpy(&reader->header, buf, sizeof(buf)); return 0; } +static int tfileReadLoadFst(TFileReader* reader) { + // current load fst into memory, refactor it later + static int FST_MAX_SIZE = 16 * 1024; + + char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); + if (buf == NULL) { return -1; } + + WriterCtx* ctx = reader->ctx; + int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); + // we assuse fst size less than FST_MAX_SIZE + assert(nread > 0 && nread < FST_MAX_SIZE); + + FstSlice st = fstSliceCreate((uint8_t*)buf, nread); + reader->fst = fstCreate(&st); + free(buf); + fstSliceDestroy(&st); + + return reader->fst == NULL ? 0 : -1; +} +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { + int32_t nid; + WriterCtx* ctx = reader->ctx; + int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); + assert(sizeof(nid) == nread); + + char* buf = calloc(1, sizeof(uint64_t) * nid); + if (buf == NULL) { return -1; } + + nread = ctx->read(ctx, buf, sizeof(uint64_t) * nid); + uint64_t* ids = (uint64_t*)buf; + for (int32_t i = 0; i < nid; i++) { + taosArrayPush(result, ids + i); + } + free(buf); + return 0; +} + +static int tfileGetFileList(const char* path, SArray* result) { + DIR* dir = opendir(path); + if (NULL == dir) { return -1; } + + struct dirent* entry; + while ((entry = readdir(dir)) != NULL) { + size_t len = strlen(entry->d_name); + char* buf = calloc(1, len + 1); + memcpy(buf, entry->d_name, len); + taosArrayPush(result, &buf); + } + closedir(dir); + return 0; +} +static void tfileDestroyFileName(void* elem) { + char* p = *(char**)elem; + free(p); +} +static int tfileCompare(const void* a, const void* b) { + const char* aName = *(char**)a; + const char* bName = *(char**)b; + + size_t aLen = strlen(aName); + size_t bLen = strlen(bName); + + return strncmp(aName, bName, aLen > bLen ? aLen : bLen); +} +// tfile name suid-colId-version.tindex +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) { + if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { + // read suid & colid & version success + return 0; + } + return -1; +} +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, version); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +} diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index 7d2e6e3aec336c2e1158501c7197ebdf790fa79b..fbd92b89f85aa789f3c50360da372ec38b2bd812 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -170,10 +170,11 @@ int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +void destroyBlockArrayList(SArray* pDataBlockList); int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 3138f0c1d8c8fe0bdca6b835e1c0afe9bb0fe407..d37ac74d218ea4df8f3dcbdf22693f5ed2a08c68 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -269,19 +269,17 @@ void destroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock); } -void* destroyBlockArrayList(SArray* pDataBlockList) { +void destroyBlockArrayList(SArray* pDataBlockList) { if (pDataBlockList == NULL) { - return NULL; + return; } size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - void* d = taosArrayGetP(pDataBlockList, i); - destroyDataBlock(d); + destroyDataBlock(taosArrayGetP(pDataBlockList, i)); } taosArrayDestroy(pDataBlockList); - return NULL; } // data block is disordered, sort it in ascending order @@ -298,6 +296,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { int32_t i = 0; int32_t j = 1; + // delete rows with timestamp conflicts while (j < pBlocks->numOfRows) { TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); @@ -430,7 +429,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->dataLen = 0; - int32_t numOfRows = htons(pBlock->numOfRows); + int32_t numOfRows = pBlock->numOfRows; if (isRawPayload) { for (int32_t i = 0; i < numOfRows; ++i) { @@ -467,18 +466,10 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } } - int32_t len = pBlock->dataLen + pBlock->schemaLen; - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - - return len; + return pBlock->dataLen + pBlock->schemaLen; } -static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { - // todo -} - -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -537,24 +528,13 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; - - pBlocks->numOfRows = 0; } p = taosHashIterate(pHashObj, p); @@ -565,12 +545,10 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t pOneTableBlock = *p; } - extractTableNameList(pHashObj, freeBlockMap); - // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); tfree(blkKeyInfo.pKeyTuple); - + *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 877ecb2c85457c9aff9de9454306c9fffc559bf8..2481896f092b650586c77ad8b6ebeede9127b4f7 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -31,33 +31,16 @@ pSql += index; \ } while (0) -#define CHECK_CODE(expr) \ +#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - return terrno; \ - } \ - } while (0) - -#define CHECK_CODE_1(expr, destroy) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy; \ - terrno = code; \ - return terrno; \ - } \ + sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE_2(expr, destroy1, destroy2) \ +#define CHECK_CODE(expr) \ do { \ int32_t code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy1; \ - (void)destroy2; \ - terrno = code; \ - return terrno; \ + return code; \ } \ } while (0) @@ -70,9 +53,12 @@ typedef struct SInsertParseContext { SParseContext* pComCxt; const char* pSql; SMsgBuf msg; - struct SCatalog* pCatalog; STableMeta* pTableMeta; - SHashObj* pTableBlockHashObj; // data block for each table. need release + SParsedDataColInfo tags; + SKVRowBuilder tagsBuilder; + SHashObj* pTableBlockHashObj; + SArray* pTableDataBlocks; + SArray* pVgDataBlocks; int32_t totalNum; SInsertStmtInfo* pOutput; } SInsertParseContext; @@ -151,14 +137,6 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b return ret; } -static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { - *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (NULL == *pInsertInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; -} - static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -194,7 +172,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0}; CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); return TSDB_CODE_SUCCESS; } @@ -209,6 +187,51 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS return -1; } +static void fillMsgHeader(SVgDataBlocks* dst) { + SMsgDesc* desc = (SMsgDesc*)dst->pData; + desc->numOfVnodes = htonl(1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + submit->header.vgId = htonl(dst->vgId); + submit->header.contLen = htonl(dst->size - sizeof(SMsgDesc)); + submit->length = submit->header.contLen; + submit->numOfBlocks = htonl(dst->numOfTables); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + int32_t numOfBlocks = dst->numOfTables; + while (numOfBlocks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->tid = htonl(blk->tid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } +} + +static int32_t buildOutput(SInsertParseContext* pCxt) { + size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks); + pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pCxt->pOutput->pDataBlocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for (size_t i = 0; i < numOfVg; ++i) { + STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); + SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dst->vgId = src->vgId; + dst->numOfTables = src->numOfTables; + dst->size = src->size; + SWAP(dst->pData, src->pData, char*); + fillMsgHeader(dst); + taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { @@ -219,16 +242,14 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) if (k == INT64_MIN) { if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; } else { if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; // client time/server time can not be mixed - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_CLI_TS; } if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { @@ -608,26 +629,24 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) { - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - destroyBoundColumnInfo(pSpd); +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision) { + if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.builder = &kvRowBuilder}; + SKvParam param = {.builder = &pCxt->tagsBuilder}; SToken sToken; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - for (int i = 0; i < pSpd->numOfBound; ++i) { + for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; param.schema = pSchema; - CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); + CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m)); } - destroyBoundColumnInfo(pSpd); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); + destroyBoundColumnInfo(&pCxt->tags); + SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); + tdDestroyKVRowBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } @@ -650,13 +669,12 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) } SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); - SParsedDataColInfo spd = {0}; - setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); if (TK_LP == sToken.type) { - CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd)); + CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -668,7 +686,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); return TSDB_CODE_SUCCESS; } @@ -732,10 +750,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" SToken sToken; while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); if (TK_LP != sToken.type) { break; } + pCxt->pSql += index; if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; @@ -815,7 +835,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -842,9 +862,18 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } - return TSDB_CODE_SUCCESS; + return buildOutput(pCxt); +} + +static void destroyInsertParseContext(SInsertParseContext* pCxt) { + tfree(pCxt->pTableMeta); + destroyBoundColumnInfo(&pCxt->tags); + tdDestroyKVRowBuilder(&pCxt->tagsBuilder); + taosHashCleanup(pCxt->pTableBlockHashObj); + destroyBlockArrayList(pCxt->pTableDataBlocks); + destroyBlockArrayList(pCxt->pVgDataBlocks); } // INSERT INTO @@ -854,26 +883,30 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { - CHECK_CODE(createInsertStmtInfo(pInfo)); - SInsertParseContext context = { .pComCxt = pContext, .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pCatalog = NULL, .pTableMeta = NULL, .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, - .pOutput = *pInfo + .pOutput = calloc(1, sizeof(SInsertStmtInfo)) }; - if (NULL == context.pTableBlockHashObj) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (NULL == context.pTableBlockHashObj || NULL == context.pOutput) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } - CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog)); - CHECK_CODE(skipInsertInto(&context)); - CHECK_CODE(parseInsertBody(&context)); + *pInfo = context.pOutput; + context.pOutput->schemaAttache = pContext->schemaAttached; + context.pOutput->payloadType = PAYLOAD_TYPE_KV; - return TSDB_CODE_SUCCESS; + int32_t code = skipInsertInto(&context); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(&context); + } + destroyInsertParseContext(&context); + terrno = code; + return (TSDB_CODE_SUCCESS == code ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); } diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index feae00827310a93e9117361cc5d7c682e3582a76..e722b873b6882f625fbc81e9347dcd90437a3337 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,5 +17,3 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common parser catalog transport gtest function planner qcom ) - -TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..433d189a453053c84bb11672e4f9586d30c02566 --- /dev/null +++ b/source/libs/parser/test/insertParserTest.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "insertParser.h" +// #include "mockCatalog.h" + +using namespace std; +using namespace testing; + +namespace { + string toString(int32_t code) { + return tstrerror(code); + } +} + +// syntax: +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +class InsertTest : public Test { +protected: + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.pAcctId = acctId_.c_str(); + cxt_.pDbname = db_.c_str(); + strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + + } + + int32_t run() { + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; + } + return code_; + } + + SInsertStmtInfo* reslut() { + return res_; + } + + void dumpReslut() { + size_t num = taosArrayGetSize(res_->pDataBlocks); + cout << "schemaAttache:" << (int32_t)res_->schemaAttache << ", payloadType:" << (int32_t)res_->payloadType << ", insertType:" << res_->insertType << ", numOfVgs:" << num << endl; + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + cout << "vgId:" << vg->vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + cout << "numOfVnodes:" << ntohl(desc->numOfVnodes) << endl; + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + cout << "length:" << ntohl(submit->length) << ", numOfBlocks:" << ntohl(submit->numOfBlocks) << endl; + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + cout << "Block:" << i << endl; + cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << ntohl(blk->tid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) + << ", dataLen:" << ntohl(blk->dataLen) << ", schemaLen:" << ntohl(blk->schemaLen) << ", numOfRows:" << ntohs(blk->numOfRows) << endl; + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + + void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { + ASSERT_EQ(res_->schemaAttache, 0); + ASSERT_EQ(res_->payloadType, PAYLOAD_TYPE_KV); + ASSERT_EQ(res_->insertType, TSDB_QUERY_TYPE_INSERT); + size_t num = taosArrayGetSize(res_->pDataBlocks); + ASSERT_GE(num, 0); + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + ASSERT_EQ(vg->numOfTables, numOfTables); + ASSERT_GE(vg->size, 0); + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + ASSERT_EQ(ntohl(desc->numOfVnodes), 1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + ASSERT_GE(ntohl(submit->length), 0); + ASSERT_GE(ntohl(submit->numOfBlocks), 0); + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + ASSERT_EQ(ntohs(blk->numOfRows), (0 == i ? numOfRows1 : (numOfRows2 > 0 ? numOfRows2 : numOfRows1))); + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; + } + + string acctId_; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; +}; + +// INSERT INTO tb_name VALUES (field1_value, ...) +TEST_F(InsertTest, singleTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 1); +} + +// INSERT INTO tb_name VALUES (field1_value, ...)(field1_value, ...) +TEST_F(InsertTest, singleTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\") st1s2 values (now, 10, \"131028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 1); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")" + " st1s2 values (now, 10, \"131028\")(now+1s, 20, \"132028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 3, 2); +} + +TEST_F(InsertTest, toleranceTest) { + setDatabase("root", "test"); + + bind("insert into"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); + bind("insert into t"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp deleted file mode 100644 index 85db46d7bf1cfa29576926a0dafc7ef52310a2a8..0000000000000000000000000000000000000000 --- a/source/libs/parser/test/insertTest.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -#include "insertParser.h" -#include "mockCatalog.h" - -using namespace std; -using namespace testing; - -namespace { - string toString(int32_t code) { - return tstrerror(code); - } -} - -extern "C" { - -#include - -void *__real_malloc(size_t); - -void *__wrap_malloc(size_t c) { - // printf("My MALLOC called: %d\n", c); - // void *array[32]; - // int size = backtrace(array, 32); - // char **symbols = backtrace_symbols(array, size); - // for (int i = 0; i < size; ++i) { - // cout << symbols[i] << endl; - // } - // free(symbols); - - return __real_malloc(c); -} - -} - -// syntax: -// INSERT INTO -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -class InsertTest : public Test { -protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.pAcctId = acctId_.c_str(); - cxt_.pDbname = db_.c_str(); - strcpy(sqlBuf_, sql); - cxt_.sqlLen = strlen(sql); - sqlBuf_[cxt_.sqlLen] = '\0'; - cxt_.pSql = sqlBuf_; - - } - - int32_t run() { - code_ = parseInsertSql(&cxt_, &res_); - if (code_ != TSDB_CODE_SUCCESS) { - cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; - } - return code_; - } - - SInsertStmtInfo* reslut() { - return res_; - } - -private: - static const int max_err_len = 1024; - static const int max_sql_len = 1024 * 1024; - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - code_ = TSDB_CODE_SUCCESS; - res_ = nullptr; - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - char sqlBuf_[max_sql_len]; - SParseContext cxt_; - int32_t code_; - SInsertStmtInfo* res_; -}; - -// INSERT INTO tb_name VALUES (field1_value, ...) -TEST_F(InsertTest, simpleTest) { - setDatabase("root", "test"); - - bind("insert into t1 values (now, 1, \"wxy\")"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); - SInsertStmtInfo* res = reslut(); - // todo check - ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); - // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); -} - -TEST_F(InsertTest, toleranceTest) { - setDatabase("root", "test"); - - bind("insert into"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); - bind("insert into t"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); -} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index dc341f6af7eb65a5197dedac76437668984d1d83..fdcb6e0433b89a84f05646690d506387765a9d9d 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -25,15 +25,15 @@ namespace { void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); } void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); mcs->createSubTable("root.test", "st1", "st1s1", 1); mcs->createSubTable("root.test", "st1", "st1s2", 2); diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index 35eb1ddcd9ddfdd96c749db35b1271bcf6c531ff..f8e80a48aa5dacc7dc6d90d1a9db4623bfc73278 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -22,8 +22,4 @@ void initMetaDataEnv(); void generateMetaData(); void destroyMetaDataEnv(); -// mock -// int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); -// int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); - #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index e7271c3cb2a2901f339500369030d3cb95ccf9e4..ad366e8f744ccb81bab33fc8f02cf426ee1eff4b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,6 +33,7 @@ public: col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); + rowsize_ += bytes; return *this; } @@ -147,11 +148,11 @@ public: for (const auto& db : meta_) { std::cout << "Databse:" << db.first << std::endl; - std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << IH("RowSize") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { const auto& schema = table.second->schema; - std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << IF(schema->tableInfo.rowSize) << std::endl; } std::cout << std::endl; } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 1d29e48e308e1b60ee60e35ea6e6c3dcf489e643..0696a5b24e82b771d897257d1de008853b93dc08 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -221,7 +221,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { - SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); + SPhyNode* child = createPhyNode(pCxt, taosArrayGetP(pPlanNode->pChildren, i)); child->pParent = node; taosArrayPush(node->pChildren, &child); } diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4eafa50bdc200615391cfc2f364fc9c9c20430bd --- /dev/null +++ b/source/libs/qworker/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QWORKER_SRC) +add_library(qworker ${QWORKER_SRC}) +target_include_directories( + qworker + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + qworker + PRIVATE os util transport planner qcom +) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h new file mode 100644 index 0000000000000000000000000000000000000000..1adc09def41234a26d3557916eb40917e6e1b57e --- /dev/null +++ b/source/libs/qworker/inc/qworkerInt.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_INT_H_ +#define _TD_QWORKER_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 +#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000 +#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 + +enum { + QW_READY_NOT_RECEIVED = 0, + QW_READY_RECEIVED, + QW_READY_RESPONSED, +}; + +typedef struct SQWorkerTaskStatus { + int8_t status; + int8_t ready; +} SQWorkerTaskStatus; + +typedef struct SQWorkerResCache { + void *data; +} SQWorkerResCache; + +typedef struct SQWorkerSchTaskStatus { + int32_t lastAccessTs; // timestamp in second + SHashObj *taskStatus; // key:queryId+taskId, value: SQWorkerTaskStatus +} SQWorkerSchTaskStatus; + +// Qnode/Vnode level task management +typedef struct SQWorkerMgmt { + SQWorkerCfg cfg; + SHashObj *scheduleHash; //key: schedulerId, value: SQWorkerSchTaskStatus + SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache +} SQWorkerMgmt; + +#define QW_TASK_DONE(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == status == JOB_TASK_STATUS_CANCELLED) +#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0) +#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0) + +#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_INT_H_*/ diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c new file mode 100644 index 0000000000000000000000000000000000000000..82bfd75b6a1d2ffb22a4c11b6f286b185d6a643a --- /dev/null +++ b/source/libs/qworker/src/qworker.c @@ -0,0 +1,298 @@ +#include "taosmsg.h" +#include "query.h" +#include "qworker.h" +#include "qworkerInt.h" +#include "planner.h" + +int32_t qwAddTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t taskStatus) { + SQWorkerTaskStatus tStatus = {0}; + tStatus.status = taskStatus; + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + SQWorkerSchTaskStatus newSchStatus = {0}; + newSchStatus.taskStatus = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == newSchStatus.taskStatus) { + qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (0 != taosHashPut(newSchStatus.taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + newSchStatus.lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId), &newSchStatus, sizeof(newSchStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"] to scheduleHash failed", schedulerId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(schStatus->taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SQWorkerTaskStatus **taskStatus) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerTaskStatus *tStatus = taosHashGet(schStatus->taskStatus, id, sizeof(id)); + if (NULL == tStatus) { + qError("no task status for schedulerId[%"PRIx64"] queryId[%"PRIx64"] taskId[%"PRIx64"]", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *taskStatus = tStatus; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void *data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache resCache = {0}; + resCache.data = data; + + if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwGetTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void **data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache *resCache = taosHashGet(mgmt->resHash, id, sizeof(id)); + if (NULL == resCache) { + qError("no task res for queryId[%"PRIx64"] taskId[%"PRIx64"]", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *data = resCache->data; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t schedulerId) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SSchedulerStatusRsp **rsp) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + int32_t i = 0; + int32_t taskNum = taosHashGetSize(schStatus->taskStatus); + int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; + *rsp = calloc(1, size); + if (NULL == *rsp) { + qError("calloc %d failed", size); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *key = NULL; + size_t keyLen = 0; + void *pIter = taosHashIterate(schStatus->taskStatus, NULL); + while (pIter) { + SQWorkerTaskStatus *taskStatus = (SQWorkerTaskStatus *)pIter; + taosHashGetKey(pIter, &key, &keyLen); + + QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); + (*rsp)->status[i].status = taskStatus->status; + + pIter = taosHashIterate(schStatus->taskStatus, pIter); + } + + (*rsp)->num = taskNum; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwBuildRspMsg(void *data, int32_t msgType); + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + if (NULL == mgmt) { + qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (cfg) { + mgmt->cfg = *cfg; + } else { + mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER; + mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + } + + mgmt->scheduleHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->scheduleHash) { + tfree(mgmt); + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum); + } + + mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->resHash) { + taosHashCleanup(mgmt->scheduleHash); + mgmt->scheduleHash = NULL; + tfree(mgmt); + + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum); + } + + *qWorkerMgmt = mgmt; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSubplan *plan = NULL; + SQWorkerTaskStatus *tStatus = NULL; + + int32_t code = qStringToSubplan(msg->msg, &plan); + if (TSDB_CODE_SUCCESS != code) { + qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->schedulerId, msg->queryId, msg->taskId, code); + return code; + } + + //TODO call executer to init subquery + + QW_ERR_JRET(qwAddTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING)); + + QW_ERR_JRET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_QUERY_RSP)); + + //TODO call executer to execute subquery + code = 0; + void *data = NULL; + //TODO call executer to execute subquery + + QW_ERR_JRET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + tStatus->status = (code) ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_SUCCEED; + + QW_ERR_JRET(qwAddTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, data)); + +_return: + + if (tStatus && QW_TASK_DONE(tStatus->status) && QW_READY_RECEIVED == tStatus->ready) { + QW_ERR_RET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_RES_READY_RSP)); + } + + qDestroySubplan(plan); + + return code; +} + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp){ + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SQWorkerTaskStatus *tStatus = NULL; + + QW_ERR_RET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + if (QW_TASK_DONE(tStatus->status)) { + QW_ERR_RET(qwBuildRspMsg(tStatus, TSDB_MSG_TYPE_RES_READY_RSP)); + } else { + tStatus->ready = QW_READY_RECEIVED; + + return TSDB_CODE_SUCCESS; + } + + tStatus->ready = QW_READY_RESPONSED; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSchedulerStatusRsp *sStatus = NULL; + + QW_ERR_RET(qwGetSchTasksStatus(qWorkerMgmt, msg->schedulerId, &sStatus)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->schedulerId)); + + void *data = NULL; + + QW_ERR_RET(qwGetTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, &data)); + + QW_ERR_RET(qwBuildRspMsg(data, TSDB_MSG_TYPE_FETCH_RSP)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt) { + if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { + return; + } + + SQWorkerMgmt *mgmt = *qWorkerMgmt; + + //TODO STOP ALL QUERY + + //TODO FREE ALL + + tfree(*qWorkerMgmt); +} + + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 73e7c4d24e1c2693d174dcb7acad4112939778c3..3fab91edacd77096fbacceee5df3f762452ad16d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -31,17 +31,9 @@ extern "C" { #define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA -enum { - SCH_STATUS_NOT_START = 1, - SCH_STATUS_EXECUTING, - SCH_STATUS_SUCCEED, - SCH_STATUS_FAILED, - SCH_STATUS_CANCELLING, - SCH_STATUS_CANCELLED -}; - typedef struct SSchedulerMgmt { - uint64_t taskId; + uint64_t taskId; + uint64_t schedulerId; SSchedulerCfg cfg; SHashObj *Jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6014ff9ab6282248f52bfe1aba0535e5b4d4cb71..99a9b06fe441832d42fbc5bc86ad24bcc3ebc263 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -161,7 +161,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { SArray *levelPlans = NULL; int32_t levelPlanNum = 0; - level.status = SCH_STATUS_NOT_START; + level.status = JOB_TASK_STATUS_NOT_START; for (int32_t i = 0; i < levelNum; ++i) { level.level = i; @@ -191,7 +191,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); task.plan = plan; - task.status = SCH_STATUS_NOT_START; + task.status = JOB_TASK_STATUS_NOT_START; void *p = taosArrayPush(level.subTasks, &task); if (NULL == p) { @@ -304,13 +304,15 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerQueryMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; - pMsg->contentLen = len; + + pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + pMsg->contentLen = htonl(len); memcpy(pMsg->msg, task->msg, len); break; } - case TSDB_MSG_TYPE_RSP_READY: { + case TSDB_MSG_TYPE_RES_READY: { msgSize = sizeof(SSchedulerReadyMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -319,8 +321,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerReadyMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } case TSDB_MSG_TYPE_FETCH: { @@ -332,8 +334,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerFetchMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } default: @@ -376,7 +378,7 @@ _return: int32_t schProcessOnJobSuccess(SQueryJob *job) { - job->status = SCH_STATUS_SUCCEED; + job->status = JOB_TASK_STATUS_SUCCEED; if (job->userFetch) { SCH_ERR_RET(schFetchFromRemote(job)); @@ -386,7 +388,7 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) { } int32_t schProcessOnJobFailure(SQueryJob *job) { - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); @@ -413,7 +415,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { return TSDB_CODE_SUCCESS; } - task->status = SCH_STATUS_SUCCEED; + task->status = JOB_TASK_STATUS_SUCCEED; int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); if (parentNum == 0) { @@ -459,7 +461,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod if (!needRetry) { SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; SCH_ERR_RET(schProcessOnJobFailure(job)); return TSDB_CODE_SUCCESS; @@ -478,13 +480,13 @@ int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32 if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { - code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RSP_READY); + code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RES_READY); if (code) { goto _task_error; } } break; - case TSDB_MSG_TYPE_RSP_READY: + case TSDB_MSG_TYPE_RES_READY: if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { @@ -534,7 +536,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(schPushTaskToExecList(job, task)); - task->status = SCH_STATUS_EXECUTING; + task->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } @@ -546,22 +548,26 @@ int32_t schLaunchJob(SQueryJob *job) { SCH_ERR_RET(schLaunchTask(job, task)); } - job->status = SCH_STATUS_EXECUTING; + job->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } int32_t schedulerInit(SSchedulerCfg *cfg) { - schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == schMgmt.Jobs) { - SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); - } - if (cfg) { schMgmt.cfg = *cfg; + } else { + schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } + schMgmt.Jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == schMgmt.Jobs) { + SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum); + } + + schMgmt.schedulerId = 1; //TODO GENERATE A UUID + return TSDB_CODE_SUCCESS; } @@ -605,7 +611,7 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - job->status = SCH_STATUS_NOT_START; + job->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(job)); @@ -634,7 +640,7 @@ int32_t scheduleFetchRows(void *pJob, void **data) { return TSDB_CODE_QRY_APP_ERROR; } - if (job->status == SCH_STATUS_SUCCEED) { + if (job->status == JOB_TASK_STATUS_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(job)); } @@ -668,7 +674,7 @@ void scheduleFreeJob(void *pJob) { return; } - if (job->status == SCH_STATUS_EXECUTING) { + if (job->status == JOB_TASK_STATUS_EXECUTING) { scheduleCancelJob(pJob); } } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index bfc6b47fa6abd80245b021954ee908156e9eebeb..c4f6f78106e1ee6703e0429e705f62ab83db1881 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -776,9 +776,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } -FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { +FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { + if (NULL == data || NULL == key) { + return -1; + } + SHashNode * node = GET_HASH_PNODE(data); - return GET_HASH_NODE_KEY(node); + *key = GET_HASH_NODE_KEY(node); + *keyLen = node->keyLen; + + return 0; } FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) {