Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
“3be59e4e24ef1f478685da85d1e76b17485295c2”上不存在“tests/pytest/cluster/basicTest.py”
未验证 提交 1e32e1f8 编写于 作者: S Shengliang Guan 提交者： GitHub
浏览文件
操作

Merge pull request #9264 from taosdata/feature/index_cache 

update index TFile write
上级 a630d82c ebcb9be3
隐藏空白更改
内联 并排
Showing with 1056 addition and 1215 deletion
include/libs/index/index.h
浏览文件 @ 1e32e1f8
...@@ -44,41 +44,46 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = ...@@ -44,41 +44,46 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX =
* @param: oper * @param: oper
* *
*/ */
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
void indexClose(SIndex *index); void indexClose(SIndex* index);
int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
int indexDelete(SIndex *index, SIndexMultiTermQuery *query); int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
int indexRebuild(SIndex *index, SIndexOpts *opt); int indexRebuild(SIndex* index, SIndexOpts* opt);
/* /*
* @param * @param
* @param * @param
*/ */
SIndexMultiTerm *indexMultiTermCreate(); SIndexMultiTerm* indexMultiTermCreate();
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term);
void indexMultiTermDestroy(SIndexMultiTerm *terms); void indexMultiTermDestroy(SIndexMultiTerm* terms);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
SIndexOpts *indexOptsCreate(); SIndexOpts* indexOptsCreate();
void indexOptsDestroy(SIndexOpts *opts); void indexOptsDestroy(SIndexOpts* opts);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName, SIndexTerm* indexTermCreate(int64_t suid,
int32_t nColName, const char *colVal, int32_t nColVal); SIndexOperOnColumn operType,
void indexTermDestroy(SIndexTerm *p); uint8_t colType,
const char* colName,
int32_t nColName,
const char* colVal,
int32_t nColVal);
void indexTermDestroy(SIndexTerm* p);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/indexInt.h
浏览文件 @ 1e32e1f8
...@@ -42,11 +42,11 @@ typedef struct SIndexStat { ...@@ -42,11 +42,11 @@ typedef struct SIndexStat {
struct SIndex { struct SIndex {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t *index; index_t* index;
#endif #endif
void * cache; void* cache;
void * tindex; void* tindex;
SHashObj *colObj; // < field name, field id> SHashObj* colObj; // < field name, field id>
int64_t suid; // current super table id, -1 is normal table int64_t suid; // current super table id, -1 is normal table
int colId; // field id allocated to cache int colId; // field id allocated to cache
...@@ -58,7 +58,7 @@ struct SIndex { ...@@ -58,7 +58,7 @@ struct SIndex {
struct SIndexOpts { struct SIndexOpts {
#ifdef USE_LUCENE #ifdef USE_LUCENE
void *opts; void* opts;
#endif #endif
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
...@@ -69,7 +69,7 @@ struct SIndexOpts { ...@@ -69,7 +69,7 @@ struct SIndexOpts {
struct SIndexMultiTermQuery { struct SIndexMultiTermQuery {
EIndexOperatorType opera; EIndexOperatorType opera;
SArray * query; SArray* query;
}; };
// field and key; // field and key;
...@@ -77,14 +77,14 @@ typedef struct SIndexTerm { ...@@ -77,14 +77,14 @@ typedef struct SIndexTerm {
int64_t suid; int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json uint8_t colType; // term data type, str/interger/json
char * colName; char* colName;
int32_t nColName; int32_t nColName;
char * colVal; char* colVal;
int32_t nColVal; int32_t nColVal;
} SIndexTerm; } SIndexTerm;
typedef struct SIndexTermQuery { typedef struct SIndexTermQuery {
SIndexTerm * term; SIndexTerm* term;
EIndexQueryType qType; EIndexQueryType qType;
} SIndexTermQuery; } SIndexTermQuery;
......
source/libs/index/inc/index_cache.h
浏览文件 @ 1e32e1f8
...@@ -34,19 +34,18 @@ extern "C" { ...@@ -34,19 +34,18 @@ extern "C" {
typedef struct IndexCache { typedef struct IndexCache {
T_REF_DECLARE() T_REF_DECLARE()
SSkipList *skiplist; SSkipList* skiplist;
} IndexCache; } IndexCache;
// //
IndexCache *indexCacheCreate(); IndexCache* indexCacheCreate();
void indexCacheDestroy(void *cache); void indexCacheDestroy(void* cache);
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid);
// int indexCacheGet(void *cache, uint64_t *rst); // int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch( int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s);
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/index_fst.h
浏览文件 @ 1e32e1f8
...@@ -40,10 +40,10 @@ typedef struct FstBoundWithData { ...@@ -40,10 +40,10 @@ typedef struct FstBoundWithData {
} FstBoundWithData; } FstBoundWithData;
typedef struct FstStreamBuilder { typedef struct FstStreamBuilder {
Fst * fst; Fst* fst;
AutomationCtx * aut; AutomationCtx* aut;
FstBoundWithData *min; FstBoundWithData* min;
FstBoundWithData *max; FstBoundWithData* max;
} FstStreamBuilder, FstStreamWithStateBuilder; } FstStreamBuilder, FstStreamWithStateBuilder;
typedef struct FstRange { typedef struct FstRange {
...@@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType; ...@@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data); FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data);
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice);
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); bool fstBoundWithDataIsEmpty(FstBoundWithData* bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); bool fstBoundWithDataIsIncluded(FstBoundWithData* bound);
typedef struct FstOutput { typedef struct FstOutput {
bool null; bool null;
...@@ -71,46 +71,46 @@ typedef struct FstOutput { ...@@ -71,46 +71,46 @@ typedef struct FstOutput {
* TODO: simple function name * TODO: simple function name
*/ */
typedef struct FstUnFinishedNodes { typedef struct FstUnFinishedNodes {
SArray *stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes; SArray* stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
} FstUnFinishedNodes; } FstUnFinishedNodes;
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) #define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes *fstUnFinishedNodesCreate(); FstUnFinishedNodes* fstUnFinishedNodesCreate();
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); void fstUnFinishedNodesDestroy(FstUnFinishedNodes* node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs);
FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes);
FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr);
FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes);
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out);
typedef struct FstBuilder { typedef struct FstBuilder {
FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`. FstCountingWriter* wrt; // The FST raw data is written directly to `wtr`.
FstUnFinishedNodes *unfinished; // The stack of unfinished nodes FstUnFinishedNodes* unfinished; // The stack of unfinished nodes
FstRegistry * registry; // A map of finished nodes. FstRegistry* registry; // A map of finished nodes.
FstSlice last; // The last word added FstSlice last; // The last word added
CompiledAddr lastAddr; // The address of the last compiled node CompiledAddr lastAddr; // The address of the last compiled node
uint64_t len; // num of keys added uint64_t len; // num of keys added
} FstBuilder; } FstBuilder;
FstBuilder *fstBuilderCreate(void *w, FstType ty); FstBuilder* fstBuilderCreate(void* w, FstType ty);
void fstBuilderDestroy(FstBuilder *b); void fstBuilderDestroy(FstBuilder* b);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate);
void * fstBuilerIntoInner(FstBuilder *b); void* fstBuilerIntoInner(FstBuilder* b);
void fstBuilderFinish(FstBuilder *b); void fstBuilderFinish(FstBuilder* b);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn);
typedef struct FstTransitions { typedef struct FstTransitions {
FstNode *node; FstNode* node;
FstRange range; FstRange range;
} FstTransitions; } FstTransitions;
...@@ -121,56 +121,55 @@ typedef struct FstState { ...@@ -121,56 +121,55 @@ typedef struct FstState {
uint8_t val; uint8_t val;
} FstState; } FstState;
FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr); FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr);
FstState fstStateCreate(State state); FstState fstStateCreate(State state);
// compile // compile
void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp);
void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn);
void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node);
// set_comm_input // set_comm_input
void fstStateSetCommInput(FstState *state, uint8_t inp); void fstStateSetCommInput(FstState* state, uint8_t inp);
// comm_input // comm_input
uint8_t fstStateCommInput(FstState *state, bool *null); uint8_t fstStateCommInput(FstState* state, bool* null);
// input_len // input_len
uint64_t fstStateInputLen(FstState *state); uint64_t fstStateInputLen(FstState* state);
// end_addr // end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data); uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data);
uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes);
uint64_t fstStateEndAddrForAnyTrans( uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans);
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
// input // input
uint8_t fstStateInput(FstState *state, FstNode *node); uint8_t fstStateInput(FstState* state, FstNode* node);
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i);
// trans_addr // trans_addr
CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); CompiledAddr fstStateTransAddr(FstState* state, FstNode* node);
CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); CompiledAddr fstStateTransAddrForAnyTrans(FstState* state, FstNode* node, uint64_t i);
// sizes // sizes
PackSizes fstStateSizes(FstState *state, FstSlice *data); PackSizes fstStateSizes(FstState* state, FstSlice* data);
// Output // Output
Output fstStateOutput(FstState *state, FstNode *node); Output fstStateOutput(FstState* state, FstNode* node);
Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); Output fstStateOutputForAnyTrans(FstState* state, FstNode* node, uint64_t i);
// anyTrans specify function // anyTrans specify function
void fstStateSetFinalState(FstState *state, bool yes); void fstStateSetFinalState(FstState* state, bool yes);
bool fstStateIsFinalState(FstState *state); bool fstStateIsFinalState(FstState* state);
void fstStateSetStateNtrans(FstState *state, uint8_t n); void fstStateSetStateNtrans(FstState* state, uint8_t n);
// state_ntrans // state_ntrans
uint8_t fstStateStateNtrans(FstState *state, bool *null); uint8_t fstStateStateNtrans(FstState* state, bool* null);
uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); uint64_t fstStateTotalTransSize(FstState* state, uint64_t version, PackSizes size, uint64_t nTrans);
uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); uint64_t fstStateTransIndexSize(FstState* state, uint64_t version, uint64_t nTrans);
uint64_t fstStateNtransLen(FstState *state); uint64_t fstStateNtransLen(FstState* state);
uint64_t fstStateNtrans(FstState *state, FstSlice *slice); uint64_t fstStateNtrans(FstState* state, FstSlice* slice);
Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans);
uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
...@@ -187,13 +186,13 @@ typedef struct FstLastTransition { ...@@ -187,13 +186,13 @@ typedef struct FstLastTransition {
* TODO: simple function name * TODO: simple function name
*/ */
typedef struct FstBuilderNodeUnfinished { typedef struct FstBuilderNodeUnfinished {
FstBuilderNode * node; FstBuilderNode* node;
FstLastTransition *last; FstLastTransition* last;
} FstBuilderNodeUnfinished; } FstBuilderNodeUnfinished;
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* node, CompiledAddr addr);
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* node, Output out);
/* /*
* FstNode and helper function * FstNode and helper function
...@@ -224,18 +223,18 @@ typedef struct FstNode { ...@@ -224,18 +223,18 @@ typedef struct FstNode {
// Return the address of this node. // Return the address of this node.
#define FST_NODE_ADDR(node) node->start #define FST_NODE_ADDR(node) node->start
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* data);
void fstNodeDestroy(FstNode *fstNode); void fstNodeDestroy(FstNode* fstNode);
FstTransitions fstNodeTransitionIter(FstNode *node); FstTransitions fstNodeTransitionIter(FstNode* node);
FstTransitions *fstNodeTransitions(FstNode *node); FstTransitions* fstNodeTransitions(FstNode* node);
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* res);
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res);
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res);
bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode);
FstSlice fstNodeAsSlice(FstNode *node); FstSlice fstNodeAsSlice(FstNode* node);
// ops // ops
...@@ -244,8 +243,8 @@ typedef struct FstIndexedValue { ...@@ -244,8 +243,8 @@ typedef struct FstIndexedValue {
uint64_t value; uint64_t value;
} FstIndexedValue; } FstIndexedValue;
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out);
void fstLastTransitionDestroy(FstLastTransition *trn); void fstLastTransitionDestroy(FstLastTransition* trn);
typedef struct FstMeta { typedef struct FstMeta {
uint64_t version; uint64_t version;
...@@ -256,75 +255,74 @@ typedef struct FstMeta { ...@@ -256,75 +255,74 @@ typedef struct FstMeta {
} FstMeta; } FstMeta;
typedef struct Fst { typedef struct Fst {
FstMeta * meta; FstMeta* meta;
FstSlice *data; // FstSlice* data; //
FstNode * root; // FstNode* root; //
} Fst; } Fst;
// refactor simple function // refactor simple function
Fst *fstCreate(FstSlice *data); Fst* fstCreate(FstSlice* data);
void fstDestroy(Fst *fst); void fstDestroy(Fst* fst);
bool fstGet(Fst *fst, FstSlice *b, Output *out); bool fstGet(Fst* fst, FstSlice* b, Output* out);
FstNode * fstGetNode(Fst *fst, CompiledAddr); FstNode* fstGetNode(Fst* fst, CompiledAddr);
FstNode * fstGetRoot(Fst *fst); FstNode* fstGetRoot(Fst* fst);
FstType fstGetType(Fst *fst); FstType fstGetType(Fst* fst);
CompiledAddr fstGetRootAddr(Fst *fst); CompiledAddr fstGetRootAddr(Fst* fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null); Output fstEmptyFinalOutput(Fst* fst, bool* null);
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx);
// into stream to expand later // into stream to expand later
StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb); StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb);
bool fstVerify(Fst *fst); bool fstVerify(Fst* fst);
// refactor this function // refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
typedef struct StreamState { typedef struct StreamState {
FstNode * node; FstNode* node;
uint64_t trans; uint64_t trans;
FstOutput out; FstOutput out;
void * autState; void* autState;
} StreamState; } StreamState;
void streamStateDestroy(void *s); void streamStateDestroy(void* s);
typedef struct StreamWithState { typedef struct StreamWithState {
Fst * fst; Fst* fst;
AutomationCtx * aut; AutomationCtx* aut;
SArray * inp; SArray* inp;
FstOutput emptyOutput; FstOutput emptyOutput;
SArray * stack; // <StreamState> SArray* stack; // <StreamState>
FstBoundWithData *endAt; FstBoundWithData* endAt;
} StreamWithState; } StreamWithState;
typedef struct StreamWithStateResult { typedef struct StreamWithStateResult {
FstSlice data; FstSlice data;
FstOutput out; FstOutput out;
void * state; void* state;
} StreamWithStateResult; } StreamWithStateResult;
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
void swsResultDestroy(StreamWithStateResult *result); void swsResultDestroy(StreamWithStateResult* result);
typedef void *(*StreamCallback)(void *); typedef void* (*StreamCallback)(void*);
StreamWithState *streamWithStateCreate( StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max);
void streamWithStateDestroy(StreamWithState *sws); void streamWithStateDestroy(StreamWithState* sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min);
StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback);
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
// set up bound range // set up bound range
// refator, simple code by marco // refator, simple code by marco
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/index_fst_automation.h
浏览文件 @ 1e32e1f8
...@@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx; ...@@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
typedef struct StartWith { typedef struct StartWith {
AutomationCtx *autoSelf; AutomationCtx* autoSelf;
} StartWith; } StartWith;
typedef struct Complement { typedef struct Complement {
AutomationCtx *autoSelf; AutomationCtx* autoSelf;
} Complement; } Complement;
// automation // automation
typedef struct AutomationCtx { typedef struct AutomationCtx {
AutomationType type; AutomationType type;
void * stdata; void* stdata;
char * data; char* data;
} AutomationCtx; } AutomationCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
...@@ -48,27 +48,27 @@ typedef struct StartWithStateValue { ...@@ -48,27 +48,27 @@ typedef struct StartWithStateValue {
ValueType type; ValueType type;
union { union {
int val; int val;
char * ptr; char* ptr;
SArray *arr; SArray* arr;
// add more type // add more type
}; };
} StartWithStateValue; } StartWithStateValue;
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val);
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv);
void startWithStateValueDestroy(void *sv); void startWithStateValueDestroy(void* sv);
typedef struct AutomationFunc { typedef struct AutomationFunc {
void *(*start)(AutomationCtx *ctx); void* (*start)(AutomationCtx* ctx);
bool (*isMatch)(AutomationCtx *ctx, void *); bool (*isMatch)(AutomationCtx* ctx, void*);
bool (*canMatch)(AutomationCtx *ctx, void *data); bool (*canMatch)(AutomationCtx* ctx, void* data);
bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state);
void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte); void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte);
void *(*acceptEof)(AutomationCtx *ct, void *state); void* (*acceptEof)(AutomationCtx* ct, void* state);
} AutomationFunc; } AutomationFunc;
AutomationCtx *automCtxCreate(void *data, AutomationType atype); AutomationCtx* automCtxCreate(void* data, AutomationType atype);
void automCtxDestroy(AutomationCtx *ctx); void automCtxDestroy(AutomationCtx* ctx);
extern AutomationFunc automFuncs[]; extern AutomationFunc automFuncs[];
#ifdef __cplusplus #ifdef __cplusplus
......
source/libs/index/inc/index_fst_counting_writer.h
浏览文件 @ 1e32e1f8
...@@ -28,9 +28,9 @@ static char tmpFile[] = "./index"; ...@@ -28,9 +28,9 @@ static char tmpFile[] = "./index";
typedef enum WriterType { TMemory, TFile } WriterType; typedef enum WriterType { TMemory, TFile } WriterType;
typedef struct WriterCtx { typedef struct WriterCtx {
int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len);
int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len);
int (*flush)(struct WriterCtx *ctx); int (*flush)(struct WriterCtx* ctx);
WriterType type; WriterType type;
union { union {
struct { struct {
...@@ -39,41 +39,41 @@ typedef struct WriterCtx { ...@@ -39,41 +39,41 @@ typedef struct WriterCtx {
} file; } file;
struct { struct {
int32_t capa; int32_t capa;
char * buf; char* buf;
} mem; } mem;
}; };
int32_t offset; int32_t offset;
int32_t limit; int32_t limit;
} WriterCtx; } WriterCtx;
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len);
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len);
static int writeCtxDoFlush(WriterCtx *ctx); static int writeCtxDoFlush(WriterCtx* ctx);
WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity);
void writerCtxDestroy(WriterCtx *w); void writerCtxDestroy(WriterCtx* w);
typedef uint32_t CheckSummer; typedef uint32_t CheckSummer;
typedef struct FstCountingWriter { typedef struct FstCountingWriter {
void * wrt; // wrap any writer that counts and checksum bytes written void* wrt; // wrap any writer that counts and checksum bytes written
uint64_t count; uint64_t count;
CheckSummer summer; CheckSummer summer;
} FstCountingWriter; } FstCountingWriter;
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len);
int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len);
int fstCountingWriterFlush(FstCountingWriter *write); int fstCountingWriterFlush(FstCountingWriter* write);
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write);
FstCountingWriter *fstCountingWriterCreate(void *wtr); FstCountingWriter* fstCountingWriterCreate(void* wtr);
void fstCountingWriterDestroy(FstCountingWriter *w); void fstCountingWriterDestroy(FstCountingWriter* w);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes);
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n);
#define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
......
source/libs/index/inc/index_fst_node.h
浏览文件 @ 1e32e1f8
...@@ -36,20 +36,20 @@ typedef struct FstTransition { ...@@ -36,20 +36,20 @@ typedef struct FstTransition {
typedef struct FstBuilderNode { typedef struct FstBuilderNode {
bool isFinal; bool isFinal;
Output finalOutput; Output finalOutput;
SArray *trans; // <FstTransition> SArray* trans; // <FstTransition>
} FstBuilderNode; } FstBuilderNode;
FstBuilderNode *fstBuilderNodeDefault(); FstBuilderNode* fstBuilderNodeDefault();
FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src);
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src);
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, // bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr); // CompiledAddr lastAddr, CompiledAddr startAddr);
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2);
void fstBuilderNodeDestroy(FstBuilderNode *node); void fstBuilderNodeDestroy(FstBuilderNode* node);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/index_fst_registry.h
浏览文件 @ 1e32e1f8
...@@ -25,7 +25,7 @@ extern "C" { ...@@ -25,7 +25,7 @@ extern "C" {
typedef struct FstRegistryCell { typedef struct FstRegistryCell {
CompiledAddr addr; CompiledAddr addr;
FstBuilderNode *node; FstBuilderNode* node;
} FstRegistryCell; } FstRegistryCell;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
...@@ -45,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState; ...@@ -45,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState;
typedef struct FstRegistryEntry { typedef struct FstRegistryEntry {
FstRegistryEntryState state; FstRegistryEntryState state;
CompiledAddr addr; CompiledAddr addr;
FstRegistryCell * cell; FstRegistryCell* cell;
} FstRegistryEntry; } FstRegistryEntry;
// Registry relation function // Registry relation function
typedef struct FstRegistry { typedef struct FstRegistry {
SArray * table; //<FstRegistryCell> SArray* table; //<FstRegistryCell>
uint64_t tableSize; // num of rows uint64_t tableSize; // num of rows
uint64_t mruSize; // num of columns uint64_t mruSize; // num of columns
} FstRegistry; } FstRegistry;
// //
FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
void fstRegistryDestroy(FstRegistry *registry); void fstRegistryDestroy(FstRegistry* registry);
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode);
void fstRegistryEntryDestroy(FstRegistryEntry *entry); void fstRegistryEntryDestroy(FstRegistryEntry* entry);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/index_fst_util.h
浏览文件 @ 1e32e1f8
...@@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD; ...@@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
// uint8_t commonIdx(uint8_t v, uint8_t max); // uint8_t commonIdx(uint8_t v, uint8_t max);
uint8_t packSize(uint64_t n); uint8_t packSize(uint64_t n);
uint64_t unpackUint64(uint8_t *ch, uint8_t sz); uint64_t unpackUint64(uint8_t* ch, uint8_t sz);
uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr);
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr);
typedef struct FstString { typedef struct FstString {
uint8_t *data; uint8_t* data;
uint32_t len; uint32_t len;
int32_t ref; int32_t ref;
} FstString; } FstString;
typedef struct FstSlice { typedef struct FstSlice {
FstString *str; FstString* str;
int32_t start; int32_t start;
int32_t end; int32_t end;
} FstSlice; } FstSlice;
FstSlice fstSliceCreate(uint8_t *data, uint64_t len); FstSlice fstSliceCreate(uint8_t* data, uint64_t len);
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end);
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end);
bool fstSliceIsEmpty(FstSlice *s); bool fstSliceIsEmpty(FstSlice* s);
int fstSliceCompare(FstSlice *s1, FstSlice *s2); int fstSliceCompare(FstSlice* s1, FstSlice* s2);
void fstSliceDestroy(FstSlice *s); void fstSliceDestroy(FstSlice* s);
uint8_t *fstSliceData(FstSlice *s, int32_t *sz); uint8_t* fstSliceData(FstSlice* s, int32_t* sz);
#define FST_SLICE_LEN(s) (s->end - s->start + 1) #define FST_SLICE_LEN(s) (s->end - s->start + 1)
......
source/libs/index/inc/index_tfile.h
浏览文件 @ 1e32e1f8
...@@ -26,84 +26,89 @@ ...@@ -26,84 +26,89 @@
extern "C" { extern "C" {
#endif #endif
// tfile header // tfile header content
// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| // |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->|
// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| // |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->|
typedef struct TFileReadHeader { typedef struct TFileHeader {
uint64_t suid; uint64_t suid;
int32_t version; int32_t version;
char colName[128]; // char colName[128]; //
uint8_t colType; uint8_t colType;
} TFileReadHeader; } TFileHeader;
#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); #define TFILE_HEADER_SIZE (sizeof(TFileHeader) + sizeof(uint32_t))
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) #define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileCacheKey { typedef struct TFileCacheKey {
uint64_t suid; uint64_t suid;
uint8_t colType; uint8_t colType;
int32_t version; int32_t version;
const char *colName; char* colName;
int32_t nColName; int32_t nColName;
} TFileCacheKey; } TFileCacheKey;
// table cache // table cache
// refactor to LRU cache later // refactor to LRU cache later
typedef struct TFileCache { typedef struct TFileCache {
SHashObj *tableCache; SHashObj* tableCache;
int16_t capacity; int16_t capacity;
// add more param // add more param
} TFileCache; } TFileCache;
typedef struct TFileWriter { typedef struct TFileWriter {
FstBuilder *fb; FstBuilder* fb;
WriterCtx * ctx; WriterCtx* ctx;
TFileHeader header;
uint32_t offset;
} TFileWriter; } TFileWriter;
typedef struct TFileReader { typedef struct TFileReader {
T_REF_DECLARE() T_REF_DECLARE()
Fst * fst; Fst* fst;
WriterCtx * ctx; WriterCtx* ctx;
TFileReadHeader header; TFileHeader header;
} TFileReader; } TFileReader;
typedef struct IndexTFile { typedef struct IndexTFile {
char * path; char* path;
TFileCache * cache; TFileCache* cache;
TFileWriter *tw; TFileWriter* tw;
} IndexTFile; } IndexTFile;
typedef struct TFileWriterOpt { typedef struct TFileWriterOpt {
uint64_t suid; uint64_t suid;
int8_t colType; int8_t colType;
char * colName; char* colName;
int32_t nColName; int32_t nColName;
int32_t version; int32_t version;
} TFileWriterOpt; } TFileWriterOpt;
typedef struct TFileReaderOpt { typedef struct TFileReaderOpt {
uint64_t suid; uint64_t suid;
char * colName; char* colName;
int32_t nColName; int32_t nColName;
} TFileReaderOpt; } TFileReaderOpt;
// tfile cache, manage tindex reader // tfile cache, manage tindex reader
TFileCache * tfileCacheCreate(const char *path); TFileCache* tfileCacheCreate(const char* path);
void tfileCacheDestroy(TFileCache *tcache); void tfileCacheDestroy(TFileCache* tcache);
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key);
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader);
TFileReader *tfileReaderCreate(); TFileReader* tfileReaderCreate(WriterCtx* ctx);
void TFileReaderDestroy(TFileReader *reader); void tfileReaderDestroy(TFileReader* reader);
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header);
void tfileWriterDestroy(TFileWriter *tw); void tfileWriterDestroy(TFileWriter* tw);
int tfileWriterPut(TFileWriter* tw, void* data);
int tfileWriterFinish(TFileWriter* tw);
// //
IndexTFile *indexTFileCreate(const char *path); IndexTFile* indexTFileCreate(const char* path);
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/index/inc/index_util.h
浏览文件 @ 1e32e1f8
...@@ -19,29 +19,29 @@ ...@@ -19,29 +19,29 @@
extern "C" { extern "C" {
#endif #endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ #define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \ do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \ buf += sizeof(key->mem); \
} while (0) } while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \ do { \
memcpy((void *)buf, (void *)key->mem, len); \ memcpy((void*)buf, (void*)key->mem, len); \
buf += len; \ buf += len; \
} while (0) } while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ #define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \ do { \
type c = var; \ type c = var; \
assert(sizeof(var) == sizeof(type)); \ assert(sizeof(var) == sizeof(type)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \ memcpy((void*)buf, (void*)&c, sizeof(c)); \
buf += sizeof(c); \ buf += sizeof(c); \
} while (0) } while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \ do { \
memcpy((void *)buf, (void *)var, len); \ memcpy((void*)buf, (void*)var, len); \
buf += len; \ buf += len; \
} while (0) } while (0)
......
source/libs/index/src/index.c
浏览文件 @ 1e32e1f8
...@@ -23,9 +23,9 @@ ...@@ -23,9 +23,9 @@
#include "lucene++/Lucene_c.h" #include "lucene++/Lucene_c.h"
#endif #endif
static int uidCompare(const void *a, const void *b) { static int uidCompare(const void* a, const void* b) {
uint64_t u1 = *(uint64_t *)a; uint64_t u1 = *(uint64_t*)a;
uint64_t u2 = *(uint64_t *)b; uint64_t u2 = *(uint64_t*)b;
if (u1 == u2) { if (u1 == u2) {
return 0; return 0;
} else { } else {
...@@ -40,25 +40,26 @@ typedef struct SIdxColInfo { ...@@ -40,25 +40,26 @@ typedef struct SIdxColInfo {
static pthread_once_t isInit = PTHREAD_ONCE_INIT; static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit(); static void indexInit();
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
static int indexMergeCacheIntoTindex(SIndex *sIdx); static int indexMergeCacheIntoTindex(SIndex* sIdx);
static void indexInterResultsDestroy(SArray *results); static void indexInterResultsDestroy(SArray* results);
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult);
int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit); pthread_once(&isInit, indexInit);
SIndex *sIdx = calloc(1, sizeof(SIndex)); SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { if (sIdx == NULL) {
return -1; return -1;
} }
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t *index = index_open(path); index_t* index = index_open(path);
sIdx->index = index; sIdx->index = index;
#endif #endif
sIdx->cache = (void *)indexCacheCreate(); #ifdef USE_INVERTED_INDEX
sIdx->cache = (void*)indexCacheCreate();
sIdx->tindex = NULL; sIdx->tindex = NULL;
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colId = 1; sIdx->colId = 1;
...@@ -67,9 +68,13 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { ...@@ -67,9 +68,13 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
*index = sIdx; *index = sIdx;
return 0; return 0;
#endif
*index = NULL;
return -1;
} }
void indexClose(SIndex *sIdx) { void indexClose(SIndex* sIdx) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_close(sIdex->index); index_close(sIdex->index);
sIdx->index = NULL; sIdx->index = NULL;
...@@ -84,16 +89,16 @@ void indexClose(SIndex *sIdx) { ...@@ -84,16 +89,16 @@ void indexClose(SIndex *sIdx) {
return; return;
} }
int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_document_t *doc = index_document_create(); index_document_t* doc = index_document_create();
char buf[16] = {0}; char buf[16] = {0};
sprintf(buf, "%d", uid); sprintf(buf, "%d", uid);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i); SIndexTerm* p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1);
} }
index_document_add(doc, NULL, 0, buf, strlen(buf), 0); index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
...@@ -106,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { ...@@ -106,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
// TODO(yihao): reduce the lock range // TODO(yihao): reduce the lock range
pthread_mutex_lock(&index->mtx); pthread_mutex_lock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm * p = taosArrayGetP(fVals, i); SIndexTerm* p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName);
if (fi == NULL) { if (fi == NULL) {
SIdxColInfo tfi = {.colId = index->colId}; SIdxColInfo tfi = {.colId = index->colId};
index->cVersion++; index->cVersion++;
...@@ -120,8 +125,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { ...@@ -120,8 +125,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
pthread_mutex_unlock(&index->mtx); pthread_mutex_unlock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm * p = taosArrayGetP(fVals, i); SIndexTerm* p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName);
assert(fi != NULL); assert(fi != NULL);
int32_t colId = fi->colId; int32_t colId = fi->colId;
int32_t version = index->cVersion; int32_t version = index->cVersion;
...@@ -134,18 +139,18 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { ...@@ -134,18 +139,18 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
return 0; return 0;
} }
int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
EIndexOperatorType opera = multiQuerys->opera; EIndexOperatorType opera = multiQuerys->opera;
int nQuery = taosArrayGetSize(multiQuerys->query); int nQuery = taosArrayGetSize(multiQuerys->query);
char **fields = malloc(sizeof(char *) * nQuery); char** fields = malloc(sizeof(char*) * nQuery);
char **keys = malloc(sizeof(char *) * nQuery); char** keys = malloc(sizeof(char*) * nQuery);
int * types = malloc(sizeof(int) * nQuery); int* types = malloc(sizeof(int) * nQuery);
for (int i = 0; i < nQuery; i++) { for (int i = 0; i < nQuery; i++) {
SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i);
SIndexTerm * term = p->field_value; SIndexTerm* term = p->field_value;
fields[i] = calloc(1, term->nKey + 1); fields[i] = calloc(1, term->nKey + 1);
keys[i] = calloc(1, term->nVal + 1); keys[i] = calloc(1, term->nVal + 1);
...@@ -154,9 +159,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result ...@@ -154,9 +159,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
memcpy(keys[i], term->val, term->nVal); memcpy(keys[i], term->val, term->nVal);
types[i] = (int)(p->type); types[i] = (int)(p->type);
} }
int *tResult = NULL; int* tResult = NULL;
int tsz = 0; int tsz = 0;
index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz);
for (int i = 0; i < tsz; i++) { for (int i = 0; i < tsz; i++) {
taosArrayPush(result, &tResult[i]); taosArrayPush(result, &tResult[i]);
...@@ -174,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result ...@@ -174,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
EIndexOperatorType opera = multiQuerys->opera; // relation of querys EIndexOperatorType opera = multiQuerys->opera; // relation of querys
SArray *interResults = taosArrayInit(4, POINTER_BYTES); SArray* interResults = taosArrayInit(4, POINTER_BYTES);
int nQuery = taosArrayGetSize(multiQuerys->query); int nQuery = taosArrayGetSize(multiQuerys->query);
for (size_t i = 0; i < nQuery; i++) { for (size_t i = 0; i < nQuery; i++) {
SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i);
SArray * tResult = NULL; SArray* tResult = NULL;
indexTermSearch(index, qTerm, &tResult); indexTermSearch(index, qTerm, &tResult);
taosArrayPush(interResults, (void *)&tResult); taosArrayPush(interResults, (void*)&tResult);
} }
indexMergeFinalResults(interResults, opera, result); indexMergeFinalResults(interResults, opera, result);
indexInterResultsDestroy(interResults); indexInterResultsDestroy(interResults);
...@@ -189,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result ...@@ -189,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return 1; return 1;
} }
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { int indexDelete(SIndex* index, SIndexMultiTermQuery* query) {
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
#endif #endif
return 1; return 1;
} }
int indexRebuild(SIndex *index, SIndexOpts *opts){ int indexRebuild(SIndex* index, SIndexOpts* opts){
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
#endif #endif
} }
SIndexOpts *indexOptsCreate() { SIndexOpts* indexOptsCreate() {
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
return NULL; return NULL;
} }
void indexOptsDestroy(SIndexOpts *opts){ void indexOptsDestroy(SIndexOpts* opts){
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
} /* } /*
...@@ -214,8 +219,8 @@ void indexOptsDestroy(SIndexOpts *opts){ ...@@ -214,8 +219,8 @@ void indexOptsDestroy(SIndexOpts *opts){
* *
*/ */
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { if (p == NULL) {
return NULL; return NULL;
} }
...@@ -223,23 +228,28 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { ...@@ -223,23 +228,28 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p; return p;
} }
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) {
for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) {
SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); SIndexTermQuery* p = (SIndexTermQuery*)taosArrayGet(pQuery->query, i);
indexTermDestroy(p->term); indexTermDestroy(p->term);
} }
taosArrayDestroy(pQuery->query); taosArrayDestroy(pQuery->query);
free(pQuery); free(pQuery);
}; };
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) { int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) {
SIndexTermQuery q = {.qType = qType, .term = term}; SIndexTermQuery q = {.qType = qType, .term = term};
taosArrayPush(pQuery->query, &q); taosArrayPush(pQuery->query, &q);
return 0; return 0;
} }
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, SIndexTerm* indexTermCreate(int64_t suid,
int32_t nColName, const char *colVal, int32_t nColVal) { SIndexOperOnColumn oper,
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); uint8_t colType,
const char* colName,
int32_t nColName,
const char* colVal,
int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { if (t == NULL) {
return NULL; return NULL;
} }
...@@ -248,30 +258,32 @@ SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy ...@@ -248,30 +258,32 @@ SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
t->operType = oper; t->operType = oper;
t->colType = colType; t->colType = colType;
t->colName = (char *)calloc(1, nColName + 1); t->colName = (char*)calloc(1, nColName + 1);
memcpy(t->colName, colName, nColName); memcpy(t->colName, colName, nColName);
t->nColName = nColName; t->nColName = nColName;
t->colVal = (char *)calloc(1, nColVal + 1); t->colVal = (char*)calloc(1, nColVal + 1);
memcpy(t->colVal, colVal, nColVal); memcpy(t->colVal, colVal, nColVal);
t->nColVal = nColVal; t->nColVal = nColVal;
return t; return t;
} }
void indexTermDestroy(SIndexTerm *p) { void indexTermDestroy(SIndexTerm* p) {
free(p->colName); free(p->colName);
free(p->colVal); free(p->colVal);
free(p); free(p);
} }
SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } SIndexMultiTerm* indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm*));
}
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) {
taosArrayPush(terms, &term); taosArrayPush(terms, &term);
return 0; return 0;
} }
void indexMultiTermDestroy(SIndexMultiTerm *terms) { void indexMultiTermDestroy(SIndexMultiTerm* terms) {
for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
SIndexTerm *p = taosArrayGetP(terms, i); SIndexTerm* p = taosArrayGetP(terms, i);
indexTermDestroy(p); indexTermDestroy(p);
} }
taosArrayDestroy(terms); taosArrayDestroy(terms);
...@@ -280,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) { ...@@ -280,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) {
void indexInit() { void indexInit() {
// do nothing // do nothing
} }
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
int32_t version = -1; int32_t version = -1;
int16_t colId = -1; int16_t colId = -1;
SIdxColInfo *colInfo = NULL; SIdxColInfo* colInfo = NULL;
SIndexTerm *term = query->term; SIndexTerm* term = query->term;
const char *colName = term->colName; const char* colName = term->colName;
int32_t nColName = term->nColName; int32_t nColName = term->nColName;
pthread_mutex_lock(&sIdx->mtx); pthread_mutex_lock(&sIdx->mtx);
...@@ -319,23 +331,24 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result ...@@ -319,23 +331,24 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
} }
return 0; return 0;
} }
static void indexInterResultsDestroy(SArray *results) { static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) { if (results == NULL) {
return; return;
} }
size_t sz = taosArrayGetSize(results); size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
SArray *p = taosArrayGetP(results, i); SArray* p = taosArrayGetP(results, i);
taosArrayDestroy(p); taosArrayDestroy(p);
} }
taosArrayDestroy(results); taosArrayDestroy(results);
} }
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType // refactor, merge interResults into fResults by oType
SArray *first = taosArrayGetP(interResults, 0); SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare); taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL); taosArrayRemoveDuplicate(first, uidCompare, NULL);
if (oType == MUST) { if (oType == MUST) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, first);
...@@ -350,7 +363,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType ...@@ -350,7 +363,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType
} }
return 0; return 0;
} }
static int indexMergeCacheIntoTindex(SIndex *sIdx) { static int indexMergeCacheIntoTindex(SIndex* sIdx) {
if (sIdx == NULL) { if (sIdx == NULL) {
return -1; return -1;
} }
......
source/libs/index/src/index_cache.c
浏览文件 @ 1e32e1f8
...@@ -20,14 +20,15 @@ ...@@ -20,14 +20,15 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later #define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22 // ref index_cache.h:22
#define CACHE_KEY_LEN(p) \ #define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \ (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
sizeof(p->operType))
static char * getIndexKey(const void *pData) { return NULL; } static char* getIndexKey(const void* pData) {
static int32_t compareKey(const void *l, const void *r) { return NULL;
char *lp = (char *)l; }
char *rp = (char *)r; static int32_t compareKey(const void* l, const void* r) {
char* lp = (char*)l;
char* rp = (char*)r;
// skip total len, not compare // skip total len, not compare
int32_t ll, rl; // len int32_t ll, rl; // len
...@@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) { ...@@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) {
int16_t lf, rf; // field id int16_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf)); memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf)); memcpy(&rf, rp, sizeof(rf));
if (lf != rf) { if (lf != rf) { return lf < rf ? -1 : 1; }
return lf < rf ? -1 : 1;
}
lp += sizeof(lf); lp += sizeof(lf);
rp += sizeof(rf); rp += sizeof(rf);
...@@ -89,41 +88,41 @@ static int32_t compareKey(const void *l, const void *r) { ...@@ -89,41 +88,41 @@ static int32_t compareKey(const void *l, const void *r) {
int32_t lv, rv; int32_t lv, rv;
memcpy(&lv, lp, sizeof(lv)); memcpy(&lv, lp, sizeof(lv));
memcpy(&rv, rp, sizeof(rv)); memcpy(&rv, rp, sizeof(rv));
if (lv != rv) { if (lv != rv) { return lv > rv ? -1 : 1; }
return lv > rv ? -1 : 1;
}
lp += sizeof(lv); lp += sizeof(lv);
rp += sizeof(rv); rp += sizeof(rv);
// not care item type // not care item type
return 0; return 0;
} }
IndexCache *indexCacheCreate() { IndexCache* indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache)); IndexCache* cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate( if (cache == NULL) {
MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); indexError("failed to create index cache");
return NULL;
}
cache->skiplist =
tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache; return cache;
} }
void indexCacheDestroy(void *cache) { void indexCacheDestroy(void* cache) {
IndexCache *pCache = cache; IndexCache* pCache = cache;
if (pCache == NULL) { if (pCache == NULL) { return; }
return;
}
tSkipListDestroy(pCache->skiplist); tSkipListDestroy(pCache->skiplist);
free(pCache); free(pCache);
} }
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
if (cache == NULL) { if (cache == NULL) { return -1; }
return -1;
}
IndexCache *pCache = cache; IndexCache* pCache = cache;
// encode data // encode data
int32_t total = CACHE_KEY_LEN(term); int32_t total = CACHE_KEY_LEN(term);
char * buf = calloc(1, total);
char * p = buf; char* buf = calloc(1, total);
char* p = buf;
SERIALIZE_VAR_TO_BUF(p, total, int32_t); SERIALIZE_VAR_TO_BUF(p, total, int32_t);
SERIALIZE_VAR_TO_BUF(p, colId, int16_t); SERIALIZE_VAR_TO_BUF(p, colId, int16_t);
...@@ -137,30 +136,31 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, ...@@ -137,30 +136,31 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version,
SERIALIZE_MEM_TO_BUF(p, term, operType); SERIALIZE_MEM_TO_BUF(p, term, operType);
tSkipListPut(pCache->skiplist, (void *)buf); tSkipListPut(pCache->skiplist, (void*)buf);
return 0; return 0;
// encode end // encode end
} }
int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { int indexCacheDel(void* cache, int32_t fieldId, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache; IndexCache* pCache = cache;
return 0; return 0;
} }
int indexCacheSearch( int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { if (cache == NULL) { return -1; }
if (cache == NULL) { IndexCache* pCache = cache;
return -1; SIndexTerm* term = query->term;
}
IndexCache * pCache = cache;
SIndexTerm * term = query->term;
EIndexQueryType qtype = query->qType; EIndexQueryType qtype = query->qType;
int32_t keyLen = CACHE_KEY_LEN(term); int32_t keyLen = CACHE_KEY_LEN(term);
char *buf = calloc(1, keyLen); char* buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) { if (qtype == QUERY_TERM) {
//
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
//
} else if (qtype == QUERY_SUFFIX) { } else if (qtype == QUERY_SUFFIX) {
//
} else if (qtype == QUERY_REGEX) { } else if (qtype == QUERY_REGEX) {
//
} }
return 0; return 0;
......
source/libs/index/src/index_fst.c
浏览文件 @ 1e32e1f8
...@@ -19,43 +19,39 @@ ...@@ -19,43 +19,39 @@
#include "tchecksum.h" #include "tchecksum.h"
#include "tcoding.h" #include "tcoding.h"
static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr;
fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes);
} }
static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) {
uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); uint8_t nBytes = packDeltaSize(nodeAddr, transAddr);
fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes);
return nBytes; return nBytes;
} }
FstUnFinishedNodes *fstUnFinishedNodesCreate() { FstUnFinishedNodes* fstUnFinishedNodesCreate() {
FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes));
if (nodes == NULL) { if (nodes == NULL) { return NULL; }
return NULL;
}
nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
fstUnFinishedNodesPushEmpty(nodes, false); fstUnFinishedNodesPushEmpty(nodes, false);
return nodes; return nodes;
} }
void unFinishedNodeDestroyElem(void *elem) { void unFinishedNodeDestroyElem(void* elem) {
FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished *)elem; FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem;
fstBuilderNodeDestroy(b->node); fstBuilderNodeDestroy(b->node);
free(b->last); free(b->last);
b->last = NULL; b->last = NULL;
} }
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) {
if (nodes == NULL) { if (nodes == NULL) { return; }
return;
}
taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
free(nodes); free(nodes);
} }
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); FstBuilderNode* node = malloc(sizeof(FstBuilderNode));
node->isFinal = isFinal; node->isFinal = isFinal;
node->finalOutput = 0; node->finalOutput = 0;
node->trans = taosArrayInit(16, sizeof(FstTransition)); node->trans = taosArrayInit(16, sizeof(FstTransition));
...@@ -63,56 +59,53 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { ...@@ -63,56 +59,53 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) {
FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; FstBuilderNodeUnfinished un = {.node = node, .last = NULL};
taosArrayPush(nodes->stack, &un); taosArrayPush(nodes->stack, &un);
} }
FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) {
assert(taosArrayGetSize(nodes->stack) == 1); assert(taosArrayGetSize(nodes->stack) == 1);
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL); assert(un->last == NULL);
return un->node; return un->node;
} }
FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) {
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
fstBuilderNodeUnfinishedLastCompiled(un, addr); fstBuilderNodeUnfinishedLastCompiled(un, addr);
// free(un->last); // TODO add func FstLastTransitionFree() // free(un->last); // TODO add func FstLastTransitionFree()
// un->last = NULL; // un->last = NULL;
return un->node; return un->node;
} }
FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) { FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) {
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL); assert(un->last == NULL);
return un->node; return un->node;
} }
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) {
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0);
un->node->isFinal = true; un->node->isFinal = true;
un->node->finalOutput = out; un->node->finalOutput = out;
// un->node->trans = NULL; // un->node->trans = NULL;
} }
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) {
size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1);
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz);
fstBuilderNodeUnfinishedLastCompiled(un, addr); fstBuilderNodeUnfinishedLastCompiled(un, addr);
} }
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) {
FstSlice *s = &bs; FstSlice* s = &bs;
if (fstSliceIsEmpty(s)) { if (fstSliceIsEmpty(s)) { return; }
return;
}
size_t sz = taosArrayGetSize(nodes->stack) - 1; size_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL); assert(un->last == NULL);
// FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // FstLastTransition *trn = malloc(sizeof(FstLastTransition));
// trn->inp = s->data[s->start]; // trn->inp = s->data[s->start];
// trn->out = out; // trn->out = out;
int32_t len = 0; int32_t len = 0;
uint8_t *data = fstSliceData(s, &len); uint8_t* data = fstSliceData(s, &len);
un->last = fstLastTransitionCreate(data[0], out); un->last = fstLastTransitionCreate(data[0], out);
for (uint64_t i = 1; i < len; i++) { for (uint64_t i = 1; i < len; i++) {
FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); FstBuilderNode* n = malloc(sizeof(FstBuilderNode));
n->isFinal = false; n->isFinal = false;
n->finalOutput = 0; n->finalOutput = 0;
n->trans = taosArrayInit(16, sizeof(FstTransition)); n->trans = taosArrayInit(16, sizeof(FstTransition));
...@@ -120,7 +113,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output ...@@ -120,7 +113,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
// FstLastTransition *trn = malloc(sizeof(FstLastTransition)); // FstLastTransition *trn = malloc(sizeof(FstLastTransition));
// trn->inp = s->data[i]; // trn->inp = s->data[i];
// trn->out = out; // trn->out = out;
FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); FstLastTransition* trn = fstLastTransitionCreate(data[i], 0);
FstBuilderNodeUnfinished un = {.node = n, .last = trn}; FstBuilderNodeUnfinished un = {.node = n, .last = trn};
taosArrayPush(nodes->stack, &un); taosArrayPush(nodes->stack, &un);
...@@ -128,15 +121,15 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output ...@@ -128,15 +121,15 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
fstUnFinishedNodesPushEmpty(nodes, true); fstUnFinishedNodesPushEmpty(nodes, true);
} }
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) {
FstSlice *s = &bs; FstSlice* s = &bs;
size_t ssz = taosArrayGetSize(node->stack); // stack size size_t ssz = taosArrayGetSize(node->stack); // stack size
uint64_t count = 0; uint64_t count = 0;
int32_t lsz; // data len int32_t lsz; // data len
uint8_t *data = fstSliceData(s, &lsz); uint8_t* data = fstSliceData(s, &lsz);
for (size_t i = 0; i < ssz && i < lsz; i++) { for (size_t i = 0; i < ssz && i < lsz; i++) {
FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i);
if (un->last->inp == data[i]) { if (un->last->inp == data[i]) {
count++; count++;
} else { } else {
...@@ -145,19 +138,19 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) ...@@ -145,19 +138,19 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs)
} }
return count; return count;
} }
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) { uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) {
FstSlice *s = &bs; FstSlice* s = &bs;
size_t lsz = (size_t)(s->end - s->start + 1); // data len size_t lsz = (size_t)(s->end - s->start + 1); // data len
size_t ssz = taosArrayGetSize(node->stack); // stack size size_t ssz = taosArrayGetSize(node->stack); // stack size
*out = in; *out = in;
uint64_t i = 0; uint64_t i = 0;
for (i = 0; i < lsz && i < ssz; i++) { for (i = 0; i < lsz && i < ssz; i++) {
FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i);
FstLastTransition *t = un->last; FstLastTransition* t = un->last;
uint64_t addPrefix = 0; uint64_t addPrefix = 0;
uint8_t * data = fstSliceData(s, NULL); uint8_t* data = fstSliceData(s, NULL);
if (t && t->inp == data[i]) { if (t && t->inp == data[i]) {
uint64_t commPrefix = MIN(t->out, *out); uint64_t commPrefix = MIN(t->out, *out);
uint64_t tAddPrefix = t->out - commPrefix; uint64_t tAddPrefix = t->out - commPrefix;
...@@ -169,7 +162,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, ...@@ -169,7 +162,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
} }
if (addPrefix != 0) { if (addPrefix != 0) {
if (i + 1 < ssz) { if (i + 1 < ssz) {
FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1);
fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix);
} }
} }
...@@ -177,13 +170,11 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, ...@@ -177,13 +170,11 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
return i; return i;
} }
FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
FstState fs = {.state = EmptyFinal, .val = 0}; FstState fs = {.state = EmptyFinal, .val = 0};
if (addr == EMPTY_ADDRESS) { if (addr == EMPTY_ADDRESS) { return fs; }
return fs;
}
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
uint8_t v = data[addr]; uint8_t v = data[addr];
uint8_t t = (v & 0b11000000) >> 6; uint8_t t = (v & 0b11000000) >> 6;
if (t == 0b11) { if (t == 0b11) {
...@@ -197,17 +188,19 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { ...@@ -197,17 +188,19 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) {
return fs; return fs;
} }
static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000}, static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000},
{.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}}; {.state = OneTrans, .val = 0b10000000},
{.state = AnyTrans, .val = 0b00000000},
{.state = EmptyFinal, .val = 0b00000000}};
// debug // debug
static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"};
FstState fstStateCreate(State state) { FstState fstStateCreate(State state) {
uint8_t idx = (uint8_t)state; uint8_t idx = (uint8_t)state;
return fstStateDict[idx]; return fstStateDict[idx];
} }
// compile // compile
void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp) { void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) {
FstState s = fstStateCreate(OneTransNext); FstState s = fstStateCreate(OneTransNext);
fstStateSetCommInput(&s, inp); fstStateSetCommInput(&s, inp);
...@@ -221,7 +214,7 @@ void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uin ...@@ -221,7 +214,7 @@ void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uin
// w->write_all(&[s.val]) // w->write_all(&[s.val])
return; return;
} }
void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn) { void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) {
Output out = trn->out; Output out = trn->out;
uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out));
uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); uint8_t transPackSize = fstPackDetla(w, addr, trn->addr);
...@@ -229,20 +222,18 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran ...@@ -229,20 +222,18 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran
FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize);
FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize);
fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes));
FstState st = fstStateCreate(OneTrans); FstState st = fstStateCreate(OneTrans);
fstStateSetCommInput(&st, trn->inp); fstStateSetCommInput(&st, trn->inp);
bool null = false; bool null = false;
uint8_t inp = fstStateCommInput(&st, &null); uint8_t inp = fstStateCommInput(&st, &null);
if (null == true) { if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); }
fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp)); fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
}
fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val));
return; return;
} }
void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) {
size_t sz = taosArrayGetSize(node->trans); size_t sz = taosArrayGetSize(node->trans);
assert(sz <= 256); assert(sz <= 256);
...@@ -252,7 +243,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil ...@@ -252,7 +243,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// finalOutput.is_zero() // finalOutput.is_zero()
bool anyOuts = (node->finalOutput != 0); bool anyOuts = (node->finalOutput != 0);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
tSize = MAX(tSize, packDeltaSize(addr, t->addr)); tSize = MAX(tSize, packDeltaSize(addr, t->addr));
oSize = MAX(oSize, packSize(t->out)); oSize = MAX(oSize, packSize(t->out));
anyOuts = anyOuts || (t->out != 0); anyOuts = anyOuts || (t->out != 0);
...@@ -272,21 +263,19 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil ...@@ -272,21 +263,19 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
fstStateSetStateNtrans(&st, (uint8_t)sz); fstStateSetStateNtrans(&st, (uint8_t)sz);
if (anyOuts) { if (anyOuts) {
if (FST_BUILDER_NODE_IS_FINAL(node)) { if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); }
fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
}
for (int32_t i = sz - 1; i >= 0; i--) { for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition *t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
fstCountingWriterPackUintIn(w, t->out, oSize); fstCountingWriterPackUintIn(w, t->out, oSize);
} }
} }
for (int32_t i = sz - 1; i >= 0; i--) { for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition *t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
fstPackDeltaIn(w, addr, t->addr, tSize); fstPackDeltaIn(w, addr, t->addr, tSize);
} }
for (int32_t i = sz - 1; i >= 0; i--) { for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition *t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
fstCountingWriterWrite(w, (char *)&t->inp, 1); fstCountingWriterWrite(w, (char*)&t->inp, 1);
// fstPackDeltaIn(w, addr, t->addr, tSize); // fstPackDeltaIn(w, addr, t->addr, tSize);
} }
if (sz > TRANS_INDEX_THRESHOLD) { if (sz > TRANS_INDEX_THRESHOLD) {
...@@ -294,20 +283,20 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil ...@@ -294,20 +283,20 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// at that index. (Except when there are 256 transitions.) Namely, // at that index. (Except when there are 256 transitions.) Namely,
// any value greater than or equal to the number of transitions in // any value greater than or equal to the number of transitions in
// this node indicates an absent transition. // this node indicates an absent transition.
uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256);
memset(index, 255, sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256);
/// for (uint8_t i = 0; i < 256; i++) { /// for (uint8_t i = 0; i < 256; i++) {
// index[i] = 255; // index[i] = 255;
///} ///}
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
index[t->inp] = i; index[t->inp] = i;
// fstPackDeltaIn(w, addr, t->addr, tSize); // fstPackDeltaIn(w, addr, t->addr, tSize);
} }
fstCountingWriterWrite(w, (char *)index, 256); fstCountingWriterWrite(w, (char*)index, 256);
free(index); free(index);
} }
fstCountingWriterWrite(w, (char *)&packSizes, 1); fstCountingWriterWrite(w, (char*)&packSizes, 1);
bool null = false; bool null = false;
fstStateStateNtrans(&st, &null); fstStateStateNtrans(&st, &null);
if (null == true) { if (null == true) {
...@@ -316,17 +305,17 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil ...@@ -316,17 +305,17 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// encoded in the state byte. // encoded in the state byte.
uint8_t v = 1; uint8_t v = 1;
if (sz == 256) { if (sz == 256) {
fstCountingWriterWrite(w, (char *)&v, 1); fstCountingWriterWrite(w, (char*)&v, 1);
} else { } else {
fstCountingWriterWrite(w, (char *)&sz, 1); fstCountingWriterWrite(w, (char*)&sz, 1);
} }
} }
fstCountingWriterWrite(w, (char *)(&(st.val)), 1); fstCountingWriterWrite(w, (char*)(&(st.val)), 1);
return; return;
} }
// set_comm_input // set_comm_input
void fstStateSetCommInput(FstState *s, uint8_t inp) { void fstStateSetCommInput(FstState* s, uint8_t inp) {
assert(s->state == OneTransNext || s->state == OneTrans); assert(s->state == OneTransNext || s->state == OneTrans);
uint8_t val; uint8_t val;
...@@ -335,7 +324,7 @@ void fstStateSetCommInput(FstState *s, uint8_t inp) { ...@@ -335,7 +324,7 @@ void fstStateSetCommInput(FstState *s, uint8_t inp) {
} }
// comm_input // comm_input
uint8_t fstStateCommInput(FstState *s, bool *null) { uint8_t fstStateCommInput(FstState* s, bool* null) {
assert(s->state == OneTransNext || s->state == OneTrans); assert(s->state == OneTransNext || s->state == OneTrans);
uint8_t v = s->val & 0b00111111; uint8_t v = s->val & 0b00111111;
if (v == 0) { if (v == 0) {
...@@ -348,7 +337,7 @@ uint8_t fstStateCommInput(FstState *s, bool *null) { ...@@ -348,7 +337,7 @@ uint8_t fstStateCommInput(FstState *s, bool *null) {
// input_len // input_len
uint64_t fstStateInputLen(FstState *s) { uint64_t fstStateInputLen(FstState* s) {
assert(s->state == OneTransNext || s->state == OneTrans); assert(s->state == OneTransNext || s->state == OneTrans);
bool null = false; bool null = false;
fstStateCommInput(s, &null); fstStateCommInput(s, &null);
...@@ -356,17 +345,16 @@ uint64_t fstStateInputLen(FstState *s) { ...@@ -356,17 +345,16 @@ uint64_t fstStateInputLen(FstState *s) {
} }
// end_addr // end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState *s, FstSlice *data) { uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) {
assert(s->state == OneTransNext); assert(s->state == OneTransNext);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s);
} }
uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) {
assert(s->state == OneTrans); assert(s->state == OneTrans);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size
- FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes);
} }
uint64_t fstStateEndAddrForAnyTrans( uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) {
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes;
return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size
...@@ -374,29 +362,29 @@ uint64_t fstStateEndAddrForAnyTrans( ...@@ -374,29 +362,29 @@ uint64_t fstStateEndAddrForAnyTrans(
- finalOsize; // final output - finalOsize; // final output
} }
// input // input
uint8_t fstStateInput(FstState *s, FstNode *node) { uint8_t fstStateInput(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans); assert(s->state == OneTransNext || s->state == OneTrans);
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
bool null = false; bool null = false;
uint8_t inp = fstStateCommInput(s, &null); uint8_t inp = fstStateCommInput(s, &null);
uint8_t * data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return null == false ? inp : data[-1]; return null == false ? inp : data[-1];
} }
uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return data[at]; return data[at];
} }
// trans_addr // trans_addr
CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans); assert(s->state == OneTransNext || s->state == OneTrans);
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
if (s->state == OneTransNext) { if (s->state == OneTransNext) {
return (CompiledAddr)(node->end) - 1; return (CompiledAddr)(node->end) - 1;
} else { } else {
...@@ -406,23 +394,23 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { ...@@ -406,23 +394,23 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) {
- tSizes; - tSizes;
// refactor error logic // refactor error logic
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return unpackDelta(data + i, tSizes, node->end); return unpackDelta(data + i, tSizes, node->end);
} }
} }
CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans -
node->nTrans - (i * tSizes) - tSizes; (i * tSizes) - tSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return unpackDelta(data + at, tSizes, node->end); return unpackDelta(data + at, tSizes, node->end);
} }
// sizes // sizes
PackSizes fstStateSizes(FstState *s, FstSlice *slice) { PackSizes fstStateSizes(FstState* s, FstSlice* slice) {
assert(s->state == OneTrans || s->state == AnyTrans); assert(s->state == OneTrans || s->state == AnyTrans);
uint64_t i; uint64_t i;
if (s->state == OneTrans) { if (s->state == OneTrans) {
...@@ -431,33 +419,29 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) { ...@@ -431,33 +419,29 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) {
i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1;
} }
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return (PackSizes)(*(data + i)); return (PackSizes)(*(data + i));
} }
// Output // Output
Output fstStateOutput(FstState *s, FstNode *node) { Output fstStateOutput(FstState* s, FstNode* node) {
assert(s->state == OneTrans); assert(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { if (oSizes == 0) { return 0; }
return 0; FstSlice* slice = &node->data;
}
FstSlice *slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return unpackUint64(data + i, oSizes); return unpackUint64(data + i, oSizes);
} }
Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { if (oSizes == 0) { return 0; }
return 0; FstSlice* slice = &node->data;
} uint8_t* data = fstSliceData(slice, NULL);
FstSlice *slice = &node->data;
uint8_t * data = fstSliceData(slice, NULL);
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes;
...@@ -466,27 +450,23 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { ...@@ -466,27 +450,23 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
// anyTrans specify function // anyTrans specify function
void fstStateSetFinalState(FstState *s, bool yes) { void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (yes) { if (yes) { s->val |= 0b01000000; }
s->val |= 0b01000000;
}
return; return;
} }
bool fstStateIsFinalState(FstState *s) { bool fstStateIsFinalState(FstState* s) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
return (s->val & 0b01000000) == 0b01000000; return (s->val & 0b01000000) == 0b01000000;
} }
void fstStateSetStateNtrans(FstState *s, uint8_t n) { void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (n <= 0b00111111) { if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; }
s->val = (s->val & 0b11000000) | n;
}
return; return;
} }
// state_ntrans // state_ntrans
uint8_t fstStateStateNtrans(FstState *s, bool *null) { uint8_t fstStateStateNtrans(FstState* s, bool* null) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
*null = false; *null = false;
uint8_t n = s->val & 0b00111111; uint8_t n = s->val & 0b00111111;
...@@ -496,58 +476,52 @@ uint8_t fstStateStateNtrans(FstState *s, bool *null) { ...@@ -496,58 +476,52 @@ uint8_t fstStateStateNtrans(FstState *s, bool *null) {
} }
return n; return n;
} }
uint64_t fstStateTotalTransSize(FstState *s, uint64_t version, PackSizes sizes, uint64_t nTrans) { uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans);
return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize;
} }
uint64_t fstStateTransIndexSize(FstState *s, uint64_t version, uint64_t nTrans) { uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0;
} }
uint64_t fstStateNtransLen(FstState *s) { uint64_t fstStateNtransLen(FstState* s) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
bool null = false; bool null = false;
fstStateStateNtrans(s, &null); fstStateStateNtrans(s, &null);
return null == true ? 1 : 0; return null == true ? 1 : 0;
} }
uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
bool null = false; bool null = false;
uint8_t n = fstStateStateNtrans(s, &null); uint8_t n = fstStateStateNtrans(s, &null);
if (null != true) { if (null != true) { return n; }
return n;
}
int32_t len; int32_t len;
uint8_t *data = fstSliceData(slice, &len); uint8_t* data = fstSliceData(slice, &len);
n = data[len - 2]; n = data[len - 2];
// n = data[slice->end - 1]; // data[data.len() - 2] // n = data[slice->end - 1]; // data[data.len() - 2]
return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition,
// then it is encoded in the state byte // then it is encoded in the state byte
} }
Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
if (oSizes == 0 || !fstStateIsFinalState(s)) { if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; }
return 0;
}
uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size
- fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return unpackUint64(data + at, (uint8_t)oSizes); return unpackUint64(data + at, (uint8_t)oSizes);
} }
uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) {
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateTransIndexSize(s, node->version, node->nTrans); - fstStateTransIndexSize(s, node->version, node->nTrans);
int32_t dlen = 0; int32_t dlen = 0;
uint8_t *data = fstSliceData(slice, &dlen); uint8_t* data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b]; uint64_t i = data[at + b];
// uint64_t i = slice->data[slice->start + at + b]; // uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) { if (i >= node->nTrans) { *null = true; }
*null = true;
}
return i; return i;
} else { } else {
uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size
...@@ -555,7 +529,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { ...@@ -555,7 +529,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
uint64_t end = start + node->nTrans; uint64_t end = start + node->nTrans;
FstSlice t = fstSliceCopy(slice, start, end - 1); FstSlice t = fstSliceCopy(slice, start, end - 1);
int32_t len = 0; int32_t len = 0;
uint8_t *data = fstSliceData(&t, &len); uint8_t* data = fstSliceData(&t, &len);
int i = 0; int i = 0;
for (; i < len; i++) { for (; i < len; i++) {
uint8_t v = data[i]; uint8_t v = data[i];
...@@ -564,20 +538,16 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { ...@@ -564,20 +538,16 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
return node->nTrans - i - 1; // bug return node->nTrans - i - 1; // bug
} }
} }
if (i == len) { if (i == len) { *null = true; }
*null = true;
}
fstSliceDestroy(&t); fstSliceDestroy(&t);
} }
} }
// fst node function // fst node function
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) {
FstNode *n = (FstNode *)malloc(sizeof(FstNode)); FstNode* n = (FstNode*)malloc(sizeof(FstNode));
if (n == NULL) { if (n == NULL) { return NULL; }
return NULL;
}
FstState st = fstStateCreateFrom(slice, addr); FstState st = fstStateCreateFrom(slice, addr);
...@@ -625,27 +595,24 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { ...@@ -625,27 +595,24 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->isFinal = fstStateIsFinalState(&st); // s.is_final_state();
n->nTrans = nTrans; n->nTrans = nTrans;
n->sizes = sz; n->sizes = sz;
n->finalOutput = n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans);
fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans);
} }
return n; return n;
} }
// debug state transition // debug state transition
static const char *fstNodeState(FstNode *node) { static const char* fstNodeState(FstNode* node) {
FstState *st = &node->state; FstState* st = &node->state;
return fstStateStr[st->state]; return fstStateStr[st->state];
} }
void fstNodeDestroy(FstNode *node) { void fstNodeDestroy(FstNode* node) {
fstSliceDestroy(&node->data); fstSliceDestroy(&node->data);
free(node); free(node);
} }
FstTransitions *fstNodeTransitions(FstNode *node) { FstTransitions* fstNodeTransitions(FstNode* node) {
FstTransitions *t = malloc(sizeof(FstTransitions)); FstTransitions* t = malloc(sizeof(FstTransitions));
if (NULL == t) { if (NULL == t) { return NULL; }
return NULL;
}
FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; FstRange range = {.start = 0, .end = FST_NODE_LEN(node)};
t->range = range; t->range = range;
t->node = node; t->node = node;
...@@ -653,9 +620,9 @@ FstTransitions *fstNodeTransitions(FstNode *node) { ...@@ -653,9 +620,9 @@ FstTransitions *fstNodeTransitions(FstNode *node) {
} }
// Returns the transition at index `i`. // Returns the transition at index `i`.
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) {
bool s = true; bool s = true;
FstState *st = &node->state; FstState* st = &node->state;
if (st->state == OneTransNext) { if (st->state == OneTransNext) {
trn->inp = fstStateInput(st, node); trn->inp = fstStateInput(st, node);
trn->out = 0; trn->out = 0;
...@@ -675,9 +642,9 @@ bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { ...@@ -675,9 +642,9 @@ bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) {
} }
// Returns the transition address of the `i`th transition // Returns the transition address of the `i`th transition
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) {
bool s = true; bool s = true;
FstState *st = &node->state; FstState* st = &node->state;
if (st->state == OneTransNext) { if (st->state == OneTransNext) {
assert(i == 0); assert(i == 0);
fstStateTransAddr(st, node); fstStateTransAddr(st, node);
...@@ -696,9 +663,9 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { ...@@ -696,9 +663,9 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) {
// Finds the `i`th transition corresponding to the given input byte. // Finds the `i`th transition corresponding to the given input byte.
// If no transition for this byte exists, then `false` is returned. // If no transition for this byte exists, then `false` is returned.
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) {
bool s = true; bool s = true;
FstState *st = &node->state; FstState* st = &node->state;
if (st->state == OneTransNext) { if (st->state == OneTransNext) {
if (fstStateInput(st, node) == b) { if (fstStateInput(st, node) == b) {
*res = 0; *res = 0;
...@@ -723,7 +690,7 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { ...@@ -723,7 +690,7 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) {
return s; return s;
} }
bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode) { bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) {
size_t sz = taosArrayGetSize(builderNode->trans); size_t sz = taosArrayGetSize(builderNode->trans);
assert(sz < 256); assert(sz < 256);
if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) {
...@@ -732,7 +699,7 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr ...@@ -732,7 +699,7 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr
fstStateCompileForAnyTrans(w, addr, builderNode); fstStateCompileForAnyTrans(w, addr, builderNode);
// AnyTrans->Compile(w, addr, node); // AnyTrans->Compile(w, addr, node);
} else { } else {
FstTransition *tran = taosArrayGet(builderNode->trans, 0); FstTransition* tran = taosArrayGet(builderNode->trans, 0);
if (tran->addr == lastAddr && tran->out == 0) { if (tran->addr == lastAddr && tran->out == 0) {
fstStateCompileForOneTransNext(w, addr, tran->inp); fstStateCompileForOneTransNext(w, addr, tran->inp);
// OneTransNext::compile(w, lastAddr, tran->inp); // OneTransNext::compile(w, lastAddr, tran->inp);
...@@ -746,15 +713,13 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr ...@@ -746,15 +713,13 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr
return true; return true;
} }
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b);
} }
FstBuilder *fstBuilderCreate(void *w, FstType ty) { FstBuilder* fstBuilderCreate(void* w, FstType ty) {
FstBuilder *b = malloc(sizeof(FstBuilder)); FstBuilder* b = malloc(sizeof(FstBuilder));
if (NULL == b) { if (NULL == b) { return b; }
return b;
}
b->wrt = fstCountingWriterCreate(w); b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate(); b->unfinished = fstUnFinishedNodesCreate();
...@@ -764,7 +729,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { ...@@ -764,7 +729,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
b->len = 0; b->len = 0;
char buf64[8] = {0}; char buf64[8] = {0};
void *pBuf64 = buf64; void* pBuf64 = buf64;
taosEncodeFixedU64(&pBuf64, VERSION); taosEncodeFixedU64(&pBuf64, VERSION);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
...@@ -775,10 +740,8 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { ...@@ -775,10 +740,8 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
return b; return b;
} }
void fstBuilderDestroy(FstBuilder *b) { void fstBuilderDestroy(FstBuilder* b) {
if (b == NULL) { if (b == NULL) { return; }
return;
}
fstCountingWriterDestroy(b->wrt); fstCountingWriterDestroy(b->wrt);
fstUnFinishedNodesDestroy(b->unfinished); fstUnFinishedNodesDestroy(b->unfinished);
...@@ -787,19 +750,19 @@ void fstBuilderDestroy(FstBuilder *b) { ...@@ -787,19 +750,19 @@ void fstBuilderDestroy(FstBuilder *b) {
free(b); free(b);
} }
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) {
OrderType t = fstBuilderCheckLastKey(b, bs, true); OrderType t = fstBuilderCheckLastKey(b, bs, true);
if (t == Ordered) { if (t == Ordered) {
// add log info // add log info
fstBuilderInsertOutput(b, bs, in); fstBuilderInsertOutput(b, bs, in);
return true; return true;
} }
indexInfo("key must be ordered"); indexInfo("fst write key must be ordered");
return false; return false;
} }
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) {
FstSlice *s = &bs; FstSlice* s = &bs;
if (fstSliceIsEmpty(s)) { if (fstSliceIsEmpty(s)) {
b->len = 1; b->len = 1;
fstUnFinishedNodesSetRootOutput(b->unfinished, in); fstUnFinishedNodesSetRootOutput(b->unfinished, in);
...@@ -828,8 +791,8 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { ...@@ -828,8 +791,8 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) {
return; return;
} }
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) {
FstSlice *input = &bs; FstSlice* input = &bs;
if (fstSliceIsEmpty(&b->last)) { if (fstSliceIsEmpty(&b->last)) {
fstSliceDestroy(&b->last); fstSliceDestroy(&b->last);
// deep copy or not // deep copy or not
...@@ -847,10 +810,10 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { ...@@ -847,10 +810,10 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) {
} }
return Ordered; return Ordered;
} }
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
CompiledAddr addr = NONE_ADDRESS; CompiledAddr addr = NONE_ADDRESS;
while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) {
FstBuilderNode *bn = NULL; FstBuilderNode* bn = NULL;
if (addr == NONE_ADDRESS) { if (addr == NONE_ADDRESS) {
bn = fstUnFinishedNodesPopEmpty(b->unfinished); bn = fstUnFinishedNodesPopEmpty(b->unfinished);
} else { } else {
...@@ -865,11 +828,11 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { ...@@ -865,11 +828,11 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) {
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return; return;
} }
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
return EMPTY_ADDRESS; return EMPTY_ADDRESS;
} }
FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn);
if (entry->state == FOUND) { if (entry->state == FOUND) {
CompiledAddr ret = entry->addr; CompiledAddr ret = entry->addr;
fstRegistryEntryDestroy(entry); fstRegistryEntryDestroy(entry);
...@@ -879,23 +842,21 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { ...@@ -879,23 +842,21 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) {
fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);
b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);
if (entry->state == NOTFOUND) { if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); }
FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);
}
fstRegistryEntryDestroy(entry); fstRegistryEntryDestroy(entry);
return b->lastAddr; return b->lastAddr;
} }
void *fstBuilderInsertInner(FstBuilder *b) { void* fstBuilderInsertInner(FstBuilder* b) {
fstBuilderCompileFrom(b, 0); fstBuilderCompileFrom(b, 0);
FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished);
CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode);
fstBuilderNodeDestroy(rootNode); fstBuilderNodeDestroy(rootNode);
char buf64[8] = {0}; char buf64[8] = {0};
void *pBuf64 = buf64; void* pBuf64 = buf64;
taosEncodeFixedU64(&pBuf64, b->len); taosEncodeFixedU64(&pBuf64, b->len);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
...@@ -904,7 +865,7 @@ void *fstBuilderInsertInner(FstBuilder *b) { ...@@ -904,7 +865,7 @@ void *fstBuilderInsertInner(FstBuilder *b) {
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
char buf32[4] = {0}; char buf32[4] = {0};
void * pBuf32 = buf32; void* pBuf32 = buf32;
uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt);
taosEncodeFixedU32(&pBuf32, sum); taosEncodeFixedU32(&pBuf32, sum);
fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32));
...@@ -914,31 +875,31 @@ void *fstBuilderInsertInner(FstBuilder *b) { ...@@ -914,31 +875,31 @@ void *fstBuilderInsertInner(FstBuilder *b) {
// b->wrt = NULL; // b->wrt = NULL;
return b->wrt; return b->wrt;
} }
void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); } void fstBuilderFinish(FstBuilder* b) {
fstBuilderInsertInner(b);
}
FstSlice fstNodeAsSlice(FstNode *node) { FstSlice fstNodeAsSlice(FstNode* node) {
FstSlice *slice = &node->data; FstSlice* slice = &node->data;
FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1);
return s; return s;
} }
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
FstLastTransition *trn = malloc(sizeof(FstLastTransition)); FstLastTransition* trn = malloc(sizeof(FstLastTransition));
if (trn == NULL) { if (trn == NULL) { return NULL; }
return NULL;
}
trn->inp = inp; trn->inp = inp;
trn->out = out; trn->out = out;
return trn; return trn;
} }
void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); } void fstLastTransitionDestroy(FstLastTransition* trn) {
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { free(trn);
FstLastTransition *trn = unNode->last; }
if (trn == NULL) { void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
return; FstLastTransition* trn = unNode->last;
} if (trn == NULL) { return; }
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
taosArrayPush(unNode->node->trans, &t); taosArrayPush(unNode->node->trans, &t);
fstLastTransitionDestroy(trn); fstLastTransitionDestroy(trn);
...@@ -946,36 +907,28 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp ...@@ -946,36 +907,28 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp
return; return;
} }
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, Output out) { void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) {
if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; }
unNode->node->finalOutput += out;
}
size_t sz = taosArrayGetSize(unNode->node->trans); size_t sz = taosArrayGetSize(unNode->node->trans);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(unNode->node->trans, i); FstTransition* trn = taosArrayGet(unNode->node->trans, i);
trn->out += out; trn->out += out;
} }
if (unNode->last) { if (unNode->last) { unNode->last->out += out; }
unNode->last->out += out;
}
return; return;
} }
Fst *fstCreate(FstSlice *slice) { Fst* fstCreate(FstSlice* slice) {
int32_t slen; int32_t slen;
char * buf = fstSliceData(slice, &slen); char* buf = fstSliceData(slice, &slen);
if (slen < 36) { if (slen < 36) { return NULL; }
return NULL;
}
uint64_t len = slen; uint64_t len = slen;
uint64_t skip = 0; uint64_t skip = 0;
uint64_t version; uint64_t version;
taosDecodeFixedU64(buf, &version); taosDecodeFixedU64(buf, &version);
skip += sizeof(version); skip += sizeof(version);
if (version == 0 || version > VERSION) { if (version == 0 || version > VERSION) { return NULL; }
return NULL;
}
uint64_t type; uint64_t type;
taosDecodeFixedU64(buf + skip, &type); taosDecodeFixedU64(buf + skip, &type);
...@@ -993,15 +946,11 @@ Fst *fstCreate(FstSlice *slice) { ...@@ -993,15 +946,11 @@ Fst *fstCreate(FstSlice *slice) {
len -= sizeof(fstLen); len -= sizeof(fstLen);
taosDecodeFixedU64(buf + len, &fstLen); taosDecodeFixedU64(buf + len, &fstLen);
// TODO(validate root addr) // TODO(validate root addr)
Fst *fst = (Fst *)calloc(1, sizeof(Fst)); Fst* fst = (Fst*)calloc(1, sizeof(Fst));
if (fst == NULL) { if (fst == NULL) { return NULL; }
return NULL;
}
fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); fst->meta = (FstMeta*)malloc(sizeof(FstMeta));
if (NULL == fst->meta) { if (NULL == fst->meta) { goto FST_CREAT_FAILED; }
goto FST_CREAT_FAILED;
}
fst->meta->version = version; fst->meta->version = version;
fst->meta->rootAddr = rootAddr; fst->meta->rootAddr = rootAddr;
...@@ -1009,7 +958,7 @@ Fst *fstCreate(FstSlice *slice) { ...@@ -1009,7 +958,7 @@ Fst *fstCreate(FstSlice *slice) {
fst->meta->len = fstLen; fst->meta->len = fstLen;
fst->meta->checkSum = checkSum; fst->meta->checkSum = checkSum;
FstSlice *s = calloc(1, sizeof(FstSlice)); FstSlice* s = calloc(1, sizeof(FstSlice));
*s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice));
fst->data = s; fst->data = s;
...@@ -1019,7 +968,7 @@ FST_CREAT_FAILED: ...@@ -1019,7 +968,7 @@ FST_CREAT_FAILED:
free(fst->meta); free(fst->meta);
free(fst); free(fst);
} }
void fstDestroy(Fst *fst) { void fstDestroy(Fst* fst) {
if (fst) { if (fst) {
free(fst->meta); free(fst->meta);
fstSliceDestroy(fst->data); fstSliceDestroy(fst->data);
...@@ -1028,20 +977,18 @@ void fstDestroy(Fst *fst) { ...@@ -1028,20 +977,18 @@ void fstDestroy(Fst *fst) {
free(fst); free(fst);
} }
bool fstGet(Fst *fst, FstSlice *b, Output *out) { bool fstGet(Fst* fst, FstSlice* b, Output* out) {
FstNode *root = fstGetRoot(fst); FstNode* root = fstGetRoot(fst);
Output tOut = 0; Output tOut = 0;
int32_t len; int32_t len;
uint8_t *data = fstSliceData(b, &len); uint8_t* data = fstSliceData(b, &len);
SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*));
taosArrayPush(nodes, &root); taosArrayPush(nodes, &root);
for (uint32_t i = 0; i < len; i++) { for (uint32_t i = 0; i < len; i++) {
uint8_t inp = data[i]; uint8_t inp = data[i];
Output res = 0; Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) { if (false == fstNodeFindInput(root, inp, &res)) { return false; }
return false;
}
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(root, res, &trn); fstNodeGetTransitionAt(root, res, &trn);
...@@ -1056,7 +1003,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { ...@@ -1056,7 +1003,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
} }
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
FstNode **node = (FstNode **)taosArrayGet(nodes, i); FstNode** node = (FstNode**)taosArrayGet(nodes, i);
fstNodeDestroy(*node); fstNodeDestroy(*node);
} }
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
...@@ -1066,30 +1013,36 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { ...@@ -1066,30 +1013,36 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
return true; return true;
} }
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { return fstStreamBuilderCreate(fst, ctx);
if (sb == NULL) { }
return NULL; StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
} if (sb == NULL) { return NULL; }
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
} }
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
return fstStreamBuilderCreate(fst, ctx);
}
FstNode *fstGetRoot(Fst *fst) { FstNode* fstGetRoot(Fst* fst) {
if (fst->root != NULL) { if (fst->root != NULL) { return fst->root; }
return fst->root;
}
CompiledAddr rAddr = fstGetRootAddr(fst); CompiledAddr rAddr = fstGetRootAddr(fst);
fst->root = fstGetNode(fst, rAddr); fst->root = fstGetNode(fst, rAddr);
return fst->root; return fst->root;
} }
FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) {
FstType fstGetType(Fst *fst) { return fst->meta->ty; } return fstNodeCreate(fst->meta->version, addr, fst->data);
CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; } }
FstType fstGetType(Fst* fst) {
return fst->meta->ty;
}
CompiledAddr fstGetRootAddr(Fst* fst) {
return fst->meta->rootAddr;
}
Output fstEmptyFinalOutput(Fst *fst, bool *null) { Output fstEmptyFinalOutput(Fst* fst, bool* null) {
Output res = 0; Output res = 0;
FstNode *node = fstGetRoot(fst); FstNode* node = fstGetRoot(fst);
if (FST_NODE_IS_FINAL(node)) { if (FST_NODE_IS_FINAL(node)) {
*null = false; *null = false;
res = FST_NODE_FINAL_OUTPUT(node); res = FST_NODE_FINAL_OUTPUT(node);
...@@ -1099,23 +1052,19 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { ...@@ -1099,23 +1052,19 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) {
return res; return res;
} }
bool fstVerify(Fst *fst) { bool fstVerify(Fst* fst) {
uint32_t checkSum = fst->meta->checkSum; uint32_t checkSum = fst->meta->checkSum;
int32_t len; int32_t len;
uint8_t *data = fstSliceData(fst->data, &len); uint8_t* data = fstSliceData(fst->data, &len);
TSCKSUM initSum = 0; TSCKSUM initSum = 0;
if (!taosCheckChecksumWhole(data, len)) { if (!taosCheckChecksumWhole(data, len)) { return false; }
return false;
}
return true; return true;
} }
// data bound function // data bound function
FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) {
FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { if (b == NULL) { return NULL; }
return NULL;
}
if (data != NULL) { if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end); b->data = fstSliceCopy(data, data->start, data->end);
...@@ -1127,7 +1076,7 @@ FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { ...@@ -1127,7 +1076,7 @@ FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) {
return b; return b;
} }
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) {
int comp = fstSliceCompare(slice, &bound->data); int comp = fstSliceCompare(slice, &bound->data);
if (bound->type == Included) { if (bound->type == Included) {
return comp > 0 ? true : false; return comp > 0 ? true : false;
...@@ -1137,7 +1086,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { ...@@ -1137,7 +1086,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) {
return true; return true;
} }
} }
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) {
if (bound->type == Unbounded) { if (bound->type == Unbounded) {
return true; return true;
} else { } else {
...@@ -1145,34 +1094,33 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { ...@@ -1145,34 +1094,33 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
} }
} }
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; } bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) {
return bound->type == Excluded ? false : true;
}
void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } void fstBoundDestroy(FstBoundWithData* bound) {
free(bound);
}
StreamWithState *streamWithStateCreate( StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) {
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { StreamWithState* sws = calloc(1, sizeof(StreamWithState));
StreamWithState *sws = calloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; }
if (sws == NULL) {
return NULL;
}
sws->fst = fst; sws->fst = fst;
sws->aut = automation; sws->aut = automation;
sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t));
sws->emptyOutput.null = false; sws->emptyOutput.null = false;
sws->emptyOutput.out = 0; sws->emptyOutput.out = 0;
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
sws->endAt = max; sws->endAt = max;
streamWithStateSeekMin(sws, min); streamWithStateSeekMin(sws, min);
return sws; return sws;
} }
void streamWithStateDestroy(StreamWithState *sws) { void streamWithStateDestroy(StreamWithState* sws) {
if (sws == NULL) { if (sws == NULL) { return; }
return;
}
taosArrayDestroy(sws->inp); taosArrayDestroy(sws->inp);
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
...@@ -1180,22 +1128,19 @@ void streamWithStateDestroy(StreamWithState *sws) { ...@@ -1180,22 +1128,19 @@ void streamWithStateDestroy(StreamWithState *sws) {
free(sws); free(sws);
} }
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
AutomationCtx *aut = sws->aut; AutomationCtx* aut = sws->aut;
if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsEmpty(min)) {
if (fstBoundWithDataIsIncluded(min)) { if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); }
sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
}
StreamState s = {.node = fstGetRoot(sws->fst), StreamState s = {.node = fstGetRoot(sws->fst),
.trans = 0, .trans = 0,
.out = {.null = false, .out = 0}, .out = {.null = false, .out = 0},
.autState = automFuncs[aut->type].start(aut)}; // auto.start callback .autState = automFuncs[aut->type].start(aut)}; // auto.start callback
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
FstSlice *key = NULL; FstSlice* key = NULL;
bool inclusize = false; bool inclusize = false;
;
if (min->type == Included) { if (min->type == Included) {
key = &min->data; key = &min->data;
...@@ -1206,13 +1151,13 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1206,13 +1151,13 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
return false; return false;
} }
FstNode *node = fstGetRoot(sws->fst); FstNode* node = fstGetRoot(sws->fst);
Output out = 0; Output out = 0;
// void* autState = sws->aut->start(); // void* autState = sws->aut->start();
void *autState = automFuncs[aut->type].start(aut); void* autState = automFuncs[aut->type].start(aut);
int32_t len; int32_t len;
uint8_t *data = fstSliceData(key, &len); uint8_t* data = fstSliceData(key, &len);
for (uint32_t i = 0; i < len; i++) { for (uint32_t i = 0; i < len; i++) {
uint8_t b = data[i]; uint8_t b = data[i];
uint64_t res = 0; uint64_t res = 0;
...@@ -1220,7 +1165,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1220,7 +1165,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
if (null == false) { if (null == false) {
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(node, res, &trn); fstNodeGetTransitionAt(node, res, &trn);
void *preState = autState; void* preState = autState;
// autState = sws->aut->accept(preState, b); // autState = sws->aut->accept(preState, b);
autState = automFuncs[aut->type].accept(aut, preState, b); autState = automFuncs[aut->type].accept(aut, preState, b);
taosArrayPush(sws->inp, &b); taosArrayPush(sws->inp, &b);
...@@ -1235,13 +1180,11 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1235,13 +1180,11 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
// Since this is a minimum bound, we need to find the // Since this is a minimum bound, we need to find the
// first transition in this node that proceeds the current // first transition in this node that proceeds the current
// input byte. // input byte.
FstTransitions *trans = fstNodeTransitions(node); FstTransitions* trans = fstNodeTransitions(node);
uint64_t i = 0; uint64_t i = 0;
for (i = trans->range.start; i < trans->range.end; i++) { for (i = trans->range.start; i < trans->range.end; i++) {
FstTransition trn; FstTransition trn;
if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; }
break;
}
} }
StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
...@@ -1251,17 +1194,16 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1251,17 +1194,16 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
} }
uint32_t sz = taosArrayGetSize(sws->stack); uint32_t sz = taosArrayGetSize(sws->stack);
if (sz != 0) { if (sz != 0) {
StreamState *s = taosArrayGet(sws->stack, sz - 1); StreamState* s = taosArrayGet(sws->stack, sz - 1);
if (inclusize) { if (inclusize) {
s->trans -= 1; s->trans -= 1;
taosArrayPop(sws->inp); taosArrayPop(sws->inp);
} else { } else {
FstNode * n = s->node; FstNode* n = s->node;
uint64_t trans = s->trans; uint64_t trans = s->trans;
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(n, trans - 1, &trn); fstNodeGetTransitionAt(n, trans - 1, &trn);
StreamState s = { StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
...@@ -1269,48 +1211,44 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1269,48 +1211,44 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
} }
} }
StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
AutomationCtx *aut = sws->aut; AutomationCtx* aut = sws->aut;
FstOutput output = sws->emptyOutput; FstOutput output = sws->emptyOutput;
if (output.null == false) { if (output.null == false) {
FstSlice emptySlice = fstSliceCreate(NULL, 0); FstSlice emptySlice = fstSliceCreate(NULL, 0);
if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
return NULL; return NULL;
} }
void *start = automFuncs[aut->type].start(aut); void* start = automFuncs[aut->type].start(aut);
if (automFuncs[aut->type].isMatch(aut, start)) { if (automFuncs[aut->type].isMatch(aut, start)) {
FstSlice s = fstSliceCreate(NULL, 0); FstSlice s = fstSliceCreate(NULL, 0);
return swsResultCreate(&s, output, callback(start)); return swsResultCreate(&s, output, callback(start));
} }
} }
SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); SArray* nodes = taosArrayInit(8, sizeof(FstNode*));
while (taosArrayGetSize(sws->stack) > 0) { while (taosArrayGetSize(sws->stack) > 0) {
StreamState *p = (StreamState *)taosArrayPop(sws->stack); StreamState* p = (StreamState*)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); }
taosArrayPop(sws->inp);
}
streamStateDestroy(p); streamStateDestroy(p);
continue; continue;
} }
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(p->node, p->trans, &trn); fstNodeGetTransitionAt(p->node, p->trans, &trn);
Output out = p->out.out + trn.out; Output out = p->out.out + trn.out;
void * nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp);
void * tState = callback(nextState); void* tState = callback(nextState);
bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState);
FstNode *nextNode = fstGetNode(sws->fst, trn.addr); FstNode* nextNode = fstGetNode(sws->fst, trn.addr);
taosArrayPush(nodes, &nextNode); taosArrayPush(nodes, &nextNode);
taosArrayPush(sws->inp, &(trn.inp)); taosArrayPush(sws->inp, &(trn.inp));
if (FST_NODE_IS_FINAL(nextNode)) { if (FST_NODE_IS_FINAL(nextNode)) {
// void *eofState = sws->aut->acceptEof(nextState); // void *eofState = sws->aut->acceptEof(nextState);
void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
if (eofState != NULL) { if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); }
isMatch = automFuncs[aut->type].isMatch(aut, eofState);
}
} }
StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
taosArrayPush(sws->stack, &s1); taosArrayPush(sws->stack, &s1);
...@@ -1319,21 +1257,21 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb ...@@ -1319,21 +1257,21 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
taosArrayPush(sws->stack, &s2); taosArrayPush(sws->stack, &s2);
size_t isz = taosArrayGetSize(sws->inp); size_t isz = taosArrayGetSize(sws->inp);
uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t));
for (uint32_t i = 0; i < isz; i++) { for (uint32_t i = 0; i < isz; i++) {
buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i);
} }
FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp));
if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
free(buf); free(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
return NULL; return NULL;
} }
if (FST_NODE_IS_FINAL(nextNode) && isMatch) { if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
free(buf); free(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
return result; return result;
...@@ -1342,18 +1280,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb ...@@ -1342,18 +1280,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
} }
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
FstNode **node = (FstNode **)taosArrayGet(nodes, i); FstNode** node = (FstNode**)taosArrayGet(nodes, i);
fstNodeDestroy(*node); fstNodeDestroy(*node);
} }
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
return NULL; return NULL;
} }
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { if (result == NULL) { return NULL; }
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut; result->out = fOut;
...@@ -1361,31 +1297,25 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta ...@@ -1361,31 +1297,25 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
return result; return result;
} }
void swsResultDestroy(StreamWithStateResult *result) { void swsResultDestroy(StreamWithStateResult* result) {
if (NULL == result) { if (NULL == result) { return; }
return;
}
fstSliceDestroy(&result->data); fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state); startWithStateValueDestroy(result->state);
free(result); free(result);
} }
void streamStateDestroy(void *s) { void streamStateDestroy(void* s) {
if (NULL == s) { if (NULL == s) { return; }
return; StreamState* ss = (StreamState*)s;
}
StreamState *ss = (StreamState *)s;
fstNodeDestroy(ss->node); fstNodeDestroy(ss->node);
// free(s->autoState); // free(s->autoState);
} }
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { if (NULL == b) { return NULL; }
return NULL;
}
b->fst = fst; b->fst = fst;
b->aut = aut; b->aut = aut;
...@@ -1393,17 +1323,15 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { ...@@ -1393,17 +1323,15 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) {
b->max = fstBoundStateCreate(Unbounded, NULL); b->max = fstBoundStateCreate(Unbounded, NULL);
return b; return b;
} }
void fstStreamBuilderDestroy(FstStreamBuilder *b) { void fstStreamBuilderDestroy(FstStreamBuilder* b) {
fstSliceDestroy(&b->min->data); fstSliceDestroy(&b->min->data);
tfree(b->min); tfree(b->min);
fstSliceDestroy(&b->max->data); fstSliceDestroy(&b->max->data);
tfree(b->max); tfree(b->max);
free(b); free(b);
} }
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
if (b == NULL) { if (b == NULL) { return NULL; }
return NULL;
}
if (type == GE) { if (type == GE) {
b->min->type = Included; b->min->type = Included;
......
source/libs/index/src/index_fst_automation.c
浏览文件 @ 1e32e1f8
...@@ -15,19 +15,17 @@ ...@@ -15,19 +15,17 @@
#include "index_fst_automation.h" #include "index_fst_automation.h"
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { if (nsv == NULL) { return NULL; }
return NULL;
}
nsv->kind = kind; nsv->kind = kind;
nsv->type = ty; nsv->type = ty;
if (ty == FST_INT) { if (ty == FST_INT) {
nsv->val = *(int *)val; nsv->val = *(int*)val;
} else if (ty == FST_CHAR) { } else if (ty == FST_CHAR) {
size_t len = strlen((char *)val); size_t len = strlen((char*)val);
nsv->ptr = (char *)calloc(1, len + 1); nsv->ptr = (char*)calloc(1, len + 1);
memcpy(nsv->ptr, val, len); memcpy(nsv->ptr, val, len);
} else if (ty == FST_ARRAY) { } else if (ty == FST_ARRAY) {
// TODO, // TODO,
...@@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp ...@@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
} }
return nsv; return nsv;
} }
void startWithStateValueDestroy(void *val) { void startWithStateValueDestroy(void* val) {
StartWithStateValue *sv = (StartWithStateValue *)val; StartWithStateValue* sv = (StartWithStateValue*)val;
if (sv == NULL) { if (sv == NULL) { return; }
return;
}
if (sv->type == FST_INT) { if (sv->type == FST_INT) {
// //
...@@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) { ...@@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) {
} }
free(sv); free(sv);
} }
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { if (nsv == NULL) { return NULL; }
return NULL;
}
nsv->kind = sv->kind; nsv->kind = sv->kind;
nsv->type = sv->type; nsv->type = sv->type;
...@@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { ...@@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
nsv->val = sv->val; nsv->val = sv->val;
} else if (nsv->type == FST_CHAR) { } else if (nsv->type == FST_CHAR) {
size_t len = strlen(sv->ptr); size_t len = strlen(sv->ptr);
nsv->ptr = (char *)calloc(1, len + 1); nsv->ptr = (char*)calloc(1, len + 1);
memcpy(nsv->ptr, sv->ptr, len); memcpy(nsv->ptr, sv->ptr, len);
} else if (nsv->type == FST_ARRAY) { } else if (nsv->type == FST_ARRAY) {
//
} }
return nsv; return nsv;
} }
// prefix query, impl later // prefix query, impl later
static void *prefixStart(AutomationCtx *ctx) { static void* prefixStart(AutomationCtx* ctx) {
StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata);
return startWithStateValueDump(data); return startWithStateValueDump(data);
}; };
static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
StartWithStateValue *ssv = (StartWithStateValue *)sv; StartWithStateValue* ssv = (StartWithStateValue*)sv;
return ssv->val == strlen(ctx->data); return ssv->val == strlen(ctx->data);
} }
static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
StartWithStateValue *ssv = (StartWithStateValue *)sv; StartWithStateValue* ssv = (StartWithStateValue*)sv;
return ssv->val >= 0; return ssv->val >= 0;
} }
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) {
static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return true;
StartWithStateValue *ssv = (StartWithStateValue *)state; }
if (ssv == NULL || ctx == NULL) { static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
return NULL; StartWithStateValue* ssv = (StartWithStateValue*)state;
} if (ssv == NULL || ctx == NULL) { return NULL; }
char *data = ctx->data; char* data = ctx->data;
if (ssv->kind == Done) { if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); }
return startWithStateValueCreate(Done, FST_INT, &ssv->val);
}
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1; int val = ssv->val + 1;
StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val);
if (prefixIsMatch(ctx, nsv)) { if (prefixIsMatch(ctx, nsv)) {
nsv->kind = Done; nsv->kind = Done;
} else { } else {
...@@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { ...@@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
} }
return NULL; return NULL;
} }
static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } static void* prefixAcceptEof(AutomationCtx* ctx, void* state) {
return NULL;
}
// pattern query, impl later // pattern query, impl later
static void *patternStart(AutomationCtx *ctx) { return NULL; } static void* patternStart(AutomationCtx* ctx) {
static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; } return NULL;
static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; } }
static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } static bool patternIsMatch(AutomationCtx* ctx, void* data) {
return true;
}
static bool patternCanMatch(AutomationCtx* ctx, void* data) {
return true;
}
static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) {
return true;
}
static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; } static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
return NULL;
}
static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } static void* patternAcceptEof(AutomationCtx* ctx, void* state) {
return NULL;
}
AutomationFunc automFuncs[] = { AutomationFunc automFuncs[] = {
{prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof},
...@@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = { ...@@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = {
// add more search type // add more search type
}; };
AutomationCtx *automCtxCreate(void *data, AutomationType atype) { AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { if (ctx == NULL) { return NULL; }
return NULL;
}
StartWithStateValue *sv = NULL; StartWithStateValue* sv = NULL;
if (atype == AUTOMATION_PREFIX) { if (atype == AUTOMATION_PREFIX) {
int val = 0; int val = 0;
sv = startWithStateValueCreate(Running, FST_INT, &val); sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->stdata = (void *)sv; ctx->stdata = (void*)sv;
} else if (atype == AUTMMATION_MATCH) { } else if (atype == AUTMMATION_MATCH) {
} else { } else {
// add more search type // add more search type
} }
char * src = (char *)data; char* src = (char*)data;
size_t len = strlen(src); size_t len = strlen(src);
char * dst = (char *)malloc(len * sizeof(char) + 1); char* dst = (char*)malloc(len * sizeof(char) + 1);
memcpy(dst, src, len); memcpy(dst, src, len);
dst[len] = 0; dst[len] = 0;
ctx->data = dst; ctx->data = dst;
ctx->type = atype; ctx->type = atype;
ctx->stdata = (void *)sv; ctx->stdata = (void*)sv;
return ctx; return ctx;
} }
void automCtxDestroy(AutomationCtx *ctx) { void automCtxDestroy(AutomationCtx* ctx) {
startWithStateValueDestroy(ctx->stdata); startWithStateValueDestroy(ctx->stdata);
free(ctx->data); free(ctx->data);
free(ctx); free(ctx);
......
source/libs/index/src/index_fst_common.c
浏览文件 @ 1e32e1f8
...@@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = { ...@@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = {
}; };
const char COMMON_INPUTS_INV[] = { const char COMMON_INPUTS_INV[] = {
't', 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', '.', 'h', 'l', 'm',
'e', '-', 'd', 'u', '0', '1', '2', 'g', '=', ':', 'b', 'f', '3', 'y', '5', '&', '_',
'/', '4', 'v', '9', '6', '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I',
'o', 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'q', 'H', 'G', 'W',
'a', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 'Q', ';', ')', '(', '~', '[', ']', '$',
's', '!', '\'', '*', '@', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b',
'r', '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b',
'i', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', '#', '<', '>', '\\', '^', '`', '{', '|', '}', '\x7f',
'p', '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f',
'c', '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f',
'n', '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf',
'w', '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf',
'.', '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf',
'h', '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf',
'l', '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef',
'm', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff',
'-',
'd',
'u',
'0',
'1',
'2',
'g',
'=',
':',
'b',
'f',
'3',
'y',
'5',
'&',
'_',
'4',
'v',
'9',
'6',
'7',
'8',
'k',
'%',
'?',
'x',
'C',
'D',
'A',
'S',
'F',
'I',
'B',
'E',
'j',
'P',
'T',
'z',
'R',
'N',
'M',
'+',
'L',
'O',
'q',
'H',
'G',
'W',
'U',
'V',
',',
'Y',
'K',
'J',
'Z',
'X',
'Q',
';',
')',
'(',
'~',
'[',
']',
'$',
'!',
'\'',
'*',
'@',
'\x00',
'\x01',
'\x02',
'\x03',
'\x04',
'\x05',
'\x06',
'\x07',
'\x08',
'\t',
'\n',
'\x0b',
'\x0c',
'\r',
'\x0e',
'\x0f',
'\x10',
'\x11',
'\x12',
'\x13',
'\x14',
'\x15',
'\x16',
'\x17',
'\x18',
'\x19',
'\x1a',
'\x1b',
'\x1c',
'\x1d',
'\x1e',
'\x1f',
' ',
'"',
'#',
'<',
'>',
'\\',
'^',
'`',
'{',
'|',
'}',
'\x7f',
'\x80',
'\x81',
'\x82',
'\x83',
'\x84',
'\x85',
'\x86',
'\x87',
'\x88',
'\x89',
'\x8a',
'\x8b',
'\x8c',
'\x8d',
'\x8e',
'\x8f',
'\x90',
'\x91',
'\x92',
'\x93',
'\x94',
'\x95',
'\x96',
'\x97',
'\x98',
'\x99',
'\x9a',
'\x9b',
'\x9c',
'\x9d',
'\x9e',
'\x9f',
'\xa0',
'\xa1',
'\xa2',
'\xa3',
'\xa4',
'\xa5',
'\xa6',
'\xa7',
'\xa8',
'\xa9',
'\xaa',
'\xab',
'\xac',
'\xad',
'\xae',
'\xaf',
'\xb0',
'\xb1',
'\xb2',
'\xb3',
'\xb4',
'\xb5',
'\xb6',
'\xb7',
'\xb8',
'\xb9',
'\xba',
'\xbb',
'\xbc',
'\xbd',
'\xbe',
'\xbf',
'\xc0',
'\xc1',
'\xc2',
'\xc3',
'\xc4',
'\xc5',
'\xc6',
'\xc7',
'\xc8',
'\xc9',
'\xca',
'\xcb',
'\xcc',
'\xcd',
'\xce',
'\xcf',
'\xd0',
'\xd1',
'\xd2',
'\xd3',
'\xd4',
'\xd5',
'\xd6',
'\xd7',
'\xd8',
'\xd9',
'\xda',
'\xdb',
'\xdc',
'\xdd',
'\xde',
'\xdf',
'\xe0',
'\xe1',
'\xe2',
'\xe3',
'\xe4',
'\xe5',
'\xe6',
'\xe7',
'\xe8',
'\xe9',
'\xea',
'\xeb',
'\xec',
'\xed',
'\xee',
'\xef',
'\xf0',
'\xf1',
'\xf2',
'\xf3',
'\xf4',
'\xf5',
'\xf6',
'\xf7',
'\xf8',
'\xf9',
'\xfa',
'\xfb',
'\xfc',
'\xfd',
'\xfe',
'\xff',
}; };
source/libs/index/src/index_fst_counting_writer.c
浏览文件 @ 1e32e1f8
...@@ -17,10 +17,8 @@ ...@@ -17,10 +17,8 @@
#include "index_fst_util.h" #include "index_fst_util.h"
#include "tutil.h" #include "tutil.h"
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->offset + len > ctx->limit) { if (ctx->offset + len > ctx->limit) { return -1; }
return -1;
}
if (ctx->type == TFile) { if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len)); assert(len == tfWrite(ctx->file.fd, buf, len));
...@@ -30,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { ...@@ -30,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
ctx->offset += len; ctx->offset += len;
return len; return len;
} }
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
int nRead = 0; int nRead = 0;
if (ctx->type == TFile) { if (ctx->type == TFile) {
nRead = tfRead(ctx->file.fd, buf, len); nRead = tfRead(ctx->file.fd, buf, len);
...@@ -41,7 +39,7 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { ...@@ -41,7 +39,7 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
return nRead; return nRead;
} }
static int writeCtxDoFlush(WriterCtx *ctx) { static int writeCtxDoFlush(WriterCtx* ctx) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
// tfFsync(ctx->fd); // tfFsync(ctx->fd);
// tfFlush(ctx->file.fd); // tfFlush(ctx->file.fd);
...@@ -51,11 +49,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) { ...@@ -51,11 +49,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) {
return 1; return 1;
} }
WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); WriterCtx* ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { if (ctx == NULL) { return NULL; }
return NULL;
}
ctx->type = type; ctx->type = type;
if (ctx->type == TFile) { if (ctx->type == TFile) {
...@@ -67,8 +63,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ...@@ -67,8 +63,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
ctx->file.fd = tfOpenReadWrite(tmpFile); ctx->file.fd = tfOpenReadWrite(tmpFile);
} }
if (ctx->file.fd < 0) { if (ctx->file.fd < 0) {
goto END;
indexError("open file error %d", errno); indexError("open file error %d", errno);
goto END;
} }
} else if (ctx->type == TMemory) { } else if (ctx->type == TMemory) {
ctx->mem.buf = calloc(1, sizeof(char) * capacity); ctx->mem.buf = calloc(1, sizeof(char) * capacity);
...@@ -83,12 +79,10 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int ...@@ -83,12 +79,10 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
return ctx; return ctx;
END: END:
if (ctx->type == TMemory) { if (ctx->type == TMemory) { free(ctx->mem.buf); }
free(ctx->mem.buf);
}
free(ctx); free(ctx);
} }
void writerCtxDestroy(WriterCtx *ctx) { void writerCtxDestroy(WriterCtx* ctx) {
if (ctx->type == TMemory) { if (ctx->type == TMemory) {
free(ctx->mem.buf); free(ctx->mem.buf);
} else { } else {
...@@ -97,57 +91,53 @@ void writerCtxDestroy(WriterCtx *ctx) { ...@@ -97,57 +91,53 @@ void writerCtxDestroy(WriterCtx *ctx) {
free(ctx); free(ctx);
} }
FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter* fstCountingWriterCreate(void* wrt) {
FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { if (cw == NULL) { return NULL; }
return NULL;
}
cw->wrt = wrt; cw->wrt = wrt;
//(void *)(writerCtxCreate(TFile, readOnly)); //(void *)(writerCtxCreate(TFile, readOnly));
return cw; return cw;
} }
void fstCountingWriterDestroy(FstCountingWriter *cw) { void fstCountingWriterDestroy(FstCountingWriter* cw) {
// free wrt object: close fd or free mem // free wrt object: close fd or free mem
fstCountingWriterFlush(cw); fstCountingWriterFlush(cw);
// writerCtxDestroy((WriterCtx *)(cw->wrt)); // writerCtxDestroy((WriterCtx *)(cw->wrt));
free(cw); free(cw);
} }
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
if (write == NULL) { if (write == NULL) { return 0; }
return 0;
}
// update checksum // update checksum
// write data to file/socket or mem // write data to file/socket or mem
WriterCtx *ctx = write->wrt; WriterCtx* ctx = write->wrt;
int nWrite = ctx->write(ctx, buf, len); int nWrite = ctx->write(ctx, buf, len);
assert(nWrite == len); assert(nWrite == len);
write->count += len; write->count += len;
return len; return len;
} }
int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
if (write == NULL) { if (write == NULL) { return 0; }
return 0; WriterCtx* ctx = write->wrt;
}
WriterCtx *ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len); int nRead = ctx->read(ctx, buf, len);
// assert(nRead == len); // assert(nRead == len);
return nRead; return nRead;
} }
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) {
int fstCountingWriterFlush(FstCountingWriter *write) { return 0;
WriterCtx *ctx = write->wrt; }
int fstCountingWriterFlush(FstCountingWriter* write) {
WriterCtx* ctx = write->wrt;
ctx->flush(ctx); ctx->flush(ctx);
// write->wtr->flush // write->wtr->flush
return 1; return 1;
} }
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) {
assert(1 <= nBytes && nBytes <= 8); assert(1 <= nBytes && nBytes <= 8);
uint8_t *buf = calloc(8, sizeof(uint8_t)); uint8_t* buf = calloc(8, sizeof(uint8_t));
for (uint8_t i = 0; i < nBytes; i++) { for (uint8_t i = 0; i < nBytes; i++) {
buf[i] = (uint8_t)n; buf[i] = (uint8_t)n;
n = n >> 8; n = n >> 8;
...@@ -157,7 +147,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t ...@@ -157,7 +147,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t
return; return;
} }
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) {
uint8_t nBytes = packSize(n); uint8_t nBytes = packSize(n);
fstCountingWriterPackUintIn(writer, n, nBytes); fstCountingWriterPackUintIn(writer, n, nBytes);
return nBytes; return nBytes;
......
source/libs/index/src/index_fst_node.c
浏览文件 @ 1e32e1f8
...@@ -14,60 +14,46 @@ ...@@ -14,60 +14,46 @@
*/ */
#include "index_fst_node.h" #include "index_fst_node.h"
FstBuilderNode *fstBuilderNodeDefault() { FstBuilderNode* fstBuilderNodeDefault() {
FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); FstBuilderNode* bn = malloc(sizeof(FstBuilderNode));
bn->isFinal = false; bn->isFinal = false;
bn->finalOutput = 0; bn->finalOutput = 0;
bn->trans = taosArrayInit(16, sizeof(FstTransition)); bn->trans = taosArrayInit(16, sizeof(FstTransition));
return bn; return bn;
} }
void fstBuilderNodeDestroy(FstBuilderNode *node) { void fstBuilderNodeDestroy(FstBuilderNode* node) {
if (node == NULL) { if (node == NULL) { return; }
return;
}
taosArrayDestroy(node->trans); taosArrayDestroy(node->trans);
free(node); free(node);
} }
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) {
if (n1 == n2) { if (n1 == n2) { return true; }
return true; if (n1 == NULL || n2 == NULL) { return false; }
}
if (n1 == NULL || n2 == NULL) {
return false;
}
if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; }
return false;
}
size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0;
size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0;
if (s1 != s2) { if (s1 != s2) { return false; }
return false;
}
for (size_t i = 0; i < s1; i++) { for (size_t i = 0; i < s1; i++) {
FstTransition *t1 = taosArrayGet(n1->trans, i); FstTransition* t1 = taosArrayGet(n1->trans, i);
FstTransition *t2 = taosArrayGet(n2->trans, i); FstTransition* t2 = taosArrayGet(n2->trans, i);
if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; }
return false;
}
} }
return true; return true;
} }
FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); FstBuilderNode* node = malloc(sizeof(FstBuilderNode));
if (node == NULL) { if (node == NULL) { return NULL; }
return NULL;
}
// //
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); SArray* trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *tran = taosArrayGet(src->trans, i); FstTransition* tran = taosArrayGet(src->trans, i);
taosArrayPush(trans, tran); taosArrayPush(trans, tran);
} }
...@@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { ...@@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
return node; return node;
} }
// not destroy src, User's bussiness // not destroy src, User's bussiness
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
if (dst == NULL || src == NULL) { if (dst == NULL || src == NULL) { return; }
return;
}
dst->isFinal = src->isFinal; dst->isFinal = src->isFinal;
dst->finalOutput = src->finalOutput; dst->finalOutput = src->finalOutput;
...@@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { ...@@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
dst->trans = taosArrayInit(sz, sizeof(FstTransition)); dst->trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(src->trans, i); FstTransition* trn = taosArrayGet(src->trans, i);
taosArrayPush(dst->trans, trn); taosArrayPush(dst->trans, trn);
} }
} }
......
source/libs/index/src/index_fst_registry.c
浏览文件 @ 1e32e1f8
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
#include "index_fst_registry.h" #include "index_fst_registry.h"
uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) {
// TODO(yihaoDeng): refactor later // TODO(yihaoDeng): refactor later
const uint64_t FNV_PRIME = 1099511628211; const uint64_t FNV_PRIME = 1099511628211;
uint64_t h = 14695981039346656037u; uint64_t h = 14695981039346656037u;
...@@ -25,21 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { ...@@ -25,21 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans);
for (uint32_t i = 0; i < sz; i++) { for (uint32_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(bNode->trans, i); FstTransition* trn = taosArrayGet(bNode->trans, i);
h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME;
h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME;
h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME;
} }
return h % (registry->tableSize); return h % (registry->tableSize);
} }
static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (a >= sz || b >= sz) { if (a >= sz || b >= sz) { return; }
return;
}
FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a);
FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b);
FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node};
...@@ -51,11 +49,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { ...@@ -51,11 +49,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
return; return;
} }
static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (start >= sz && end >= sz) { if (start >= sz && end >= sz) { return; }
return;
}
assert(start >= end); assert(start >= end);
...@@ -67,14 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { ...@@ -67,14 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
} }
} }
FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
FstRegistry *registry = malloc(sizeof(FstRegistry)); FstRegistry* registry = malloc(sizeof(FstRegistry));
if (registry == NULL) { if (registry == NULL) { return NULL; }
return NULL;
}
uint64_t nCells = tableSize * mruSize; uint64_t nCells = tableSize * mruSize;
SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell));
if (NULL == tb) { if (NULL == tb) {
free(registry); free(registry);
return NULL; return NULL;
...@@ -91,32 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { ...@@ -91,32 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
return registry; return registry;
} }
void fstRegistryDestroy(FstRegistry *registry) { void fstRegistryDestroy(FstRegistry* registry) {
if (registry == NULL) { if (registry == NULL) { return; }
return;
}
SArray *tb = registry->table; SArray* tb = registry->table;
size_t sz = taosArrayGetSize(tb); size_t sz = taosArrayGetSize(tb);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstRegistryCell *cell = taosArrayGet(tb, i); FstRegistryCell* cell = taosArrayGet(tb, i);
fstBuilderNodeDestroy(cell->node); fstBuilderNodeDestroy(cell->node);
} }
taosArrayDestroy(tb); taosArrayDestroy(tb);
free(registry); free(registry);
} }
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) {
if (taosArrayGetSize(registry->table) <= 0) { if (taosArrayGetSize(registry->table) <= 0) { return NULL; }
return NULL;
}
uint64_t bucket = fstRegistryHash(registry, bNode); uint64_t bucket = fstRegistryHash(registry, bNode);
uint64_t start = registry->mruSize * bucket; uint64_t start = registry->mruSize * bucket;
uint64_t end = start + registry->mruSize; uint64_t end = start + registry->mruSize;
FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry));
if (end - start == 1) { if (end - start == 1) {
FstRegistryCell *cell = taosArrayGet(registry->table, start); FstRegistryCell* cell = taosArrayGet(registry->table, start);
// cell->isNode && // cell->isNode &&
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
...@@ -128,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo ...@@ -128,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
entry->cell = cell; // copy or not entry->cell = cell; // copy or not
} }
} else if (end - start == 2) { } else if (end - start == 2) {
FstRegistryCell *cell1 = taosArrayGet(registry->table, start); FstRegistryCell* cell1 = taosArrayGet(registry->table, start);
if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell1->addr; entry->addr = cell1->addr;
return entry; return entry;
} }
FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); FstRegistryCell* cell2 = taosArrayGet(registry->table, start + 1);
if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell2->addr; entry->addr = cell2->addr;
...@@ -146,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo ...@@ -146,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
fstBuilderNodeCloneFrom(cell2->node, bNode); fstBuilderNodeCloneFrom(cell2->node, bNode);
fstRegistryCellSwap(registry->table, start, start + 1); fstRegistryCellSwap(registry->table, start, start + 1);
FstRegistryCell *cCell = taosArrayGet(registry->table, start); FstRegistryCell* cCell = taosArrayGet(registry->table, start);
entry->state = NOTFOUND; entry->state = NOTFOUND;
entry->cell = cCell; entry->cell = cCell;
} else { } else {
uint32_t i = start; uint32_t i = start;
for (; i < end; i++) { for (; i < end; i++) {
FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, i);
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell->addr; entry->addr = cell->addr;
...@@ -162,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo ...@@ -162,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
} }
if (i >= end) { if (i >= end) {
uint64_t last = end - 1; uint64_t last = end - 1;
FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last);
// clone from bNode, refactor later // clone from bNode, refactor later
fstBuilderNodeCloneFrom(cell->node, bNode); fstBuilderNodeCloneFrom(cell->node, bNode);
fstRegistryCellPromote(registry->table, last, start); fstRegistryCellPromote(registry->table, last, start);
FstRegistryCell *cCell = taosArrayGet(registry->table, start); FstRegistryCell* cCell = taosArrayGet(registry->table, start);
entry->state = NOTFOUND; entry->state = NOTFOUND;
entry->cell = cCell; entry->cell = cCell;
} }
} }
return entry; return entry;
} }
void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); } void fstRegistryEntryDestroy(FstRegistryEntry* entry) {
free(entry);
}
source/libs/index/src/index_fst_util.c
浏览文件 @ 1e32e1f8
...@@ -61,9 +61,10 @@ uint8_t packSize(uint64_t n) { ...@@ -61,9 +61,10 @@ uint8_t packSize(uint64_t n) {
} }
} }
uint64_t unpackUint64(uint8_t *ch, uint8_t sz) { uint64_t unpackUint64(uint8_t* ch, uint8_t sz) {
uint64_t n = 0; uint64_t n = 0;
for (uint8_t i = 0; i < sz; i++) { for (uint8_t i = 0; i < sz; i++) {
//
n = n | (ch[i] << (8 * i)); n = n | (ch[i] << (8 * i));
} }
return n; return n;
...@@ -75,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { ...@@ -75,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) {
return packSize(nodeAddr - transAddr); return packSize(nodeAddr - transAddr);
} }
} }
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) {
uint64_t delta = unpackUint64(data, len); uint64_t delta = unpackUint64(data, len);
// delta_add = u64_to_usize // delta_add = u64_to_usize
if (delta == EMPTY_ADDRESS) { if (delta == EMPTY_ADDRESS) {
...@@ -88,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { ...@@ -88,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
// fst slice func // fst slice func
// //
FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
FstString *str = (FstString *)malloc(sizeof(FstString)); FstString* str = (FstString*)malloc(sizeof(FstString));
str->ref = 1; str->ref = 1;
str->len = len; str->len = len;
str->data = malloc(len * sizeof(uint8_t)); str->data = malloc(len * sizeof(uint8_t));
...@@ -99,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { ...@@ -99,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) {
return s; return s;
} }
// just shallow copy // just shallow copy
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
FstString *str = s->str; FstString* str = s->str;
str->ref++; str->ref++;
// uint8_t *buf = fstSliceData(s, &alen); // uint8_t *buf = fstSliceData(s, &alen);
// start = buf + start - (buf - s->start); // start = buf + start - (buf - s->start);
...@@ -109,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { ...@@ -109,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
return t; return t;
} }
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
int32_t tlen = end - start + 1; int32_t tlen = end - start + 1;
int32_t slen; int32_t slen;
uint8_t *data = fstSliceData(s, &slen); uint8_t* data = fstSliceData(s, &slen);
assert(tlen <= slen); assert(tlen <= slen);
uint8_t *buf = malloc(sizeof(uint8_t) * tlen); uint8_t* buf = malloc(sizeof(uint8_t) * tlen);
memcpy(buf, data + start, tlen); memcpy(buf, data + start, tlen);
FstString *str = malloc(sizeof(FstString)); FstString* str = malloc(sizeof(FstString));
str->data = buf; str->data = buf;
str->len = tlen; str->len = tlen;
str->ref = 1; str->ref = 1;
...@@ -129,17 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { ...@@ -129,17 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
ans.end = tlen - 1; ans.end = tlen - 1;
return ans; return ans;
} }
bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } bool fstSliceIsEmpty(FstSlice* s) {
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
uint8_t *fstSliceData(FstSlice *s, int32_t *size) { uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
FstString *str = s->str; FstString* str = s->str;
if (size != NULL) { if (size != NULL) { *size = s->end - s->start + 1; }
*size = s->end - s->start + 1;
}
return str->data + s->start; return str->data + s->start;
} }
void fstSliceDestroy(FstSlice *s) { void fstSliceDestroy(FstSlice* s) {
FstString *str = s->str; FstString* str = s->str;
str->ref--; str->ref--;
if (str->ref <= 0) { if (str->ref <= 0) {
free(str->data); free(str->data);
...@@ -148,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) { ...@@ -148,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) {
} }
} }
int fstSliceCompare(FstSlice *a, FstSlice *b) { int fstSliceCompare(FstSlice* a, FstSlice* b) {
int32_t alen, blen; int32_t alen, blen;
uint8_t *aBuf = fstSliceData(a, &alen); uint8_t* aBuf = fstSliceData(a, &alen);
uint8_t *bBuf = fstSliceData(b, &blen); uint8_t* bBuf = fstSliceData(b, &blen);
uint32_t i, j; uint32_t i, j;
for (i = 0, j = 0; i < alen && j < blen; i++, j++) { for (i = 0, j = 0; i < alen && j < blen; i++, j++) {
......
source/libs/index/src/index_tfile.c
浏览文件 @ 1e32e1f8
...@@ -21,15 +21,74 @@ ...@@ -21,15 +21,74 @@
#include "index_fst_counting_writer.h" #include "index_fst_counting_writer.h"
#include "index_util.h" #include "index_util.h"
#include "taosdef.h" #include "taosdef.h"
#include "tcompare.h"
static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) { #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
// static tfileGetCompareFunc(uint8_t byte) {}
static int tfileValueCompare(const void* a, const void* b, const void* param) {
__compar_fn_t fn = *(__compar_fn_t*)param;
TFileValue* av = (TFileValue*)a;
TFileValue* bv = (TFileValue*)b;
return fn(av->colVal, bv->colVal);
}
static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds) {
int tbSz = taosArrayGetSize(tableIds);
SERIALIZE_VAR_TO_BUF(buf, tbSz, int32_t);
for (size_t i = 0; i < tbSz; i++) {
uint64_t* v = taosArrayGet(tableIds, i);
SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t);
}
}
static FORCE_INLINE int tfileWriteHeader(TFileWriter* writer) {
char buf[TFILE_HEADER_SIZE] = {0};
char* p = buf;
TFileHeader* header = &writer->header;
SERIALIZE_MEM_TO_BUF(p, header, suid);
SERIALIZE_MEM_TO_BUF(p, header, version);
SERIALIZE_VAR_TO_BUF(p, strlen(header->colName), int32_t);
SERIALIZE_STR_MEM_TO_BUF(p, header, colName, strlen(header->colName));
SERIALIZE_MEM_TO_BUF(p, header, colType);
int offset = p - buf;
int nwrite = writer->ctx->write(writer->ctx, buf, offset);
if (offset != nwrite) { return -1; }
writer->offset = offset;
return 0;
}
static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
TFileHeader* header = &write->header;
uint8_t colType = header->colType;
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal));
if (fstBuilderInsert(write->fb, key, tval->offset)) {
fstSliceDestroy(&key);
return 0;
}
fstSliceDestroy(&key);
return -1;
} else {
// handle other type later
}
}
static FORCE_INLINE int tfileReadLoadHeader(TFileReader* reader) {
// TODO simple tfile header later // TODO simple tfile header later
char buf[TFILE_HADER_PRE_SIZE]; char buf[TFILE_HADER_PRE_SIZE];
char * p = buf; char* p = buf;
TFileReadHeader *header = &reader->header;
int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE);
assert(nread == TFILE_HADER_PRE_SIZE); assert(nread == TFILE_HADER_PRE_SIZE);
TFileHeader* header = &reader->header;
memcpy(&header->suid, p, sizeof(header->suid)); memcpy(&header->suid, p, sizeof(header->suid));
p += sizeof(header->suid); p += sizeof(header->suid);
...@@ -44,43 +103,44 @@ static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) { ...@@ -44,43 +103,44 @@ static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) {
nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType)); nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType));
return 0; return 0;
}; }
static int tfileGetFileList(const char *path, SArray *result) {
DIR *dir = opendir(path);
if (NULL == dir) {
return -1;
}
struct dirent *entry; static int tfileGetFileList(const char* path, SArray* result) {
DIR* dir = opendir(path);
if (NULL == dir) { return -1; }
struct dirent* entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
size_t len = strlen(entry->d_name); size_t len = strlen(entry->d_name);
char * buf = calloc(1, len + 1); char* buf = calloc(1, len + 1);
memcpy(buf, entry->d_name, len); memcpy(buf, entry->d_name, len);
taosArrayPush(result, &buf); taosArrayPush(result, &buf);
} }
closedir(dir); closedir(dir);
return 0; return 0;
} }
static void tfileDestroyFileName(void *elem) { static void tfileDestroyFileName(void* elem) {
char *p = *(char **)elem; char* p = *(char**)elem;
free(p); free(p);
} }
static int tfileCompare(const void *a, const void *b) { static int tfileCompare(const void* a, const void* b) {
const char *aName = *(char **)a; const char* aName = *(char**)a;
const char *bName = *(char **)b; const char* bName = *(char**)b;
size_t aLen = strlen(aName);
size_t bLen = strlen(bName); size_t aLen = strlen(aName);
size_t bLen = strlen(bName);
return strncmp(aName, bName, aLen > bLen ? aLen : bLen); return strncmp(aName, bName, aLen > bLen ? aLen : bLen);
} }
// tfile name suid-colId-version.tindex // tfile name suid-colId-version.tindex
static int tfileParseFileName(const char *filename, uint64_t *suid, int *colId, int *version) { static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) {
if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) {
// read suid & colid & version success // read suid & colid & version success
return 0; return 0;
} }
return -1; return -1;
} }
static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) {
SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_MEM_TO_BUF(buf, key, suid);
SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_MEM_TO_BUF(buf, key, colType); SERIALIZE_MEM_TO_BUF(buf, key, colType);
...@@ -90,38 +150,47 @@ static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { ...@@ -90,38 +150,47 @@ static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
} }
TFileCache *tfileCacheCreate(const char *path) { TFileCache* tfileCacheCreate(const char* path) {
TFileCache *tcache = calloc(1, sizeof(TFileCache)); TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { if (tcache == NULL) { return NULL; }
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
SArray *files = taosArrayInit(4, sizeof(void *)); SArray* files = taosArrayInit(4, sizeof(void*));
tfileGetFileList(path, files); tfileGetFileList(path, files);
taosArraySort(files, tfileCompare); taosArraySort(files, tfileCompare);
uint64_t suid;
int32_t colId, version;
for (size_t i = 0; i < taosArrayGetSize(files); i++) { for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char * file = taosArrayGetP(files, i); char* file = taosArrayGetP(files, i);
uint64_t suid; if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) {
int colId, version; indexInfo("try parse invalid file: %s, skip it", file);
if (0 != tfileParseFileName(file, &suid, &colId, &version)) {
goto End;
continue; continue;
} }
WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64);
WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64);
if (wc == NULL) { if (wc == NULL) {
indexError("failed to open index: %s", file); indexError("failed to open index: %s", file);
goto End; goto End;
} }
TFileReader *reader = tfileReaderCreate(wc); TFileReader* reader = tfileReaderCreate(wc);
if (0 != tfileReadLoadHeader(reader)) { if (0 != tfileReadLoadHeader(reader)) {
TFileReaderDestroy(reader); tfileReaderDestroy(reader);
indexError("failed to load index header, index Id: %s", file); indexError("failed to load index header, index Id: %s", file);
goto End; goto End;
} }
// loader fst and validate it
TFileHeader* header = &reader->header;
TFileCacheKey key = {.suid = header->suid,
.version = header->version,
.colName = header->colName,
.nColName = strlen(header->colName),
.colType = header->colType};
char buf[128] = {0};
tfileSerialCacheKey(&key, buf);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
} }
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return tcache; return tcache;
...@@ -130,78 +199,170 @@ End: ...@@ -130,78 +199,170 @@ End:
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache *tcache) { void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { if (tcache == NULL) { return; }
return;
}
// free table cache // free table cache
TFileReader **reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) { while (reader) {
TFileReader *p = *reader; TFileReader* p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType);
p->header.colType);
TFileReaderDestroy(p); tfileReaderDestroy(p);
reader = taosHashIterate(tcache->tableCache, reader); reader = taosHashIterate(tcache->tableCache, reader);
} }
taosHashCleanup(tcache->tableCache); taosHashCleanup(tcache->tableCache);
free(tcache); free(tcache);
} }
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
return reader; return reader;
} }
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
return; return;
} }
TFileReader *tfileReaderCreate(WriterCtx *ctx) { TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader *reader = calloc(1, sizeof(TFileReader)); TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { if (reader == NULL) { return NULL; }
return NULL;
}
reader->ctx = ctx;
// T_REF_INC(reader); // T_REF_INC(reader);
reader->ctx = ctx;
return reader; return reader;
} }
void TFileReaderDestroy(TFileReader *reader) { void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { if (reader == NULL) { return; }
return;
}
// T_REF_INC(reader); // T_REF_INC(reader);
writerCtxDestroy(reader->ctx); writerCtxDestroy(reader->ctx);
free(reader); free(reader);
} }
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) {
void tfileWriterDestroy(TFileWriter *tw); SIndexTerm* term = query->term;
// refactor to callback later
if (query->qType == QUERY_TERM) {
uint64_t offset;
FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
if (fstGet(reader->fst, &key, &offset)) {
//
} else {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found in tindex", term->suid, term->colName, term->colVal);
}
return 0;
} else if (query->qType == QUERY_PREFIX) {
//
//
}
return 0;
}
IndexTFile *indexTFileCreate(const char *path) { TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); // char pathBuf[128] = {0};
tfile->cache = tfileCacheCreate(path); // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
// TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
// memcpy(header.colName, );
return tfile; // char buf[TFILE_HADER_PRE_SIZE];
// int len = TFILE_HADER_PRE_SIZE;
// if (len != ctx->write(ctx, buf, len)) {
// indexError("index: %" PRIu64 " failed to write header info", header->suid);
// return NULL;
//}
TFileWriter* tw = calloc(1, sizeof(TFileWriter));
if (tw == NULL) {
indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid);
return NULL;
}
tw->ctx = ctx;
tw->header = *header;
tfileWriteHeader(tw);
return tw;
} }
void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); }
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { int TFileWriterPut(TFileWriter* tw, void* data) {
IndexTFile *pTfile = (IndexTFile *)tfile; // sort by coltype and write to tindex
__compar_fn_t fn = getComparFunc(tw->header.colType, 0);
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
int32_t bufLimit = 4096, offset = 0;
char* buf = calloc(1, sizeof(bufLimit));
char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data);
for (size_t i = 0; i < sz; i++) {
TFileValue* v = taosArrayGetP((SArray*)data, i);
SIndexTerm * term = query->term; int32_t tbsz = taosArrayGetSize(v->tableId);
TFileCacheKey key = { // check buf has enough space or not
.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);
TFileReader *reader = tfileCacheGet(pTfile->cache, &key); if (offset + ttsz > bufLimit) {
// batch write
tw->ctx->write(tw->ctx, buf, offset);
offset = 0;
memset(buf, 0, bufLimit);
p = buf;
}
tfileSerialTableIdsToBuf(p, v->tableId);
offset += ttsz;
p = buf + offset;
// set up value offset and
v->offset = tw->offset;
tw->offset += ttsz;
}
if (offset != 0) {
// write reversed data in buf to tindex
tw->ctx->write(tw->ctx, buf, offset);
}
// write fst
for (size_t i = 0; i < sz; i++) {
// TODO, fst batch write later
TFileValue* v = taosArrayGetP((SArray*)data, i);
if (tfileWriteData(tw, v) == 0) {
//
//
}
}
tfree(buf);
return 0; return 0;
} }
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { void tfileWriterDestroy(TFileWriter* tw) {
TFileWriterOpt wOpt = { if (tw == NULL) { return; }
.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
writerCtxDestroy(tw->ctx);
free(tw);
}
IndexTFile* indexTFileCreate(const char* path) {
IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
if (tfile == NULL) { return NULL; }
tfile->cache = tfileCacheCreate(path);
return tfile;
}
void IndexTFileDestroy(IndexTFile* tfile) {
free(tfile);
}
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
if (tfile == NULL) { return -1; }
IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term;
TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
return tfileReaderSearch(reader, query, result);
}
int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
return 0; return 0;
} }
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或