提交 cd653e56 编写于 作者: dengyihao's avatar dengyihao

update fst search frame

上级 f8ca5dc1
...@@ -291,11 +291,11 @@ typedef struct StreamState { ...@@ -291,11 +291,11 @@ typedef struct StreamState {
void streamStateDestroy(void *s); void streamStateDestroy(void *s);
typedef struct StreamWithState { typedef struct StreamWithState {
Fst *fst; Fst *fst;
Automation *aut; AutomationCtx *aut;
SArray *inp; SArray *inp;
FstOutput emptyOutput; FstOutput emptyOutput;
SArray *stack; // <StreamState> SArray *stack; // <StreamState>
FstBoundWithData *endAt; FstBoundWithData *endAt;
} StreamWithState ; } StreamWithState ;
...@@ -310,19 +310,19 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta ...@@ -310,19 +310,19 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
void swsResultDestroy(StreamWithStateResult *result); void swsResultDestroy(StreamWithStateResult *result);
typedef void* (*StreamCallback)(void *); typedef void* (*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ; StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ;
void streamWithStateDestroy(StreamWithState *sws); void streamWithStateDestroy(StreamWithState *sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
typedef struct FstStreamBuilder { typedef struct FstStreamBuilder {
Fst *fst; Fst *fst;
Automation *aut; AutomationCtx *aut;
FstBoundWithData *min; FstBoundWithData *min;
FstBoundWithData *max; FstBoundWithData *max;
} FstStreamBuilder; } FstStreamBuilder;
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut);
// set up bound range // set up bound range
// refator, simple code by marco // refator, simple code by marco
......
...@@ -19,33 +19,39 @@ ...@@ -19,33 +19,39 @@
extern "C" { extern "C" {
#endif #endif
#include "index_fst_util.h"
typedef struct AutomationCtx AutomationCtx; typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType {
AUTOMATION_PREFIX,
AUTMMATION_MATCH
} AutomationType;
typedef struct StartWith { typedef struct StartWith {
AutomationCtx *autoSelf; AutomationCtx *autoSelf;
} StartWith; } StartWith;
typedef struct Complement { typedef struct Complement {
AutomationCtx *autoSelf; AutomationCtx *autoSelf;
} Complement; } Complement;
// automation // automation
typedef struct AutomationCtx { typedef struct AutomationCtx {
// automation interface AutomationType type;
void *data;
} AutomationCtx; } AutomationCtx;
typedef struct Automation { typedef struct AutomationFunc {
void* (*start)() ; void* (*start)(AutomationCtx *ctx) ;
bool (*isMatch)(void *); bool (*isMatch)(AutomationCtx *ctx, void *);
bool (*canMatch)(void *data); bool (*canMatch)(AutomationCtx *ctx, void *data);
bool (*willAlwaysMatch)(void *state); bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state);
void* (*accept)(void *state, uint8_t byte); void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte);
void* (*acceptEof)(void *state); void* (*acceptEof)(AutomationCtx *ct, void *state);
void *data; } AutomationFunc;
} Automation;
AutomationCtx *automCtxCreate(void *data, AutomationType type);
extern AutomationFunc automFuncs[];
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -1177,7 +1177,7 @@ void fstBoundDestroy(FstBoundWithData *bound) { ...@@ -1177,7 +1177,7 @@ void fstBoundDestroy(FstBoundWithData *bound) {
free(bound); free(bound);
} }
StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) { StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) {
StreamWithState *sws = calloc(1, sizeof(StreamWithState)); StreamWithState *sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) { return NULL; } if (sws == NULL) { return NULL; }
...@@ -1204,6 +1204,8 @@ void streamWithStateDestroy(StreamWithState *sws) { ...@@ -1204,6 +1204,8 @@ void streamWithStateDestroy(StreamWithState *sws) {
} }
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
AutomationCtx *aut = sws->aut;
if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsEmpty(min)) {
if (fstBoundWithDataIsIncluded(min)) { if (fstBoundWithDataIsIncluded(min)) {
sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
...@@ -1211,7 +1213,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1211,7 +1213,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
StreamState s = {.node = fstGetRoot(sws->fst), StreamState s = {.node = fstGetRoot(sws->fst),
.trans = 0, .trans = 0,
.out = {.null = false, .out = 0}, .out = {.null = false, .out = 0},
.autState = sws->aut->start()}; // auto.start callback .autState = automFuncs[aut->type].start(aut)}; // auto.start callback
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
...@@ -1229,7 +1231,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1229,7 +1231,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
FstNode *node = fstGetRoot(sws->fst); FstNode *node = fstGetRoot(sws->fst);
Output out = 0; Output out = 0;
void* autState = sws->aut->start(); //void* autState = sws->aut->start();
void* autState = automFuncs[aut->type].start(aut);
int32_t len; int32_t len;
uint8_t *data = fstSliceData(key, &len); uint8_t *data = fstSliceData(key, &len);
...@@ -1241,7 +1244,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1241,7 +1244,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(node, res, &trn); fstNodeGetTransitionAt(node, res, &trn);
void *preState = autState; void *preState = autState;
autState = sws->aut->accept(preState, b); // autState = sws->aut->accept(preState, b);
autState = automFuncs[aut->type].accept(aut, preState, b);
taosArrayPush(sws->inp, &b); taosArrayPush(sws->inp, &b);
StreamState s = {.node = node, StreamState s = {.node = node,
.trans = res + 1, .trans = res + 1,
...@@ -1298,6 +1302,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { ...@@ -1298,6 +1302,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
} }
StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) {
AutomationCtx *aut = sws->aut;
FstOutput output = sws->emptyOutput; FstOutput output = sws->emptyOutput;
if (output.null == false) { if (output.null == false) {
FstSlice emptySlice = fstSliceCreate(NULL, 0); FstSlice emptySlice = fstSliceCreate(NULL, 0);
...@@ -1306,15 +1311,15 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb ...@@ -1306,15 +1311,15 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState));
return NULL; return NULL;
} }
void* start = sws->aut->start(); void *start = automFuncs[aut->type].start(aut);
if (sws->aut->isMatch(start)) { if (automFuncs[aut->type].isMatch(aut, start)) {
FstSlice s = fstSliceCreate(NULL, 0); FstSlice s = fstSliceCreate(NULL, 0);
return swsResultCreate(&s, output, callback(start)); return swsResultCreate(&s, output, callback(start));
} }
} }
while (taosArrayGetSize(sws->stack) > 0) { while (taosArrayGetSize(sws->stack) > 0) {
StreamState *p = (StreamState *)taosArrayPop(sws->stack); StreamState *p = (StreamState *)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || !sws->aut->canMatch(p->autState)) { if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
taosArrayPop(sws->inp); taosArrayPop(sws->inp);
} }
...@@ -1324,16 +1329,18 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb ...@@ -1324,16 +1329,18 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(p->node, p->trans, &trn); fstNodeGetTransitionAt(p->node, p->trans, &trn);
Output out = p->out.out + trn.out; Output out = p->out.out + trn.out;
void* nextState = sws->aut->accept(p->autState, trn.inp); void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp);
void* tState = callback(nextState); void* tState = callback(nextState);
bool isMatch = sws->aut->isMatch(nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState);
//bool isMatch = sws->aut->isMatch(nextState);
FstNode *nextNode = fstGetNode(sws->fst, trn.addr); FstNode *nextNode = fstGetNode(sws->fst, trn.addr);
taosArrayPush(sws->inp, &(trn.inp)); taosArrayPush(sws->inp, &(trn.inp));
if (FST_NODE_IS_FINAL(nextNode)) { if (FST_NODE_IS_FINAL(nextNode)) {
void *eofState = sws->aut->acceptEof(nextState); //void *eofState = sws->aut->acceptEof(nextState);
void *eofState = automFuncs[aut->type].acceptEof(aut, nextState);
if (eofState != NULL) { if (eofState != NULL) {
isMatch = sws->aut->isMatch(eofState); isMatch = automFuncs[aut->type].isMatch(aut, eofState);
} }
} }
StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
...@@ -1391,7 +1398,7 @@ void streamStateDestroy(void *s) { ...@@ -1391,7 +1398,7 @@ void streamStateDestroy(void *s) {
//free(s->autoState); //free(s->autoState);
} }
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) {
FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { return NULL; } if (NULL == b) { return NULL; }
......
...@@ -13,3 +13,77 @@ ...@@ -13,3 +13,77 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "index_fst_automation.h"
// prefix query, impl later
static void* prefixStart(AutomationCtx *ctx) {
return NULL;
};
static bool prefixIsMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool prefixCanMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) {
return true;
}
static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
return NULL;
}
static void* prefixAcceptEof(AutomationCtx *ctx, void *state) {
return NULL;
}
// pattern query, impl later
static void* patternStart(AutomationCtx *ctx) {
return NULL;
}
static bool patternIsMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool patternCanMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) {
return true;
}
static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
return NULL;
}
static void* patternAcceptEof(AutomationCtx *ctx, void *state) {
return NULL;
}
AutomationFunc automFuncs[] = {{
prefixStart,
prefixIsMatch,
prefixCanMatch,
prefixWillAlwaysMatch,
prefixAccept,
prefixAcceptEof
},
{
patternStart,
patternIsMatch,
patternCanMatch,
patternWillAlwaysMatch,
patternAccept,
patternAcceptEof
}
};
AutomationCtx* automCtxCreate(void *data, AutomationType type) {
AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; }
ctx->type = type;
if (type == AUTOMATION_PREFIX) {
}
}
...@@ -65,6 +65,7 @@ class FstReadMemory { ...@@ -65,6 +65,7 @@ class FstReadMemory {
~FstReadMemory() { ~FstReadMemory() {
fstCountingWriterDestroy(_w); fstCountingWriterDestroy(_w);
fstDestroy(_fst);
fstSliceDestroy(&_s); fstSliceDestroy(&_s);
} }
...@@ -129,10 +130,12 @@ class FstReadMemory { ...@@ -129,10 +130,12 @@ class FstReadMemory {
//} //}
#define L 100
#define M 100
#define N 100
int Performance_fstWriteRecords(FstWriter *b) { int Performance_fstWriteRecords(FstWriter *b) {
std::string str("aa"); std::string str("aa");
int L = 100, M = 100, N = 10;
for (int i = 0; i < L; i++) { for (int i = 0; i < L; i++) {
str[0] = 'a' + i; str[0] = 'a' + i;
str.resize(2); str.resize(2);
...@@ -150,22 +153,29 @@ int Performance_fstWriteRecords(FstWriter *b) { ...@@ -150,22 +153,29 @@ int Performance_fstWriteRecords(FstWriter *b) {
} }
void Performance_fstReadRecords(FstReadMemory *m) { void Performance_fstReadRecords(FstReadMemory *m) {
std::string str("a"); std::string str("aa");
for (int i = 0; i < 50; i++) { for (int i = 0; i < M; i++) {
//std::string str("aa"); str[0] = 'a' + i;
str.push_back('a'); str.resize(2);
uint64_t out, cost; for(int j = 0; j < N; j++) {
bool ok = m->GetWithTimeCostUs(str, &out, &cost); str[1] = 'a' + j;
if (ok == true) { str.resize(2);
printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost); for (int k = 0; k < L; k++) {
} else { str.push_back('a');
printf("failed to get(%s)\n", str.c_str()); uint64_t val, cost;
} if (m->GetWithTimeCostUs(str, &val, &cost)) {
} printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
} else {
printf("failed to get key: %s\n", str.c_str());
}
}
}
}
} }
void checkFstPerf() { void checkFstPerf() {
FstWriter *fw = new FstWriter; FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs(); int64_t s = taosGetTimestampUs();
int num = Performance_fstWriteRecords(fw); int num = Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs(); int64_t e = taosGetTimestampUs();
printf("write %d record cost %" PRId64"us\n", num, e - s); printf("write %d record cost %" PRId64"us\n", num, e - s);
...@@ -173,13 +183,11 @@ void checkFstPerf() { ...@@ -173,13 +183,11 @@ void checkFstPerf() {
FstReadMemory *m = new FstReadMemory(1024 * 64); FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init()) { if (m->init()) {
uint64_t val; printf("success to init fst read");
if(m->Get("aaaaaaa", &val)) {
std::cout << "succes to Get val: " << val << std::endl;
} else {
std::cout << "failed to Get " << std::endl;
}
} }
Performance_fstReadRecords(m);
delete m;
} }
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册