From 489179b980e16faff2594f5493fc569b84afefea Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 20 Dec 2021 14:09:12 +0800 Subject: [PATCH] update index TFile write --- .../index/inc/index_fst_counting_writer.h | 14 +++++-- source/libs/index/inc/index_tfile.h | 17 ++++++++- source/libs/index/src/index_fst.c | 2 +- .../index/src/index_fst_counting_writer.c | 38 ++++++++++--------- source/libs/index/src/index_tfile.c | 6 +++ source/libs/index/test/indexTests.cc | 2 +- 6 files changed, 53 insertions(+), 26 deletions(-) diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ea090389bb..ac9a59fa04 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -34,8 +34,14 @@ typedef struct WriterCtx { int (*flush)(struct WriterCtx *ctx); WriterType type; union { - int fd; - void *mem; + struct { + int fd; + bool readOnly; + } file; + struct { + int32_t capa; + char *buf; + } mem; }; int32_t offset; int32_t limit; @@ -45,7 +51,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoFlush(WriterCtx *ctx); -WriterCtx* writerCtxCreate(WriterType type, bool readOnly); +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); void writerCtxDestroy(WriterCtx *w); typedef uint32_t CheckSummer; @@ -66,7 +72,7 @@ int fstCountingWriterFlush(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); -FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); +FstCountingWriter *fstCountingWriterCreate(void *wtr); void fstCountingWriterDestroy(FstCountingWriter *w); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index c3f4bd25e5..85c4c18bee 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -18,23 +18,36 @@ #include "index.h" #include "indexInt.h" #include "tlockfree.h" -#include "tskiplist.h" +#include "index_tfile.h" +#include "index_fst.h" #ifdef __cplusplus extern "C" { #endif +typedef struct TFileWriter { + FstBuilder *fb; +} TFileWriter; + +typedef struct TFileReader { + T_REF_DECLARE() + Fst *fst; +} TFileReader; + typedef struct IndexTFile { T_REF_DECLARE() + TFileReader *tb; + TFileWriter *tw; } IndexTFile; - IndexTFile *indexTFileCreate(); +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); + #ifdef __cplusplus } diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 7aaa498864..0f00aacf3b 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -779,7 +779,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { if (NULL == b) { return b; } - b->wrt = fstCountingWriterCreate(w, false); + b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); b->registry = fstRegistryCreate(10000, 2) ; b->last = fstSliceCreate(NULL, 0); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 9ec346cebc..3497b9703d 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -23,9 +23,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { } if (ctx->type == TFile) { - assert(len == tfWrite(ctx->fd, buf, len)); + assert(len == tfWrite(ctx->file.fd, buf, len)); } else { - memcpy(ctx->mem + ctx->offset, buf, len); + memcpy(ctx->mem.buf+ ctx->offset, buf, len); } ctx->offset += len; return len; @@ -33,9 +33,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { int nRead = 0; if (ctx->type == TFile) { - nRead = tfRead(ctx->fd, buf, len); + nRead = tfRead(ctx->file.fd, buf, len); } else { - memcpy(buf, ctx->mem + ctx->offset, len); + memcpy(buf, ctx->mem.buf + ctx->offset, len); } ctx->offset += nRead; @@ -44,63 +44,64 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoFlush(WriterCtx *ctx) { if (ctx->type == TFile) { //tfFsync(ctx->fd); - //tfFlush(ctx->fd); + //tfFlush(ctx->file.fd); } else { // do nothing } return 1; } -WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { - tfInit(); // ugly code, refactor later + ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); } else { - ctx->fd = tfOpenReadWrite(tmpFile); + ctx->file.fd = tfOpenReadWrite(tmpFile); } - if (ctx->fd < 0) { + if (ctx->file.fd < 0) { indexError("open file error %d", errno); } } else if (ctx->type == TMemory) { - ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); + ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->offset = 0; - ctx->limit = DefaultMem; + ctx->limit = capacity; return ctx; } void writerCtxDestroy(WriterCtx *ctx) { if (ctx->type == TMemory) { - free(ctx->mem); + free(ctx->mem.buf); } else { - tfClose(ctx->fd); - tfCleanup(); + tfClose(ctx->file.fd); } free(ctx); } -FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { +FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); + cw->wrt = wrt; + //(void *)(writerCtxCreate(TFile, readOnly)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); - writerCtxDestroy((WriterCtx *)(cw->wrt)); + //writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } @@ -124,6 +125,7 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { + return 0; } int fstCountingWriterFlush(FstCountingWriter *write) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a1bba56391..e191ff688e 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -14,18 +14,24 @@ */ #include "index_tfile.h" +#include "index_fst.h" IndexTFile *indexTFileCreate() { IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); + return tfile; } void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } + + int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { IndexTFile *ptfile = (IndexTFile *)tfile; + return 0; } +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9dff2e9ea0..9baabb9610 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -45,7 +45,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _w = fstCountingWriterCreate(NULL, true); + _w = fstCountingWriterCreate(NULL); _size = size; memset((void *)&_s, 0, sizeof(_s)); } -- GitLab