diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 40e35306a2498e73c1af92c2ace100079f692b5f..457b5422a42cb5ebab266707e578b94adc01ee0d 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -478,6 +478,7 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { return 0; } FstSlice *slice = &node->data; + uint8_t *data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -485,7 +486,6 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - (i * oSizes) - oSizes; - uint8_t *data = fstSliceData(slice, NULL); return unpackUint64(data + at, oSizes); } @@ -555,6 +555,7 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) + - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; @@ -587,7 +588,8 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { FstSlice t = fstSliceCopy(slice, start, end - 1); int32_t len = 0; uint8_t *data = fstSliceData(&t, &len); - for(int i = 0; i < len; i++) { + int i = 0; + for(; i < len; i++) { //uint8_t v = slice->data[slice->start + i]; ////slice->data[slice->start + i]; uint8_t v = data[i]; @@ -595,6 +597,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { return node->nTrans - i - 1; // bug } } + if (i == len) { *null = true; } } } @@ -774,7 +777,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { if (NULL == b) { return b; } - b->wrt = fstCountingWriterCreate(w, false); + b->wrt = fstCountingWriterCreate(w, false); b->unfinished = fstUnFinishedNodesCreate(); b->registry = fstRegistryCreate(10000, 2) ; b->last = fstSliceCreate(NULL, 0); @@ -857,6 +860,7 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { return OutOfOrdered; } // deep copy or not + fstSliceDestroy(&b->last); b->last = fstSliceCopy(&bs, input->start, input->end); } return Ordered; @@ -1007,8 +1011,7 @@ Fst* fstCreate(FstSlice *slice) { uint64_t fstLen; len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); - //TODO(validat root addr) - // + //TODO(validate root addr) Fst *fst= (Fst *)calloc(1, sizeof(Fst)); if (fst == NULL) { return NULL; } @@ -1023,6 +1026,7 @@ Fst* fstCreate(FstSlice *slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; fst->data = slice; + return fst; FST_CREAT_FAILED: diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 9135a7a173e886f6be1993f7fcf1518fd54b990e..86f19e804460a53c93e27f0ca3a14d957d09abdc 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -2,13 +2,79 @@ #include #include #include "index.h" +#include "tutil.h" #include "indexInt.h" #include "index_fst.h" #include "index_fst_util.h" #include "index_fst_counting_writer.h" - +class FstWriter { + public: + FstWriter() { + _b = fstBuilderCreate(NULL, 0); + } + bool Put(const std::string &key, uint64_t val) { + FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + } + private: + FstBuilder *_b; +}; + +class FstReadMemory { + public: + FstReadMemory(size_t size) { + _w = fstCountingWriterCreate(NULL, true); + _size = size; + memset((void *)&_s, 0, sizeof(_s)); + } + bool init() { + char *buf = (char *)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size); + if (nRead <= 0) { return false; } + _size = nRead; + _s = fstSliceCreate((uint8_t *)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string &key, uint64_t *val) { + FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(const std::string &key, std::vector &result) { + return true; + } + + ~FstReadMemory() { + fstCountingWriterDestroy(_w); + fstSliceDestroy(&_s); + } + + private: + FstCountingWriter *_w; + Fst *_fst; + FstSlice _s; + size_t _size; + +}; //TEST(IndexTest, index_create_test) { // SIndexOpts *opts = indexOptsCreate(); @@ -62,69 +128,77 @@ // // //} -int main(int argc, char** argv) { - // test write - FstBuilder *b = fstBuilderCreate(NULL, 0); - { - std::string str("aaa"); - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 1; - fstBuilderInsert(b, key, val); + + +void Performance_fstWriteRecords(FstWriter *b) { + std::string str("aa"); + for (int i = 0; i < 26; i++) { + str[0] = 'a' + i; + str.resize(2); + for(int j = 0; j < 26; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < 10; k++) { + str.push_back('a'); + b->Put(str, k); + } + } } +} + +void Performance_fstReadRecords(FstReadMemory *m) { + std::string str("a"); + for (int i = 0; i < 500; i++) { + //std::string str("aa"); + str.push_back('a'); + uint64_t out, cost; + bool ok = m->GetWithTimeCostUs(str, &out, &cost); + if (ok == true) { + printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost); + } else { + printf("failed to get(%s)\n", str.c_str()); + } + } +} - //std::string str1("bcd"); - //FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); - //Output val2 = 10; +int main(int argc, char** argv) { + // test write // - + FstWriter *fw = new FstWriter; { - - for (size_t i = 1; i < 26; i++) { - std::string str("aaa"); - str[2] = 'a' + i ; - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 0; - fstBuilderInsert(b, key, val); + std::string key("ab"); + int64_t val = 100; + for (int i = 0; i < 26; i++) { + key.push_back('a' + i); + fw->Put(key, val++); } - - } - fstBuilderFinish(b); - fstBuilderDestroy(b); - - - char buf[64 * 1024] = {0}; - - FstSlice s; - - FstCountingWriter *w = fstCountingWriterCreate(NULL, true); - int nRead = fstCountingWriterRead(w, (uint8_t *)buf, sizeof(buf)); - assert(nRead <= sizeof(buf)); - s = fstSliceCreate((uint8_t *)buf, nRead); - fstCountingWriterDestroy(w); + } + delete fw; + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + } - // test reader - - - Fst *fst = fstCreate(&s); { - std::string str("aax"); - uint64_t out; - - - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - bool ok = fstGet(fst, &key, &out); - if (ok == true) { - printf("val = %d\n", out); - //indexInfo("Get key-value success, %s, %d", str.c_str(), out); - } else { - //indexError("Get key-value failed, %s", str.c_str()); + std::string key("ab"); + uint64_t out; + if (m->Get(key, &out)) { + printf("success to get (%s, %" PRId64")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); + } + for (int i = 0; i < 26; i++) { + key.push_back('a' + i); + if (m->Get(key, &out)) { + printf("success to get (%s, %" PRId64")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); } - } - fstSliceDestroy(&s); - - + } + } + return 1; }