fix fst bug

source/libs/index/src/index.c

@@ -360,6 +360,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
   if (sz > 0) {
     // TODO(yihao): remove duplicate tableid
     TFileValue* lv = taosArrayGetP(result, sz - 1);
+    // indexError("merge colVal: %s", lv->colVal);
     if (strcmp(lv->colVal, tv->colVal) == 0) {
       taosArrayAddAll(lv->tableId, tv->tableId);
       tfileValueDestroy(tv);
@@ -368,6 +369,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
     }
   } else {
     taosArrayPush(result, &tv);
+    // indexError("merge colVal: %s", tv->colVal);
   }
 }
 static void indexDestroyTempResult(SArray* result) {

source/libs/index/src/index_cache.c

@@ -20,7 +20,7 @@
 
 #define MAX_INDEX_KEY_LEN 256  // test only, change later
 
-#define MEM_TERM_LIMIT 10000 * 10
+#define MEM_TERM_LIMIT 10 * 10000
 // ref index_cache.h:22
 //#define CACHE_KEY_LEN(p) \
 //  (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) +
@@ -353,6 +353,9 @@ static bool indexCacheIteratorNext(Iterate* itera) {
   SSkipListIterator* iter = itera->iter;
   if (iter == NULL) { return false; }
   IterateValue* iv = &itera->val;
+  if (iv->colVal != NULL && iv->val != NULL) {
+    // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
+  }
   iterateValueDestroy(iv, false);
 
   bool next = tSkipListIterNext(iter);

source/libs/index/src/index_fst.c

@@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) {
   assert(s->state == OneTransNext || s->state == OneTrans);
 
   uint8_t val;
-  COMMON_INDEX(inp, 0x111111, val);
+  COMMON_INDEX(inp, 0b111111, val);
   s->val = (s->val & fstStateDict[s->state].val) | val;
 }
 
@@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) {
   bool      null = false;
   uint8_t   inp = fstStateCommInput(s, &null);
   uint8_t*  data = fstSliceData(slice, NULL);
-  return null == false ? inp : data[-1];
+  return null == false ? inp : data[node->start - 1];
 }
 uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
   assert(s->state == AnyTrans);

source/libs/index/src/index_tfile.c

@@ -385,8 +385,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
 }
 static bool tfileIteratorNext(Iterate* iiter) {
   IterateValue* iv = &iiter->val;
+  if (iv->colVal != NULL && iv->val != NULL) {
+    // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
+  }
   iterateValueDestroy(iv, false);
-  // SArray* tblIds = iv->val;
 
   char*    colVal = NULL;
   uint64_t offset = 0;
@@ -406,7 +408,7 @@ static bool tfileIteratorNext(Iterate* iiter) {
   if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
 
   iv->colVal = colVal;
-
+  return true;
   // std::string key(ch, sz);
 }
 

source/libs/index/test/fstTest.cc

@@ -24,8 +24,13 @@ class FstWriter {
     _b = fstBuilderCreate(_wc, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
     FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
     bool     ok = fstBuilderInsert(_b, skey, val);
+
     fstSliceDestroy(&skey);
     return ok;
   }
@@ -61,6 +66,11 @@ class FstReadMemory {
     return _fst != NULL;
   }
   bool Get(const std::string& key, uint64_t* val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
+
     FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
     bool     ok = fstGet(_fst, &skey, val);
     fstSliceDestroy(&skey);
@@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) {
   }
   return L * M * N;
 }
+void Performance_fstReadRecords(FstReadMemory* m) {
+  std::string str("aa");
+  for (int i = 0; i < M; i++) {
+    str[0] = 'a' + i;
+    str.resize(2);
+    for (int j = 0; j < N; j++) {
+      str[1] = 'a' + j;
+      str.resize(2);
+      for (int k = 0; k < L; k++) {
+        str.push_back('a');
+        uint64_t val, cost;
+        if (m->GetWithTimeCostUs(str, &val, &cost)) {
+          printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
+        } else {
+          printf("failed to get key: %s\n", str.c_str());
+        }
+      }
+    }
+  }
+}
+
+void checkMillonWriteAndReadOfFst() {
+  tfInit();
+  FstWriter* fw = new FstWriter;
+  Performance_fstWriteRecords(fw);
+  delete fw;
+  FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
+
+  if (fr->init()) { printf("success to init fst read"); }
+
+  Performance_fstReadRecords(fr);
+  tfCleanup();
+  delete fr;
+}
+void checkFstLongTerm() {
+  tfInit();
+  FstWriter* fw = new FstWriter;
+  // Performance_fstWriteRecords(fw);
+
+  fw->Put("A B", 1);
+  fw->Put("C", 2);
+  fw->Put("a", 3);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("A B", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("C", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("a", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+
+  // prefix search
+  // std::vector<uint64_t> result;
+
+  // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  // m->Search(ctx, result);
+  // std::cout << "size: " << result.size() << std::endl;
+  // assert(result.size() == count);
+  // for (int i = 0; i < result.size(); i++) {
+  // assert(result[i] == i);  // check result
+  //}
+  tfCleanup();
+  // free(ctx);
+  // delete m;
+}
 void checkFstCheckIterator() {
   tfInit();
   FstWriter* fw = new FstWriter;
   int64_t    s = taosGetTimestampUs();
   int        count = 2;
-  Performance_fstWriteRecords(fw);
+  // Performance_fstWriteRecords(fw);
   int64_t e = taosGetTimestampUs();
 
   std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("Hello world", 1);
+  fw->Put("hello world", 2);
+  fw->Put("hello worle", 3);
+  fw->Put("hello worlf", 4);
   delete fw;
 
   FstReadMemory* m = new FstReadMemory(1024 * 64);
@@ -171,7 +275,7 @@ void checkFstCheckIterator() {
 
 void fst_get(Fst* fst) {
   for (int i = 0; i < 10000; i++) {
-    std::string term = "Hello";
+    std::string term = "Hello World";
     FstSlice    key = fstSliceCreate((uint8_t*)term.c_str(), term.size());
     uint64_t    offset = 0;
     bool        ret = fstGet(fst, &key, &offset);
@@ -189,7 +293,7 @@ void validateTFile(char* arg) {
 
   std::thread threads[NUM_OF_THREAD];
   // std::vector<std::thread> threads;
-  TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1");
+  TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1");
 
   for (int i = 0; i < NUM_OF_THREAD; i++) {
     threads[i] = std::thread(fst_get, reader->fst);
@@ -203,9 +307,12 @@ void validateTFile(char* arg) {
   tfCleanup();
 }
 int main(int argc, char* argv[]) {
-  if (argc > 1) { validateTFile(argv[1]); }
+  // tool to check all kind of fst test 
+  // if (argc > 1) { validateTFile(argv[1]); }
   // checkFstCheckIterator();
+  // checkFstLongTerm();
   // checkFstPrefixSearch();
 
+  checkMillonWriteAndReadOfFst();
   return 1;
 }

source/libs/index/test/indexTests.cc

@@ -787,14 +787,15 @@ TEST_F(IndexEnv2, testIndexOpen) {
 }
 
 TEST_F(IndexEnv2, testIndex_TrigeFlush) {
-  std::string path = "/tmp/test";
+  std::string path = "/tmp/test1";
   if (index->Init(path) != 0) {
     // r
     std::cout << "failed to init" << std::endl;
   }
   int numOfTable = 100 * 10000;
-  index->WriteMillonData("tag1", "Hello", numOfTable);
-  int target = index->SearchOne("tag1", "Hello");
+  index->WriteMillonData("tag1", "Hello Wolrd", numOfTable);
+  int target = index->SearchOne("tag1", "Hello Wolrd");
+  std::cout << "Get Index: " << target << std::endl;
   assert(numOfTable == target);
 }
 
@@ -821,14 +822,6 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
     threads[i].join();
   }
 }
-TEST_F(IndexEnv2, testIndex_multi_thread_write) {
-  std::string path = "/tmp";
-  if (index->Init(path) != 0) {}
-}
-TEST_F(IndexEnv2, testIndex_multi_thread_read) {
-  std::string path = "/tmp";
-  if (index->Init(path) != 0) {}
-}
 
 TEST_F(IndexEnv2, testIndex_restart) {
   std::string path = "/tmp";