Merge pull request #9017 from taosdata/feature/fst

Feature/fst

source/libs/index/src/index_fst.c

@@ -297,15 +297,16 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
     // any value greater than or equal to the number of transitions in
     // this node indicates an absent transition.
     uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); 
-    for (uint8_t i = 0; i < 256; i++) {
-      index[i] = 255;
-    }
+    memset(index, 255, sizeof(uint8_t) * 256);
+    ///for (uint8_t i = 0; i < 256; i++) {
+    //  index[i] = 255;
+    ///}
     for (size_t i = 0; i < sz; i++) {
       FstTransition *t = taosArrayGet(node->trans, i);
       index[t->inp] = i;
-      fstCountingWriterWrite(w, (char *)index, sizeof(index)); 
       //fstPackDeltaIn(w, addr, t->addr, tSize);
     }
+    fstCountingWriterWrite(w, (char *)index, 256); 
     free(index);
   }
   fstCountingWriterWrite(w, (char *)&packSizes, 1);
@@ -1126,6 +1127,7 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) {
   return b; 
 }
 
+
 bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) {
   int comp = fstSliceCompare(slice, &bound->data);
   if (bound->type == Included) {
@@ -1378,7 +1380,9 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) {
 }
 void fstStreamBuilderDestroy(FstStreamBuilder *b) {
   fstSliceDestroy(&b->min->data);
+  tfree(b->min);  
   fstSliceDestroy(&b->max->data);
+  tfree(b->max);
   free(b);
 }
 FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) {

source/libs/index/test/indexTests.cpp

@@ -130,25 +130,28 @@ class FstReadMemory {
 
 
 
-void Performance_fstWriteRecords(FstWriter *b) {
+int Performance_fstWriteRecords(FstWriter *b) {
   std::string str("aa"); 
-  for (int i = 0; i < 26; i++) {
+  int L = 100, M = 100, N = 10;
+  for (int i = 0; i < L; i++) {
     str[0] = 'a' + i;
     str.resize(2); 
-    for(int j = 0; j < 26; j++) {
+    for(int j = 0; j < M; j++) {
       str[1] = 'a' + j;
       str.resize(2);
-      for (int k = 0; k < 10; k++) {
+      for (int k = 0; k < N; k++) {
         str.push_back('a');
         b->Put(str, k);
+        printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
       }
     } 
   }
+  return L * M * N;
 }
 
 void Performance_fstReadRecords(FstReadMemory *m) {
    std::string str("a");
-   for (int i = 0; i < 500; i++) {
+   for (int i = 0; i < 50; i++) {
      //std::string str("aa"); 
      str.push_back('a');
      uint64_t out, cost;
@@ -160,21 +163,41 @@ void Performance_fstReadRecords(FstReadMemory *m) {
      }
    }
 }
+void checkFstPerf() {
+  FstWriter *fw = new FstWriter;
+  int64_t s = taosGetTimestampUs();
+  int num = Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+  printf("write %d record cost %" PRId64"us\n", num,  e - s);
+  delete fw;
 
-int main(int argc, char** argv) {
-  // test write
-  //
+  FstReadMemory *m = new FstReadMemory(1024 * 64);
+  if (m->init()) {
+    uint64_t val;
+    if(m->Get("aaaaaaa", &val)) {
+      std::cout << "succes to Get val: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get " << std::endl;
+    } 
+  }  
+} 
+
+
+void validateFst() {
+  int val = 100;
+  int count = 100;
   FstWriter *fw = new FstWriter;
+  // write 
   {
     std::string key("ab");
-    int64_t val = 100; 
-    for (int i = 0; i < 26; i++) {
+    for (int i = 0; i < count; i++) {
       key.push_back('a' + i);
-      fw->Put(key, val++);
+      fw->Put(key, val - i);
     }
   }
   delete fw;
 
+  // read
   FstReadMemory *m = new FstReadMemory(1024 * 64);
   if (m->init() == false) { 
     std::cout << "init readMemory failed" << std::endl; 
@@ -188,17 +211,21 @@ int main(int argc, char** argv) {
    } else {
      printf("failed to get(%s)\n", key.c_str());
    }
-   for (int i = 0; i < 26; i++) {
+   for (int i = 0; i < count; i++) {
      key.push_back('a' + i);
-     if (m->Get(key, &out)) {
+     if (m->Get(key, &out) ) {
+       assert(val - i ==  out);
        printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
      } else {
        printf("failed to get(%s)\n", key.c_str());
     }
    }
   } 
+  delete m;
 
-     
+} 
+int main(int argc, char** argv) {
+  checkFstPerf(); 
   return 1;
 }