Merge pull request #8930 from taosdata/origin/3.0/fst

refactor builder struct

source/libs/index/src/index_fst.c

@@ -62,7 +62,7 @@ FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) {
   assert(taosArrayGetSize(nodes->stack) == 1);
 
   FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack);
-  assert(un->last == NULL); 
+  //assert(un->last == NULL); 
   return un->node;  
 }
 
@@ -70,6 +70,7 @@ FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledA
   FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack);
   fstBuilderNodeUnfinishedLastCompiled(un, addr);
   free(un->last); // TODO add func FstLastTransitionFree()
+  un->last = NULL;
   return un->node; 
 }
 
@@ -99,8 +100,6 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
   FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz);
   assert(un->last == NULL);
 
-     
-  
   //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
   //trn->inp = s->data[s->start]; 
   //trn->out = out;
@@ -108,7 +107,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
   uint8_t *data = fstSliceData(s, &len);
   un->last = fstLastTransitionCreate(data[0], out); 
 
-  for (uint64_t i = 0; i < len; i++) {
+  for (uint64_t i = 1; i < len; i++) {
     FstBuilderNode *n = malloc(sizeof(FstBuilderNode));
     n->isFinal     = false;
     n->finalOutput = 0;
@@ -117,7 +116,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
     //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
     //trn->inp = s->data[i];
     //trn->out = out; 
-    FstLastTransition *trn = fstLastTransitionCreate(data[i], out);
+    FstLastTransition *trn = fstLastTransitionCreate(data[i], 0);
 
     FstBuilderNodeUnfinished un = {.node = n, .last = trn}; 
     taosArrayPush(nodes->stack, &un); 
@@ -131,8 +130,8 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs)
 
   size_t ssz = taosArrayGetSize(node->stack);  // stack size
   uint64_t count = 0;
-  int32_t lsz;          // data len 
-  uint8_t *data = fstSliceData(s, &lsz);
+  int32_t  lsz;          // data len 
+  uint8_t  *data = fstSliceData(s, &lsz);
   for (size_t i = 0; i < ssz && i < lsz; i++) {
     FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); 
     if (un->last->inp == data[i]) {
@@ -854,7 +853,7 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) {
     } else {
       n = fstUnFinishedNodesPopFreeze(b->unfinished, addr);
     }
-    addr =  fstBuilderCompile(b, n);
+    addr = fstBuilderCompile(b, n);
     assert(addr != NONE_ADDRESS);      
     //fstBuilderNodeDestroy(n);
   }
@@ -890,22 +889,26 @@ void* fstBuilderInsertInner(FstBuilder *b) {
   FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); 
   CompiledAddr  rootAddr = fstBuilderCompile(b, rootNode);
 
-  uint8_t buf64[8] = {0}; 
+  char  buf64[8] = {0}; 
 
-  taosEncodeFixedU64((void **)&buf64, b->len); 
+  void  *pBuf64 = buf64; 
+  taosEncodeFixedU64(&pBuf64, b->len); 
   fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));  
     
-  taosEncodeFixedU64((void **)&buf64, rootAddr);  
+  pBuf64 = buf64;
+  taosEncodeFixedU64(&pBuf64, rootAddr);  
   fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));  
 
-  uint8_t buf32[4] = {0};
+  char buf32[4] = {0};
+  void *pBuf32  = buf32; 
   uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt);
-  taosEncodeFixedU32((void **)&buf32, sum); 
+  taosEncodeFixedU32(&pBuf32, sum); 
   fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32));  
   
   fstCountingWriterFlush(b->wrt);
+  //fstCountingWriterDestroy(b->wrt);  
+  //b->wrt = NULL;
   return b->wrt;
-  
 }
 void fstBuilderFinish(FstBuilder *b) {
   fstBuilderInsertInner(b);

source/libs/index/src/index_fst_counting_writer.c

@@ -52,9 +52,12 @@ WriterCtx* writerCtxCreate(WriterType type) {
   WriterCtx *ctx = calloc(1, sizeof(WriterCtx));
   if (ctx == NULL) { return NULL; }
 
-  ctx->type == type;
+  ctx->type = type;
   if (ctx->type == TFile) {
     ctx->fd = tfOpenCreateWriteAppend(tmpFile);  
+    if (ctx->fd < 0) {
+      
+    }
   } else if (ctx->type == TMemory) {
     ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t));
   } 

source/libs/index/src/index_fst_node.c

@@ -30,20 +30,26 @@ void fstBuilderNodeDestroy(FstBuilderNode *node) {
 
 bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
   if (n1 == n2) { return true; }
+  if (n1 == NULL || n2 == NULL ) {
+    return false;
+  }
 
-  if (n1->isFinal != n2->isFinal || 
-      n1->finalOutput != n2->finalOutput ||
-      taosArrayGetSize(n1->trans) != taosArrayGetSize(n2->trans)) {
-    return false;  
-  } 
-  size_t sz =  taosArrayGetSize(n1->trans); 
-  for (size_t i = 0; i < sz; i++) {
+  if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) {
+    return false;
+  }
+  size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0;
+  size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0;
+  if (s1 != s2) { 
+    return false;
+  }
+  for (size_t i = 0; i < s1; i++) {
     FstTransition *t1 = taosArrayGet(n1->trans, i);
     FstTransition *t2 = taosArrayGet(n2->trans, i);
     if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) {
       return false;
     }  
   }
+  
   return true;
 }
 FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {

source/libs/index/test/indexTests.cpp

@@ -65,14 +65,14 @@
 int main(int argc, char** argv) {
   std::string str("abc");
   FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size());
-  Output   val = 10;
+  Output   val = 1;
 
-  std::string str1("bcd");
-  FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size());
-  Output   val2 = 10;
+  //std::string str1("bcd");
+  //FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size());
+  //Output   val2 = 10;
   FstBuilder *b = fstBuilderCreate(NULL, 1);
   fstBuilderInsert(b, key, val); 
-  fstBuilderInsert(b, key1, val2); 
+  //fstBuilderInsert(b, key1, val2); 
   fstBuilderFinish(b);
   fstBuilderDestroy(b);
   fstSliceDestroy(&key);