Merge pull request #14732 from taosdata/enh/addTestToIdx1

enh: add test to index

source/libs/index/inc/indexFstDfa.h

@@ -51,7 +51,7 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder);
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t bytes,
                         uint32_t *result);
 
-bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result);
+bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result);
 
 /*
  * dfa related func

source/libs/index/inc/indexFstSparse.h

@@ -23,17 +23,18 @@ extern "C" {
 #endif
 
 typedef struct FstSparseSet {
-  uint32_t *dense;
-  uint32_t *sparse;
-  int32_t   size;
+  int32_t *dense;
+  int32_t *sparse;
+  int32_t  size;
+  int32_t  cap;
 } FstSparseSet;
 
 FstSparseSet *sparSetCreate(int32_t sz);
 void          sparSetDestroy(FstSparseSet *s);
 uint32_t      sparSetLen(FstSparseSet *ss);
-uint32_t      sparSetAdd(FstSparseSet *ss, uint32_t ip);
-uint32_t      sparSetGet(FstSparseSet *ss, uint32_t i);
-bool          sparSetContains(FstSparseSet *ss, uint32_t ip);
+bool          sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *val);
+bool          sparSetGet(FstSparseSet *ss, int32_t i, int32_t *val);
+bool          sparSetContains(FstSparseSet *ss, int32_t ip);
 void          sparSetClear(FstSparseSet *ss);
 
 #ifdef __cplusplus

source/libs/index/src/indexFstDfa.c

@@ -64,7 +64,7 @@ void dfaBuilderDestroy(FstDfaBuilder *builder) {
   taosMemoryFree(builder);
 }
 
-FstDfa *dfaBuilder(FstDfaBuilder *builder) {
+FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
   uint32_t      sz = taosArrayGetSize(builder->dfa->insts);
   FstSparseSet *cur = sparSetCreate(sz);
   FstSparseSet *nxt = sparSetCreate(sz);
@@ -73,7 +73,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
 
   SArray  *states = taosArrayInit(0, sizeof(uint32_t));
   uint32_t result;
-  if (dfaBuilderCachedState(builder, cur, &result)) {
+  if (dfaBuilderCacheState(builder, cur, &result)) {
     taosArrayPush(states, &result);
   }
   SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@@ -98,22 +98,21 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
   return builder->dfa;
 }
 
-FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; }
-
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
                         uint32_t *result) {
   sparSetClear(cur);
   DfaState *t = taosArrayGet(builder->dfa->states, state);
   for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
-    uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
-    sparSetAdd(cur, ip);
+    int32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
+    bool    succ = sparSetAdd(cur, ip, NULL);
+    assert(succ == true);
   }
   dfaRun(builder->dfa, cur, next, byte);
 
   t = taosArrayGet(builder->dfa->states, state);
 
   uint32_t nxtState;
-  if (dfaBuilderCachedState(builder, next, &nxtState)) {
+  if (dfaBuilderCacheState(builder, next, &nxtState)) {
     t->next[byte] = nxtState;
     *result = nxtState;
     return true;
@@ -121,12 +120,13 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet
   return false;
 }
 
-bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
+bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
   SArray *tinsts = taosArrayInit(4, sizeof(uint32_t));
   bool    isMatch = false;
 
   for (int i = 0; i < sparSetLen(set); i++) {
-    uint32_t ip = sparSetGet(set, i);
+    int32_t ip;
+    if (false == sparSetGet(set, i, &ip)) continue;
 
     Inst *inst = taosArrayGet(builder->dfa->insts, ip);
     if (inst->ty == JUMP || inst->ty == SPLIT) {
@@ -186,7 +186,8 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) {
   if (sparSetContains(set, ip)) {
     return;
   }
-  sparSetAdd(set, ip);
+  bool succ = sparSetAdd(set, ip, NULL);
+  // assert(succ == true);
   Inst *inst = taosArrayGet(dfa->insts, ip);
   if (inst->ty == MATCH || inst->ty == RANGE) {
     // do nothing
@@ -203,7 +204,8 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) {
   bool isMatch = false;
   sparSetClear(to);
   for (int i = 0; i < sparSetLen(from); i++) {
-    uint32_t ip = sparSetGet(from, i);
+    int32_t ip;
+    if (false == sparSetGet(from, i, &ip)) continue;
 
     Inst *inst = taosArrayGet(dfa->insts, ip);
     if (inst->ty == JUMP || inst->ty == SPLIT) {

source/libs/index/src/indexFstRegex.c

@@ -22,15 +22,15 @@ FstRegex *regexCreate(const char *str) {
   if (regex == NULL) {
     return NULL;
   }
-  int32_t sz = (int32_t)strlen(str);
-  char   *orig = taosMemoryCalloc(1, sz);
-  memcpy(orig, str, sz);
 
-  regex->orig = orig;
+  regex->orig = tstrdup(str);
 
   // construct insts based on str
-  SArray *insts = NULL;
-
+  SArray *insts = taosArrayInit(256, sizeof(uint8_t));
+  for (int i = 0; i < strlen(str); i++) {
+    uint8_t v = str[i];
+    taosArrayPush(insts, &v);
+  }
   FstDfaBuilder *builder = dfaBuilderCreate(insts);
   regex->dfa = dfaBuilderBuild(builder);
   return regex;

source/libs/index/src/indexFstSparse.c

@@ -15,14 +15,24 @@
 
 #include "indexFstSparse.h"
 
+static void sparSetUtil(int32_t *buf, int32_t cap) {
+  for (int32_t i = 0; i < cap; i++) {
+    buf[i] = -1;
+  }
+}
 FstSparseSet *sparSetCreate(int32_t sz) {
   FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
   if (ss == NULL) {
     return NULL;
   }
 
-  ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
-  ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
+  ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t));
+  ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t));
+  sparSetUtil(ss->dense, sz);
+  sparSetUtil(ss->sparse, sz);
+
+  ss->cap = sz;
+
   ss->size = 0;
   return ss;
 }
@@ -38,23 +48,39 @@ uint32_t sparSetLen(FstSparseSet *ss) {
   // Get occupied size
   return ss == NULL ? 0 : ss->size;
 }
-uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) {
+bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *idx) {
   if (ss == NULL) {
-    return 0;
+    return false;
+  }
+  if (ip >= ss->cap || ip < 0) {
+    return false;
   }
   uint32_t i = ss->size;
   ss->dense[i] = ip;
   ss->sparse[ip] = i;
   ss->size += 1;
-  return i;
+
+  if (idx != NULL) *idx = i;
+
+  return true;
 }
-uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) {
-  // check later
-  return ss->dense[i];
+bool sparSetGet(FstSparseSet *ss, int32_t idx, int32_t *ip) {
+  if (idx >= ss->cap || idx >= ss->size || idx < 0) {
+    return false;
+  }
+  int32_t val = ss->dense[idx];
+  if (ip != NULL) {
+    *ip = val;
+  }
+  return val == -1 ? false : true;
 }
-bool sparSetContains(FstSparseSet *ss, uint32_t ip) {
-  uint32_t i = ss->sparse[ip];
-  if (i < ss->size && ss->dense[i] == ip) {
+bool sparSetContains(FstSparseSet *ss, int32_t ip) {
+  if (ip >= ss->cap || ip < 0) {
+    return false;
+  }
+  int32_t i = ss->sparse[ip];
+
+  if (i >= 0 && i < ss->cap && i < ss->size && ss->dense[i] == ip) {
     return true;
   } else {
     return false;
@@ -64,5 +90,7 @@ void sparSetClear(FstSparseSet *ss) {
   if (ss == NULL) {
     return;
   }
+  sparSetUtil(ss->dense, ss->cap);
+  sparSetUtil(ss->sparse, ss->cap);
   ss->size = 0;
 }

source/libs/index/test/fstUtilUT.cc

@@ -51,20 +51,62 @@ class FstSparseSetEnv : public ::testing::Test {
 };
 
 // test FstDfaBuilder
-TEST_F(FstUtilEnv, test1) {}
-TEST_F(FstUtilEnv, test2) {}
-TEST_F(FstUtilEnv, test3) {}
-TEST_F(FstUtilEnv, test4) {}
+TEST_F(FstUtilEnv, test1) {
+  // test
+}
+TEST_F(FstUtilEnv, test2) {
+  // test
+}
+TEST_F(FstUtilEnv, test3) {
+  // test
+}
+TEST_F(FstUtilEnv, test4) {
+  // test
+}
 
 // test FstRegex
 
-TEST_F(FstRegexEnv, test1) {}
+TEST_F(FstRegexEnv, test1) {
+  //
+  EXPECT_EQ(regex != NULL, true);
+}
 TEST_F(FstRegexEnv, test2) {}
 TEST_F(FstRegexEnv, test3) {}
 TEST_F(FstRegexEnv, test4) {}
 
 // test FstSparseSet
-TEST_F(FstSparseSetEnv, test1) {}
-TEST_F(FstSparseSetEnv, test2) {}
-TEST_F(FstSparseSetEnv, test3) {}
-TEST_F(FstSparseSetEnv, test4) {}
+TEST_F(FstSparseSetEnv, test1) {
+  for (int8_t i = 0; i < 20; i++) {
+    int32_t val = -1;
+    bool    succ = sparSetAdd(set, 'a' + i, &val);
+  }
+  EXPECT_EQ(sparSetLen(set), 20);
+  for (int8_t i = 0; i < 20; i++) {
+    int  val = -1;
+    bool find = sparSetGet(set, i, &val);
+    EXPECT_EQ(find, true);
+    EXPECT_EQ(val, i + 'a');
+  }
+  for (int8_t i = 'a'; i < 'a' + 20; i++) {
+    EXPECT_EQ(sparSetContains(set, i), true);
+  }
+
+  for (int8_t i = 'A'; i < 20; i++) {
+    EXPECT_EQ(sparSetContains(set, 'A'), false);
+  }
+
+  for (int i = 512; i < 1000; i++) {
+    EXPECT_EQ(sparSetAdd(set, i, NULL), false);
+
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+    EXPECT_EQ(sparSetContains(set, i), false);
+  }
+  sparSetClear(set);
+
+  for (int i = 'a'; i < 'a' + 20; i++) {
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+  }
+  for (int i = 1000; i < 2000; i++) {
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+  }
+}