Merge pull request #14703 from taosdata/enh/addTestToIdx

Enh: add ut test to idx

source/libs/index/inc/indexFstCommon.h

@@ -4,6 +4,7 @@
 #include "tutil.h"
 extern const uint8_t COMMON_INPUTS[];
 extern const char    COMMON_INPUTS_INV[];
+extern const int32_t COMMON_INPUTS_LEN;
 
 #ifdef __cplusplus
 extern "C" {

source/libs/index/inc/indexFstDfa.h

@@ -29,16 +29,16 @@ extern "C" {
 typedef struct FstDfa FstDfa;
 
 typedef struct {
-  SArray * insts;
+  SArray  *insts;
   uint32_t next[256];
   bool     isMatch;
-} State;
+} DfaState;
 
 /*
  * dfa builder related func
  **/
 typedef struct FstDfaBuilder {
-  FstDfa *  dfa;
+  FstDfa   *dfa;
   SHashObj *cache;
 } FstDfaBuilder;
 

source/libs/index/inc/indexFstRegex.h

@@ -65,6 +65,7 @@ typedef struct {
 } FstRegex;
 
 FstRegex *regexCreate(const char *str);
+void      regexDestroy(FstRegex *regex);
 
 uint32_t regexAutomStart(FstRegex *regex);
 bool     regexAutomIsMatch(FstRegex *regex, uint32_t state);

source/libs/index/src/indexCache.c

@@ -22,7 +22,7 @@
 #define MAX_INDEX_KEY_LEN 256  // test only, change later
 
 #define MEM_TERM_LIMIT     10 * 10000
-#define MEM_THRESHOLD      64 * 1024
+#define MEM_THRESHOLD      512 * 1024
 #define MEM_SIGNAL_QUIT    MEM_THRESHOLD * 20
 #define MEM_ESTIMATE_RADIO 1.5
 
@@ -204,7 +204,6 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
     if (0 == strcmp(c->colVal, pCt->colVal)) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
@@ -309,7 +308,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)

source/libs/index/src/indexFst.c

@@ -1307,7 +1307,6 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
     taosArrayPush(sws->inp, &(trn.inp));
 
     if (FST_NODE_IS_FINAL(nextNode)) {
-      // void *eofState = sws->aut->acceptEof(nextState);
       void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
       if (eofState != NULL) {
         isMatch = automFuncs[aut->type].isMatch(aut, eofState);

source/libs/index/src/indexFstCommon.c

@@ -294,3 +294,4 @@ const char COMMON_INPUTS_INV[] = {
     '\xee', '\xef', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb',
     '\xfc', '\xfd', '\xfe', '\xff',
 };
+const int32_t COMMON_INPUTS_LEN = sizeof(COMMON_INPUTS) / sizeof(COMMON_INPUTS[0]);

source/libs/index/src/indexFstDfa.c

@@ -41,7 +41,7 @@ FstDfaBuilder *dfaBuilderCreate(SArray *insts) {
     return NULL;
   }
 
-  SArray *states = taosArrayInit(4, sizeof(State));
+  SArray *states = taosArrayInit(4, sizeof(DfaState));
 
   builder->dfa = dfaCreate(insts, states);
   builder->cache = taosHashInit(
@@ -71,7 +71,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
 
   dfaAdd(builder->dfa, cur, 0);
 
-  SArray * states = taosArrayInit(0, sizeof(uint32_t));
+  SArray  *states = taosArrayInit(0, sizeof(uint32_t));
   uint32_t result;
   if (dfaBuilderCachedState(builder, cur, &result)) {
     taosArrayPush(states, &result);
@@ -98,10 +98,12 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
   return builder->dfa;
 }
 
+FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; }
+
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
                         uint32_t *result) {
   sparSetClear(cur);
-  State *t = taosArrayGet(builder->dfa->states, state);
+  DfaState *t = taosArrayGet(builder->dfa->states, state);
   for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
     uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
     sparSetAdd(cur, ip);
@@ -144,7 +146,7 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *
     *result = *v;
     taosArrayDestroy(tinsts);
   } else {
-    State st;
+    DfaState st;
     st.insts = tinsts;
     st.isMatch = isMatch;
     taosArrayPush(builder->dfa->states, &st);
@@ -169,14 +171,14 @@ bool dfaIsMatch(FstDfa *dfa, uint32_t si) {
   if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
     return false;
   }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
   return st != NULL ? st->isMatch : false;
 }
 bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) {
   if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
     return false;
   }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
   *result = st->next[byte];
   return true;
 }

source/libs/index/src/indexFstRegex.c

@@ -23,7 +23,7 @@ FstRegex *regexCreate(const char *str) {
     return NULL;
   }
   int32_t sz = (int32_t)strlen(str);
-  char *  orig = taosMemoryCalloc(1, sz);
+  char   *orig = taosMemoryCalloc(1, sz);
   memcpy(orig, str, sz);
 
   regex->orig = orig;
@@ -36,6 +36,12 @@ FstRegex *regexCreate(const char *str) {
   return regex;
 }
 
+void regexDestroy(FstRegex *regex) {
+  if (regex == NULL) return;
+  taosMemoryFree(regex->orig);
+  taosMemoryFree(regex);
+}
+
 uint32_t regexAutomStart(FstRegex *regex) {
   ///// no nothing
   return 0;

source/libs/index/test/CMakeLists.txt

@@ -4,6 +4,7 @@ IF(NOT TD_DARWIN)
   add_executable(idxFstUT "")
   add_executable(idxUtilUT "")
   add_executable(idxJsonUT "")
+  add_executable(idxFstUtilUT "")
 
   target_sources(idxTest
     PRIVATE 
@@ -23,6 +24,25 @@ IF(NOT TD_DARWIN)
     "utilUT.cc" 
   )
 
+  target_sources(idxJsonUT
+    PRIVATE 
+    "jsonUT.cc" 
+  )
+  target_sources(idxFstUtilUT
+   PRIVATE 
+   "fstUtilUT.cc" 
+  )
+ 
+  target_include_directories (idxTest
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+  target_include_directories (idxFstTest 
+   PUBLIC
+  "${TD_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
   target_sources(idxJsonUT
     PRIVATE 
     "jsonUT.cc" 
@@ -50,6 +70,38 @@ IF(NOT TD_DARWIN)
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
   ) 
 
+  target_include_directories (idxJsonUT
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+  target_include_directories (idxFstUtilUT
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+
+  target_link_libraries (idxTest
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
+  target_link_libraries (idxFstTest
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
+  target_link_libraries (idxFstUT
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
   target_include_directories (idxJsonUT
     PUBLIC
     "${TD_SOURCE_DIR}/include/libs/index" 
@@ -92,15 +144,28 @@ IF(NOT TD_DARWIN)
     gtest_main
     index
   )
-
-  add_test(
-    NAME idxtest
-    COMMAND idxTest
+  target_link_libraries (idxFstUtilUT
+    os  
+    util
+    common
+    gtest_main
+    index
   )
+  
   add_test(
     NAME idxJsonUT
     COMMAND idxJsonUT 
   )
+  add_test(
+    NAME idxFstUtilUT 
+    COMMAND idxFstUtilUT  
+  
+  )
+
+  add_test(
+    NAME idxtest
+    COMMAND idxTest
+  )
   add_test(
     NAME idxUtilUT 
     COMMAND idxUtilUT 
@@ -109,4 +174,4 @@ IF(NOT TD_DARWIN)
     NAME idxFstUT 
     COMMAND idxFstUT 
   )
-ENDIF ()
\ No newline at end of file
+ENDIF ()

source/libs/index/test/fstUtilUT.cc

+
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstDfa.h"
+#include "indexFstRegex.h"
+#include "indexFstSparse.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "tskiplist.h"
+#include "tutil.h"
+class FstUtilEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    SArray *inst = taosArrayInit(4, sizeof(char));
+    builder = dfaBuilderCreate(inst);
+  }
+  virtual void TearDown() { dfaBuilderDestroy(builder); }
+
+  FstDfaBuilder *builder;
+};
+
+class FstRegexEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { regex = regexCreate("test"); }
+  virtual void TearDown() { regexDestroy(regex); }
+  FstRegex    *regex;
+};
+
+class FstSparseSetEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { set = sparSetCreate(256); }
+  virtual void TearDown() {
+    // tear down
+    sparSetDestroy(set);
+  }
+  void ReBuild(int32_t sz) {
+    sparSetDestroy(set);
+    set = sparSetCreate(sz);
+  }
+  FstSparseSet *set;
+};
+
+// test FstDfaBuilder
+TEST_F(FstUtilEnv, test1) {}
+TEST_F(FstUtilEnv, test2) {}
+TEST_F(FstUtilEnv, test3) {}
+TEST_F(FstUtilEnv, test4) {}
+
+// test FstRegex
+
+TEST_F(FstRegexEnv, test1) {}
+TEST_F(FstRegexEnv, test2) {}
+TEST_F(FstRegexEnv, test3) {}
+TEST_F(FstRegexEnv, test4) {}
+
+// test FstSparseSet
+TEST_F(FstSparseSetEnv, test1) {}
+TEST_F(FstSparseSetEnv, test2) {}
+TEST_F(FstSparseSetEnv, test3) {}
+TEST_F(FstSparseSetEnv, test4) {}

source/libs/index/test/utilUT.cc

@@ -8,6 +8,7 @@
 #include "indexCache.h"
 #include "indexComm.h"
 #include "indexFst.h"
+#include "indexFstCommon.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
@@ -356,3 +357,11 @@ TEST_F(UtilEnv, TempResultExcept) {
   idxTRsltMergeTo(relt, f);
   EXPECT_EQ(taosArrayGetSize(f), 1);
 }
+
+TEST_F(UtilEnv, testDictComm) {
+  int32_t count = COMMON_INPUTS_LEN;
+  for (int i = 0; i < 256; i++) {
+    uint8_t v = COMMON_INPUTS_INV[i];
+    EXPECT_EQ(COMMON_INPUTS[v], i);
+  }
+}