enh: add invert index to vnode

cmake/cmake.options

@@ -142,3 +142,4 @@ option(
    "If use invertedIndex" 
    ON 
 )
+

source/dnode/vnode/CMakeLists.txt

@@ -76,9 +76,14 @@ target_link_libraries(
     #PUBLIC scalar
     PUBLIC transport
     PUBLIC stream
+    PUBLIC index
 )
 target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
-
+if (${BUILD_WITH_INVERTEDINDEX})
+  add_definitions(-DUSE_INVERTED_INDEX) 
+endif(${BUILD_WITH_INVERTEDINDEX})
 if(${BUILD_TEST})
     add_subdirectory(test)
 endif(${BUILD_TEST})
+
+

source/dnode/vnode/src/inc/meta.h

@@ -17,6 +17,7 @@
 #define _TD_VNODE_META_H_
 
 #include "vnodeInt.h"
+#include "index.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -61,16 +62,20 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64()
 struct SMeta {
   TdThreadRwlock lock;
 
-  char*     path;
-  SVnode*   pVnode;
-  TDB*      pEnv;
-  TXN       txn;
-  TTB*      pTbDb;
-  TTB*      pSkmDb;
-  TTB*      pUidIdx;
-  TTB*      pNameIdx;
-  TTB*      pCtbIdx;
-  TTB*      pTagIdx;
+  char*   path;
+  SVnode* pVnode;
+  TDB*    pEnv;
+  TXN     txn;
+  TTB*    pTbDb;
+  TTB*    pSkmDb;
+  TTB*    pUidIdx;
+  TTB*    pNameIdx;
+  TTB*    pCtbIdx;
+#ifdef USE_INVERTED_INDEX
+  void* pTagIvtIdx;
+#else
+  TTB* pTagIdx;
+#endif
   TTB*      pTtlIdx;
   TTB*      pSmaIdx;
   SMetaIdx* pIdx;

source/dnode/vnode/src/meta/metaIdx.c

@@ -53,10 +53,10 @@ int metaOpenIdx(SMeta *pMeta) {
 #endif
 
 #ifdef USE_INVERTED_INDEX
-  SIndexOpts opts;
-  if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
-    return -1;
-  }
+  // SIndexOpts opts;
+  // if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
+  //  return -1;
+  //}
 
 #endif
   return 0;
@@ -71,36 +71,37 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */
 #endif
 
 #ifdef USE_INVERTED_INDEX
-  SIndexOpts opts;
-  if (indexClose(pMeta->pIdx->pIdx) != 0) {
-    return -1;
-  }
+  // SIndexOpts opts;
+  // if (indexClose(pMeta->pIdx->pIdx) != 0) {
+  //  return -1;
+  //}
+  // return 0;
 
 #endif
 }
 
 int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
 #ifdef USE_INVERTED_INDEX
-  if (pTbCfgs->type == META_CHILD_TABLE) {
-    char    buf[8] = {0};
-    int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
-    sprintf(buf, "%d", colId);  // colname
-
-    char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
-
-    tb_uid_t         suid = pTbCfg->ctbCfg.suid;  // super id
-    tb_uid_t         tuid = 0;                    // child table uid
-    SIndexMultiTerm *terms = indexMultiTermCreate();
-    SIndexTerm      *term =
-        indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
-    indexMultiTermAdd(terms, term);
-
-    int ret = indexPut(pMeta->pIdx->pIdx, terms);
-    indexMultiTermDestroy(terms);
-    return ret;
-  } else {
-    return DB_DONOTINDEX;
-  }
+  // if (pTbCfgs->type == META_CHILD_TABLE) {
+  //  char    buf[8] = {0};
+  //  int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
+  //  sprintf(buf, "%d", colId);  // colname
+
+  //  char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
+
+  //  tb_uid_t         suid = pTbCfg->ctbCfg.suid;  // super id
+  //  tb_uid_t         tuid = 0;                    // child table uid
+  //  SIndexMultiTerm *terms = indexMultiTermCreate();
+  //  SIndexTerm      *term =
+  //      indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
+  //  indexMultiTermAdd(terms, term);
+
+  //  int ret = indexPut(pMeta->pIdx->pIdx, terms);
+  //  indexMultiTermDestroy(terms);
+  //  return ret;
+  //} else {
+  //  return DB_DONOTINDEX;
+  //}
 #endif
   // TODO
   return 0;
@@ -112,4 +113,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
 #endif
   // TODO
   return 0;
-}
+}
\ No newline at end of file

source/dnode/vnode/src/meta/metaOpen.c

@@ -93,11 +93,24 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   }
 
   // open pTagIdx
+#ifdef USE_INVERTED_INDEX
+  // TODO(yihaoDeng), refactor later
+  char indexFullPath[128] = {0};
+  sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
+  taosMkDir(indexFullPath);
+  ret = indexOpen(indexOptsCreate(), indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
+  if (ret < 0) {
+    metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+#else
   ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
   if (ret < 0) {
     metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
+#endif
 
   // open pTtlIdx
   ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
@@ -128,7 +141,11 @@ _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
   if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
   if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+#ifdef USE_INVERTED_INDEX
+  if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
+#else
   if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
+#endif
   if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
   if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
   if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
@@ -145,7 +162,11 @@ int metaClose(SMeta *pMeta) {
     if (pMeta->pIdx) metaCloseIdx(pMeta);
     if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
     if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+#ifdef USE_INVERTED_INDEX
+    if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
+#else
     if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
+#endif
     if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
     if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
     if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);

source/dnode/vnode/src/meta/metaTable.c

@@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void       *pBuf = NULL;
+  void *      pBuf = NULL;
   int32_t     szBuf = 0;
-  void       *p = NULL;
+  void *      p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -71,9 +71,9 @@ _err:
 }
 
 int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
-  TBC        *pNameIdxc = NULL;
-  TBC        *pUidIdxc = NULL;
-  TBC        *pCtbIdxc = NULL;
+  TBC *       pNameIdxc = NULL;
+  TBC *       pUidIdxc = NULL;
+  TBC *       pCtbIdxc = NULL;
   SCtbIdxKey *pCtbIdxKey;
   const void *pKey = NULL;
   int         nKey;
@@ -134,8 +134,8 @@ _err:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  oStbEntry = {0};
   SMetaEntry  nStbEntry = {0};
-  TBC        *pUidIdxc = NULL;
-  TBC        *pTbDbc = NULL;
+  TBC *       pUidIdxc = NULL;
+  TBC *       pTbDbc = NULL;
   const void *pData;
   int         nData;
   int64_t     oversion;
@@ -256,9 +256,9 @@ _err:
 }
 
 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  TBC        *pTbDbc = NULL;
-  TBC        *pUidIdxc = NULL;
-  TBC        *pNameIdxc = NULL;
+  TBC *       pTbDbc = NULL;
+  TBC *       pUidIdxc = NULL;
+  TBC *       pNameIdxc = NULL;
   const void *pData;
   int         nData;
   tb_uid_t    uid;
@@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
 }
 
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void           *pVal = NULL;
+  void *          pVal = NULL;
   int             nVal = 0;
-  const void     *pData = NULL;
+  const void *    pData = NULL;
   int             nData = 0;
   int             ret = 0;
   tb_uid_t        uid;
   int64_t         oversion;
-  SSchema        *pColumn = NULL;
+  SSchema *       pColumn = NULL;
   SMetaEntry      entry = {0};
   SSchemaWrapper *pSchema;
   int             c;
@@ -530,7 +530,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
   SMetaEntry  ctbEntry = {0};
   SMetaEntry  stbEntry = {0};
-  void       *pVal = NULL;
+  void *      pVal = NULL;
   int         nVal = 0;
   int         ret;
   int         c;
@@ -561,7 +561,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   oversion = *(int64_t *)pData;
 
   // search table.db
-  TBC     *pTbDbc = NULL;
+  TBC *    pTbDbc = NULL;
   SDecoder dc = {0};
 
   /* get ctbEntry */
@@ -586,7 +586,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   tDecoderClear(&dc);
 
   SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema        *pColumn = NULL;
+  SSchema *       pColumn = NULL;
   int32_t         iCol = 0;
   for (;;) {
     pColumn = NULL;
@@ -678,8 +678,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
 
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void    *pKey = NULL;
-  void    *pVal = NULL;
+  void *   pKey = NULL;
+  void *   pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};
@@ -794,14 +794,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void          *pData = NULL;
+  void *         pData = NULL;
   int            nData = 0;
   STbDbKey       tbDbKey = {0};
   SMetaEntry     stbEntry = {0};
-  STagIdxKey    *pTagIdxKey = NULL;
+  STagIdxKey *   pTagIdxKey = NULL;
   int32_t        nTagIdxKey;
   const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void    *pTagData = NULL;  //
+  const void *   pTagData = NULL;  //
   SDecoder       dc = {0};
 
   // get super table
@@ -817,22 +817,33 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId);
 
   // update tag index
+#ifdef USE_INVERTED_INDEX
+  tb_uid_t suid = pCtbEntry->ctbEntry.suid;
+  tb_uid_t tuid = pCtbEntry->uid;
+
+  SIndexMultiTerm *tmGroup = indexMultiTermCreate();
+
+  SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name),
+                                   pTagData, pTagData == NULL ? 0 : strlen(pTagData));
+  indexMultiTermAdd(tmGroup, tm);
+  int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid);
+  indexMultiTermDestroy(tmGroup);
+#else
   if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid,
                           &pTagIdxKey, &nTagIdxKey) < 0) {
     return -1;
   }
   tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
   metaDestroyTagIdxKey(pTagIdxKey);
-
+#endif
   tDecoderClear(&dc);
   tdbFree(pData);
-
   return 0;
 }
 
 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   SEncoder              coder = {0};
-  void                 *pVal = NULL;
+  void *                pVal = NULL;
   int                   vLen = 0;
   int                   rcode = 0;
   SSkmDbKey             skmDbKey = {0};

source/libs/index/inc/indexInt.h

@@ -131,42 +131,14 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
 // int32_t indexSerialKey(ICacheKey* key, char* buf);
 // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 
-#define indexFatal(...)                                            \
-  do {                                                             \
-    if (sDebugFlag & DEBUG_FATAL) {                                \
-      taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
-    }                                                              \
-  } while (0)
-#define indexError(...)                                            \
-  do {                                                             \
-    if (sDebugFlag & DEBUG_ERROR) {                                \
-      taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
-    }                                                              \
-  } while (0)
-#define indexWarn(...)                                           \
-  do {                                                           \
-    if (sDebugFlag & DEBUG_WARN) {                               \
-      taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
-    }                                                            \
-  } while (0)
-#define indexInfo(...)                                      \
-  do {                                                      \
-    if (sDebugFlag & DEBUG_INFO) {                          \
-      taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \
-    }                                                       \
-  } while (0)
-#define indexDebug(...)                                             \
-  do {                                                              \
-    if (sDebugFlag & DEBUG_DEBUG) {                                 \
-      taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \
-    }                                                               \
-  } while (0)
-#define indexTrace(...)                                             \
-  do {                                                              \
-    if (sDebugFlag & DEBUG_TRACE) {                                 \
-      taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \
-    }                                                               \
-  } while (0)
+// clang-format off
+#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) {  taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0)
+#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) {  taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0)
+#define indexWarn(...)  do { if (sDebugFlag & DEBUG_WARN)  {  taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0)
+#define indexInfo(...)  do { if (sDebugFlag & DEBUG_INFO)  { taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0)
+#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0)
+#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0)
+// clang-format on
 
 #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
 

source/libs/index/src/indexCache.c

@@ -709,9 +709,6 @@ static bool indexCacheIteratorNext(Iterate* itera) {
     iv->type = ct->operaType;
     iv->ver = ct->version;
     iv->colVal = tstrdup(ct->colVal);
-    // printf("col Val: %s\n", iv->colVal);
-    // iv->colType = cv->colType;
-
     taosArrayPush(iv->val, &ct->uid);
   }
   return next;

source/libs/index/src/indexComm.c

@@ -22,6 +22,29 @@
 #include "ttypes.h"
 #include "tvariant.h"
 
+#define INDEX_DATA_BOOL_NULL      0x02
+#define INDEX_DATA_TINYINT_NULL   0x80
+#define INDEX_DATA_SMALLINT_NULL  0x8000
+#define INDEX_DATA_INT_NULL       0x80000000L
+#define INDEX_DATA_BIGINT_NULL    0x8000000000000000L
+#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
+
+#define INDEX_DATA_FLOAT_NULL    0x7FF00000           // it is an NAN
+#define INDEX_DATA_DOUBLE_NULL   0x7FFFFF0000000000L  // an NAN
+#define INDEX_DATA_NCHAR_NULL    0xFFFFFFFF
+#define INDEX_DATA_BINARY_NULL   0xFF
+#define INDEX_DATA_JSON_NULL     0xFFFFFFFF
+#define INDEX_DATA_JSON_null     0xFFFFFFFE
+#define INDEX_DATA_JSON_NOT_NULL 0x01
+
+#define INDEX_DATA_UTINYINT_NULL  0xFF
+#define INDEX_DATA_USMALLINT_NULL 0xFFFF
+#define INDEX_DATA_UINT_NULL      0xFFFFFFFF
+#define INDEX_DATA_UBIGINT_NULL   0xFFFFFFFFFFFFFFFFL
+
+#define INDEX_DATA_NULL_STR   "NULL"
+#define INDEX_DATA_NULL_STR_L "null"
+
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
@@ -372,7 +395,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
-      *dst = (char*) * dst - tlen;
+      *dst = (char*)*dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY