Merge pull request #13480 from taosdata/enh/json_index

Enh/json index

cmake/cmake.options

@@ -146,6 +146,6 @@ option(
 option(
    BUILD_WITH_INVERTEDINDEX
    "If use invertedIndex" 
-   OFF 
+   ON 
 )
 

include/client/taos.h

@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
 typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
-  void     *buffer;
+  void *    buffer;
   uintptr_t buffer_length;
-  int32_t  *length;
-  char     *is_null;
+  int32_t * length;
+  char *    is_null;
   int       num;
 } TAOS_MULTI_BIND;
 
@@ -157,7 +157,7 @@ DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
 DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char *    taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
@@ -179,7 +179,7 @@ DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
-DLL_EXPORT int        *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
+DLL_EXPORT int *       taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
 DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
-DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
+DLL_EXPORT char **     tmq_list_to_c_array(const tmq_list_t *);
 
 DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
+DLL_EXPORT TAOS_RES *     tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
@@ -260,7 +260,7 @@ enum tmq_conf_res_t {
 
 typedef enum tmq_conf_res_t tmq_conf_res_t;
 
-DLL_EXPORT tmq_conf_t    *tmq_conf_new();
+DLL_EXPORT tmq_conf_t *   tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);

include/libs/index/index.h

@@ -193,8 +193,9 @@ void indexInit();
 
 /* index filter */
 typedef struct SIndexMetaArg {
-  void*    metaHandle;
   void*    metaEx;
+  void*    idx;
+  void*    ivtIdx;
   uint64_t suid;
 } SIndexMetaArg;
 

source/dnode/vnode/inc/vnode.h

@@ -56,7 +56,7 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
 int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
 int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
-int32_t vnodePreprocessQueryMsg(SVnode * pVnode, SRpcMsg * pMsg);
+int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
@@ -86,7 +86,7 @@ typedef struct SMetaFltParam {
   tb_uid_t suid;
   int16_t  cid;
   int16_t  type;
-  char    *val;
+  char *   val;
   bool     reverse;
   int (*filterFunc)(void *a, void *b, int16_t type);
 
@@ -121,7 +121,8 @@ int32_t      tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo
 bool         isTsdbCacheLastRow(tsdbReaderT *pReader);
 int32_t      tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
 int32_t      tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
-void        *tsdbGetIdx(SMeta *pMeta);
+void *       tsdbGetIdx(SMeta *pMeta);
+void *       tsdbGetIvtIdx(SMeta *pMeta);
 int64_t      tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
 
 bool    tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
@@ -197,7 +198,7 @@ struct SMetaEntry {
   int64_t  version;
   int8_t   type;
   tb_uid_t uid;
-  char    *name;
+  char *   name;
   union {
     struct {
       SSchemaWrapper schemaRow;
@@ -225,17 +226,17 @@ struct SMetaEntry {
 
 struct SMetaReader {
   int32_t    flags;
-  SMeta     *pMeta;
+  SMeta *    pMeta;
   SDecoder   coder;
   SMetaEntry me;
-  void      *pBuf;
+  void *     pBuf;
   int32_t    szBuf;
 };
 
 struct SMTbCursor {
-  TBC        *pDbc;
-  void       *pKey;
-  void       *pVal;
+  TBC *       pDbc;
+  void *      pKey;
+  void *      pVal;
   int32_t     kLen;
   int32_t     vLen;
   SMetaReader mr;

source/dnode/vnode/src/inc/meta.h

@@ -69,12 +69,11 @@ struct SMeta {
   TTB*    pUidIdx;
   TTB*    pNameIdx;
   TTB*    pCtbIdx;
-#ifdef USE_INVERTED_INDEX
+  // ivt idx and idx
   void* pTagIvtIdx;
-#else
   TTB*  pTagIdx;
-#endif
   TTB*  pTtlIdx;
+
   TTB*      pSmaIdx;
   SMetaIdx* pIdx;
 };

source/dnode/vnode/src/inc/vnodeInt.h

@@ -104,6 +104,7 @@ int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppRea
 int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
 int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
+void*           metaGetIvtIdx(SMeta* pMeta);
 
 int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);

source/dnode/vnode/src/meta/metaOpen.c

@@ -93,7 +93,6 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   }
 
   // open pTagIdx
-#ifdef USE_INVERTED_INDEX
   // TODO(yihaoDeng), refactor later
   char indexFullPath[128] = {0};
   sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
@@ -104,13 +103,11 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
     goto _err;
   }
 
-#else
   ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
-#endif
 
   // open pTtlIdx
   ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
@@ -141,11 +138,8 @@ _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
   if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
   if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
-#ifdef USE_INVERTED_INDEX
   if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
-#else
   if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
-#endif
   if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
   if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
   if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);

source/dnode/vnode/src/meta/metaSma.c

@@ -28,9 +28,9 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void       *pBuf = NULL;
+  void *      pBuf = NULL;
   int32_t     szBuf = 0;
-  void       *p = NULL;
+  void *      p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -83,8 +83,8 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
 
 static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void    *pKey = NULL;
-  void    *pVal = NULL;
+  void *   pKey = NULL;
+  void *   pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};

source/dnode/vnode/src/meta/metaTable.c

@@ -15,6 +15,7 @@
 
 #include "meta.h"
 
+static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -25,7 +26,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
 
-static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
+static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
   pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema));
   if (NULL == pMetaRsp->pSchemas) {
     terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
@@ -43,6 +44,68 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema
   return 0;
 }
 
+static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
+#ifdef USE_INVERTED_INDEX
+  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
+    return -1;
+  }
+  void *      data = pCtbEntry->ctbEntry.pTags;
+  const char *tagName = pSchema->name;
+
+  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
+  tb_uid_t    tuid = pCtbEntry->uid;
+  const void *pTagData = pCtbEntry->ctbEntry.pTags;
+  int32_t     nTagData = 0;
+
+  SArray *pTagVals = NULL;
+  if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
+    return -1;
+  }
+  char key[512] = {0};
+
+  SIndexMultiTerm *terms = indexMultiTermCreate();
+  int16_t          nCols = taosArrayGetSize(pTagVals);
+  for (int i = 0; i < nCols; i++) {
+    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
+    char     type = pTagVal->type;
+    sprintf(key, "%s_%s", tagName, pTagVal->pKey);
+    int32_t nKey = strlen(key);
+
+    SIndexTerm *term = NULL;
+    if (type == TSDB_DATA_TYPE_NULL) {
+      // handle null value
+    } else if (type == TSDB_DATA_TYPE_NCHAR) {
+      if (pTagVal->nData > 0) {
+        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
+        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
+        type = TSDB_DATA_TYPE_VARCHAR;
+        term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len);
+      } else if (pTagVal->nData == 0) {
+        char *  val = NULL;
+        int32_t len = 0;
+        // handle NULL key
+      }
+    } else if (type == TSDB_DATA_TYPE_DOUBLE) {
+      double val = *(double *)(&pTagVal->i64);
+      int    len = 0;
+      term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
+    } else if (type == TSDB_DATA_TYPE_BOOL) {
+      int val = *(int *)(&pTagVal->i64);
+      int len = 0;
+      term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
+    }
+    if (term != NULL) {
+      indexMultiTermAdd(terms, term);
+    }
+    memset(key, 0, sizeof(key));
+  }
+  tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
+  indexMultiTermDestroy(terms);
+#endif
+  return -1;
+}
+
 int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  me = {0};
   int         kLen = 0;
@@ -341,7 +404,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   return 0;
 }
 
-
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
   void *          pVal = NULL;
   int             nVal = 0;
@@ -824,28 +886,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   } else {
     // pTagData = pCtbEntry->ctbEntry.pTags;
     // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
+    pTagData = pCtbEntry->ctbEntry.pTags;
+    nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
+    return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn);
   }
-
-  // update tag index
-#ifdef USE_INVERTED_INDEX
-  tb_uid_t suid = pCtbEntry->ctbEntry.suid;
-  tb_uid_t tuid = pCtbEntry->uid;
-
-  SIndexMultiTerm *tmGroup = indexMultiTermCreate();
-
-  SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name),
-                                   pTagData, pTagData == NULL ? 0 : strlen(pTagData));
-  indexMultiTermAdd(tmGroup, tm);
-  int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid);
-  indexMultiTermDestroy(tmGroup);
-#else
   if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
                           pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
     return -1;
   }
   tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
   metaDestroyTagIdxKey(pTagIdxKey);
-#endif
   tDecoderClear(&dc);
   tdbFree(pData);
   return 0;
@@ -930,10 +980,5 @@ _err:
   return -1;
 }
 // refactor later
-void *metaGetIdx(SMeta *pMeta) {
-#ifdef USE_INVERTED_INDEX
-  return pMeta->pTagIvtIdx;
-#else
-  return pMeta->pTagIdx;
-#endif
-}
+void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
+void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2834,6 +2834,12 @@ void* tsdbGetIdx(SMeta* pMeta) {
   }
   return metaGetIdx(pMeta);
 }
+void* tsdbGetIvtIdx(SMeta* pMeta) {
+  if (pMeta == NULL) {
+    return NULL;
+  }
+  return metaGetIvtIdx(pMeta);
+}
 int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
   SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid);
 

source/libs/executor/src/executorimpl.c

@@ -4797,7 +4797,8 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa
 
   if (tableType == TSDB_SUPER_TABLE) {
     if (pTagCond) {
-      SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid};
+      SIndexMetaArg metaArg = {
+          .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
 
       SArray* res = taosArrayInit(8, sizeof(uint64_t));
       code = doFilterTag(pTagCond, &metaArg, res);

source/libs/index/src/index.c

@@ -202,7 +202,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
     char      buf[128] = {0};
     ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
     int32_t   sz = indexSerialCacheKey(&key, buf);
-    indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
+    indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
 
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
@@ -330,7 +330,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   char      buf[128] = {0};
   ICacheKey key = {
       .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
-  indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
+  indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
   int32_t sz = indexSerialCacheKey(&key, buf);
 
   taosThreadMutexLock(&sIdx->mtx);

source/libs/index/src/indexComm.c

@@ -402,16 +402,16 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, strlen(src));
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
       *dst = (char*)*dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARBINARY:
-      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, strlen(src));
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
       *dst = (char*)*dst - tlen;
       break;
     default:

source/libs/index/src/indexFilter.c

@@ -31,7 +31,7 @@ typedef struct SIFParam {
   SHashObj *pFilter;
 
   SArray *result;
-  char   *condValue;
+  char *  condValue;
 
   SIdxFltStatus status;
   uint8_t       colValType;
@@ -45,7 +45,7 @@ typedef struct SIFParam {
 
 typedef struct SIFCtx {
   int32_t       code;
-  SHashObj     *pRes;    /* element is SIFParam */
+  SHashObj *    pRes;    /* element is SIFParam */
   bool          noExec;  // true: just iterate condition tree, and add hint to executor plan
   SIndexMetaArg arg;
   // SIdxFltStatus st;
@@ -128,7 +128,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
   // covert data From snode;
   SValueNode *vn = (SValueNode *)node;
 
-  char      *pData = nodesGetValueFromNode(vn);
+  char *     pData = nodesGetValueFromNode(vn);
   SDataType *pType = &vn->node.resType;
   int32_t    type = pType->type;
   int32_t    valLen = 0;
@@ -162,12 +162,27 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
+  SOperatorNode *nd = (SOperatorNode *)node;
+  assert(nodeType(node) == QUERY_NODE_OPERATOR);
+  SColumnNode *l = (SColumnNode *)nd->pLeft;
+  SValueNode * r = (SValueNode *)nd->pRight;
+
+  param->colId = l->colId;
+  param->colValType = l->node.resType.type;
+  memcpy(param->dbName, l->dbName, sizeof(l->dbName));
+  sprintf(param->colName, "%s_%s", l->colName, r->literal);
+  param->colValType = r->typeData;
+  return 0;
+  // memcpy(param->colName, l->colName, sizeof(l->colName));
+}
 static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   switch (nodeType(node)) {
     case QUERY_NODE_VALUE: {
       SValueNode *vn = (SValueNode *)node;
       SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
       param->colId = -1;
+      param->colValType = (uint8_t)(vn->node.resType.type);
       break;
     }
     case QUERY_NODE_COLUMN: {
@@ -219,17 +234,31 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
     indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
     SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
+  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+    return code;
+  }
   SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
   if (NULL == paramList) {
     SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) {
+    SNode *interNode = (node->pLeft);
+    SIF_ERR_JRET(sifInitJsonParam(interNode, &paramList[0], ctx));
+    if (nParam > 1) {
+      SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
+    }
+    paramList[0].colValType = TSDB_DATA_TYPE_JSON;
+    *params = paramList;
+    return TSDB_CODE_SUCCESS;
+  } else {
     SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
     if (nParam > 1) {
       SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
     }
     *params = paramList;
     return TSDB_CODE_SUCCESS;
+  }
 _return:
   taosMemoryFree(paramList);
   SIF_RET(code);
@@ -306,25 +335,21 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
   SIndexMetaArg *arg = &output->arg;
-#ifdef USE_INVERTED_INDEX
-  SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
+  int            ret = 0;
+
+  EIndexQueryType qtype = 0;
+  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
+  if (left->colValType == TSDB_DATA_TYPE_JSON) {
+    SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName),
                                      right->condValue, strlen(right->condValue));
     if (tm == NULL) {
       return TSDB_CODE_QRY_OUT_OF_MEMORY;
     }
 
-  EIndexQueryType qtype = 0;
-  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
-
     SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
     indexMultiTermQueryAdd(mtm, tm, qtype);
-  int ret = indexSearch(arg->metaHandle, mtm, output->result);
-  indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result));
-  indexMultiTermQueryDestroy(mtm);
-  return ret;
-#else
-  EIndexQueryType qtype = 0;
-  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
+    ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result);
+  } else {
     bool   reverse;
     Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
 
@@ -335,10 +360,9 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
                            .reverse = reverse,
                            .filterFunc = filterFunc};
 
-  int ret = metaFilteTableIds(arg->metaEx, &param, output->result);
+    ret = metaFilteTableIds(arg->metaEx, &param, output->result);
+  }
   return ret;
-#endif
-  return 0;
 }
 
 static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
@@ -392,6 +416,14 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output
   int id = OP_TYPE_NMATCH;
   return sifDoIndex(left, right, id, output);
 }
+static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
+  // return 0
+  return 0;
+}
+static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
+  // return 0
+  return 0;
+}
 
 static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   // add more except
@@ -445,6 +477,14 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
       *status = SFLT_NOT_INDEX;
       *func = sifNotMatchFunc;
       return 0;
+    case OP_TYPE_JSON_CONTAINS:
+      *status = SFLT_ACCURATE_INDEX;
+      *func = sifJsonContains;
+      return 0;
+    case OP_TYPE_JSON_GET_VALUE:
+      *status = SFLT_ACCURATE_INDEX;
+      *func = sifJsonGetValue;
+      return 0;
     default:
       *status = SFLT_NOT_INDEX;
       *func = sifNullFunc;
@@ -457,12 +497,17 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
 static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
   int32_t code = 0;
   int32_t nParam = sifGetOperParamNum(node->opType);
+  if (nParam <= 1) {
+    SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+    return code;
+  }
   SIFParam *params = NULL;
-  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
 
+  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
   // ugly code, refactor later
   output->arg = ctx->arg;
-
   sif_func_t operFn = sifNullFunc;
   code = sifGetOperFn(node->opType, &operFn, &output->status);
   if (!ctx->noExec) {
@@ -567,7 +612,9 @@ EDealRes sifCalcWalker(SNode *node, void *context) {
   if (QUERY_NODE_LOGIC_CONDITION == nodeType(node)) {
     return sifWalkLogic(node, ctx);
   }
+
   if (QUERY_NODE_OPERATOR == nodeType(node)) {
+    indexInfo("node type for index filter, type: %d", nodeType(node));
     return sifWalkOper(node, ctx);
   }
 
@@ -658,7 +705,7 @@ int32_t doFilterTag(const SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *re
   // todo move to the initialization function
   // SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
 
-  SArray  *output = taosArrayInit(8, sizeof(uint64_t));
+  SArray * output = taosArrayInit(8, sizeof(uint64_t));
   SIFParam param = {.arg = *metaArg, .result = output};
   SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, &param));
 

source/libs/index/test/indexTests.cc

@@ -500,8 +500,13 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
 
   std::string colName("voltage");
   std::string colVal("ab");
-  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                     colVal.c_str(), colVal.size());
+
+  char    buf[256] = {0};
+  int16_t sz = colVal.size();
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal.c_str(), colVal.size());
+  SIndexTerm* term =
+      indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), buf, sizeof(buf));
   SIndexTermQuery query = {term, QUERY_TERM};
 
   SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@@ -564,6 +569,18 @@ class IndexCacheEnv : public ::testing::Test {
   CacheObj* coj;
 };
 
+SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
+                             int32_t nColName, const char* colVal, int32_t nColVal) {
+  char    buf[256] = {0};
+  int16_t sz = nColVal;
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal, nColVal);
+  if (colType == TSDB_DATA_TYPE_BINARY) {
+    return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+  } else {
+    return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+  }
+}
 #define MAX_TERM_KEY_LEN 128
 TEST_F(IndexCacheEnv, cache_test) {
   int     version = 0;
@@ -574,7 +591,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::string colName("voltage");
   {
     std::string colVal("v1");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -582,28 +599,28 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v2");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -612,14 +629,14 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::cout << "--------first----------" << std::endl;
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v4");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
@@ -630,7 +647,7 @@ TEST_F(IndexCacheEnv, cache_test) {
     std::string colVal("v4");
     for (size_t i = 0; i < 10; i++) {
       colVal[colVal.size() - 1] = 'a' + i;
-      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                           colVal.c_str(), colVal.size());
       coj->Put(term, colId, version++, suid++);
       indexTermDestroy(term);
@@ -640,7 +657,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   // begin query
   {
     std::string     colVal("v3");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -655,7 +672,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string     colVal("v2");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -690,7 +707,7 @@ class IndexObj {
     return ret;
   }
   void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
-    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -699,7 +716,7 @@ class IndexObj {
   }
   int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                       size_t numOfTable = 100 * 10000) {
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -721,7 +738,7 @@ class IndexObj {
         // opt
         tColVal[taosRand() % colValSize] = 'a' + k % 26;
       }
-      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                           tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
@@ -757,7 +774,7 @@ class IndexObj {
 
   int SearchOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -779,7 +796,7 @@ class IndexObj {
   }
   int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -804,7 +821,7 @@ class IndexObj {
 
   void PutOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, 10);
@@ -812,7 +829,7 @@ class IndexObj {
   }
   void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, val);
@@ -858,7 +875,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -873,7 +890,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -888,7 +905,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -903,7 +920,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string           colName("tag1"), colVal("Hello");
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -926,7 +943,7 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);

source/libs/index/test/jsonUT.cc

@@ -19,6 +19,18 @@
 static std::string dir = TD_TMP_DIR_PATH "json";
 static std::string logDir = TD_TMP_DIR_PATH "log";
 
+SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
+                             int32_t nColName, const char* colVal, int32_t nColVal) {
+  char    buf[256] = {0};
+  int16_t sz = nColVal;
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal, nColVal);
+  if (colType == TSDB_DATA_TYPE_BINARY) {
+    return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+  } else {
+    return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+  }
+}
 static void initLog() {
   const char*   defaultLogFileNamePrefix = "taoslog";
   const int32_t maxLogFileNum = 10;
@@ -59,8 +71,8 @@ class JsonEnv : public ::testing::Test {
 
 static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
                       int8_t operType = ADD_VALUE) {
-  SIndexTerm* term =
-      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexTerm*      term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
+                                      (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
   tIndexJsonPut(index, terms, (int64_t)tableId);
@@ -70,8 +82,8 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
 
 static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
                     int8_t operType = DEL_VALUE) {
-  SIndexTerm* term =
-      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexTerm*      term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
+                                      (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
   tIndexJsonPut(index, terms, (int64_t)tableId);
@@ -83,7 +95,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v
   std::string colName(colNam);
 
   SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-  SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
 
   SArray* res = taosArrayInit(1, sizeof(uint64_t));
   indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
@@ -95,7 +107,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -108,7 +120,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("ab1");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -121,7 +133,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("123");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -136,7 +148,7 @@ TEST_F(JsonEnv, testWrite) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                      colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -150,7 +162,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -165,7 +177,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("abxxxxxxxxxxxx");
     for (int i = 0; i < 10; i++) {
       colVal[i % colVal.size()] = '0' + i % 128;
-      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                           colVal.c_str(), colVal.size());
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -179,7 +191,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("voltagefdadfa");
     std::string colVal("abxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -194,7 +206,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                      colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -209,7 +221,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
       std::string colVal("ab");
 
       SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-      SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
       SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -224,7 +236,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
         std::string colVal("ab");
 
         SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-        SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+        SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
 
         SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -241,7 +253,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     std::string colName("test");
     // std::string colVal("10");
     int         val = 10;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -254,7 +266,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test2");
     int         val = 20;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -267,7 +279,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test");
     int         val = 15;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -280,7 +292,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test2");
     const char* val = "test";
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         (const char*)val, strlen(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -294,7 +306,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     std::string           colName("test");
     int                   val = 15;
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -308,7 +320,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     int         val = 15;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -323,7 +335,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     ;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -338,7 +350,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -353,7 +365,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -368,7 +380,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
     int         val = 10;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -381,7 +393,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test");
     std::string colVal("xxxxxxxxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -396,7 +408,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -410,7 +422,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -425,7 +437,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -439,7 +451,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -453,7 +465,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -466,7 +478,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     std::string colName("other_column");
     int         val = 100;
 
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -482,7 +494,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -494,7 +506,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
     int         val = 15;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                         (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -510,7 +522,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
                                      (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));