diff --git a/cmake/cmake.options b/cmake/cmake.options index edaab3bd45fe5bf63cf3514c0fc5bec07f076856..e44a38b3f516352ee21ceaa3a51e5580ac664d0a 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -37,6 +37,7 @@ option( off ) + option( BUILD_WITH_NURAFT "If build with NuRaft" @@ -54,3 +55,9 @@ option( "If use doxygen build documents" OFF ) + +option( + USE_INVERTEDINDEX + "If use invertedIndex" + ON +) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0885ce151ef1cf13e4e6c3b408440d0fdd7f135c..2cc2aeb6a0ce1bdf18302fcf0e33c0e10a14d5d7 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -24,6 +24,7 @@ extern "C" { #endif typedef struct SIndex SIndex; +typedef struct SIndexTerm SIndexTerm; typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; @@ -35,7 +36,7 @@ typedef enum { ADD_INDEX, // add index on specify column DROP_INDEX, // drop existed index DROP_SATBLE // drop stable -} SIndexColumnType; +} SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; @@ -45,7 +46,7 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = */ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type); +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); /* * @param: * @param: @@ -61,8 +62,8 @@ int indexRebuild(SIndex *index, SIndexOpts *opt); * @param */ SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, const char *field, int32_t nFields, const char *value, int32_t nValue); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); +void indexMultiTermDestroy(SIndexMultiTerm *terms); /* * @param: * @param: @@ -70,6 +71,17 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms); SIndexOpts *indexOptsCreate(); void indexOptsDestroy(SIndexOpts *opts); + +/* + * @param: + * @param: + */ + +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, + const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); +void indexTermDestroy(SIndexTerm *p); + + #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index cc740826e9add90f9ac9d5cf28cf6bb0989e2d49..fb5a9e40b5dea446e4d89b2836e5a50d1c52f75f 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -60,22 +60,21 @@ struct SIndexMultiTermQuery { // field and key; typedef struct SIndexTerm { - uint8_t type; // term data type, str/interger/json - char *key; - int32_t nKey; - char *val; - int32_t nVal; + int64_t suid; + SIndexOperOnColumn operType; // oper type, add/del/update + uint8_t colType; // term data type, str/interger/json + char *colName; + int32_t nColName; + char *colVal; + int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { - SIndexTerm* field_value; - EIndexQueryType type; + SIndexTerm* term; + EIndexQueryType qType; } SIndexTermQuery; -SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal); -void indexTermDestroy(SIndexTerm *p); - #define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) #define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 39107a78aca3e38d4417c291afef64ad1a1f3db7..b952e16a8e0929d61595f1d833e045b261acfa74 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -38,13 +38,13 @@ typedef struct IndexCache { // IndexCache *indexCacheCreate(); -void indexCacheDestroy(IndexCache *cache); +void indexCacheDestroy(void *cache); -int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, +int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, uint32_t version, uint64_t uid, int8_t operType); -int indexCacheGet(IndexCache *cache, uint64_t *rst); -int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result); +int indexCacheGet(void *cache, uint64_t *rst); +int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result); #ifdef __cplusplus } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 6a2697491d4b570d80842e2ee2e4394193663b23..ca6c2062f144bd24642019d6ac7f3ad520219ac6 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -46,7 +46,7 @@ SIndex *indexOpen(SIndexOpts *opts, const char *path) { index_t *index = index_open(path); sIdx->index = index; #endif - + sIdx->cache = (void*)indexCacheCreate(); sIdx->tindex = NULL; sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); @@ -61,9 +61,12 @@ void indexClose(SIndex *sIdx) { index_close(sIdex->index); sIdx->index = NULL; #endif + +#ifdef USE_INVERTEDINDEX indexCacheDestroy(sIdx->cache); taosHashCleanup(sIdx->fieldObj); pthread_mutex_destroy(&sIdx->mtx); +#endif free(sIdx); return; } @@ -86,6 +89,7 @@ int indexPut(SIndex *index, SArray* fVals, int uid) { index_document_destroy(doc); #endif +#ifdef USE_INVERTEDINDEX //TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { @@ -106,11 +110,16 @@ int indexPut(SIndex *index, SArray* fVals, int uid) { SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); assert(fi != NULL); int32_t fieldId = fi->fieldId; - int32_t colType = fi->type; + int32_t fieldType = fi->type; int32_t version = index->cVersion; - + int res = indexCachePut(index->cache, fieldId, fieldType, p->val, p->nVal, version, uid, p->operType); + if (ret != 0) { + return + } } pthread_mutex_unlock(&index->mtx); +#endif + return 1; } int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { @@ -148,16 +157,26 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result free(fields); free(keys); free(types); +#endif + +#ifdef USE_INVERTEDINDEX + #endif return 1; } int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { +#ifdef USE_INVERTEDINDEX +#endif return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts); +int indexRebuild(SIndex *index, SIndexOpts *opts) { +#ifdef USE_INVERTEDINDEX +#endif + +} SIndexOpts *indexOptsCreate() { @@ -184,53 +203,55 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); - indexTermDestroy(p->field_value); + indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type){ - SIndexTerm *t = indexTermCreate(field, nFields, value, nValue); - if (t == NULL) {return -1;} - SIndexTermQuery q = {.type = type, .field_value = t}; +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){ + SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } -SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal) { - SIndexTerm *t = (SIndexTerm *)malloc(sizeof(SIndexTerm)); - t->key = (char *)calloc(nKey + 1, 1); - memcpy(t->key, key, nKey); - t->nKey = nKey; +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { + SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); + if (t == NULL) { return NULL; } + + t->suid = suid; + t->operType= oper; + t->colType = colType; + + t->colName = (char *)calloc(1, nColName + 1); + memcpy(t->colName, colName, nColName); + t->nColName = nColName; - t->val = (char *)calloc(nVal + 1, 1); - memcpy(t->val, val, nVal); - t->nVal = nVal; + t->colVal = (char *)calloc(1, nColVal + 1); + memcpy(t->colVal, colVal, nColVal); + t->nColVal = nColVal; return t; } void indexTermDestroy(SIndexTerm *p) { - free(p->key); - free(p->val); + free(p->colName); + free(p->colVal); free(p); } -SArray *indexMultiTermCreate() { +SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } -int indexMultiTermAdd(SArray *array, const char *field, int32_t nField, const char *val, int32_t nVal) { - SIndexTerm *term = indexTermCreate(field, nField, val, nVal); - if (term == NULL) { return -1; } - taosArrayPush(array, &term); +int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { + taosArrayPush(terms, &term); return 0; } -void indexMultiTermDestroy(SArray *array) { - for (int32_t i = 0; i < taosArrayGetSize(array); i++) { - SIndexTerm *p = taosArrayGetP(array, i); +void indexMultiTermDestroy(SIndexMultiTerm *terms) { + for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { + SIndexTerm *p = taosArrayGetP(terms, i); indexTermDestroy(p); } - taosArrayDestroy(array); + taosArrayDestroy(terms); } void indexInit() { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index acb8e32157f51bb31efecb92e93fbf83dd99da8f..23f7a088231186d599b4d850512ac8302a89d9ac 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -16,7 +16,7 @@ #include "index_cache.h" #include "tcompare.h" -#define MAX_INDEX_KEY_LEN 128 // test only, change later +#define MAX_INDEX_KEY_LEN 256// test only, change later static char* getIndexKey(const void *pData) { return NULL; @@ -96,16 +96,19 @@ IndexCache *indexCacheCreate() { } -void indexCacheDestroy(IndexCache *cache) { - if (cache == NULL) { return; } - tSkipListDestroy(cache->skiplist); - free(cache); +void indexCacheDestroy(void *cache) { + IndexCache *pCache = cache; + if (pCache == NULL) { return; } + tSkipListDestroy(pCache->skiplist); + free(pCache); } -int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, +int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, uint32_t version, uint64_t uid, int8_t operType) { if (cache == NULL) { return -1;} + IndexCache *pCache = cache; + // encode data int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType); @@ -135,20 +138,15 @@ int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const c memcpy(p, &operType, sizeof(operType)); p += sizeof(operType); - tSkipListPut(cache->skiplist, (void *)buf); + tSkipListPut(pCache->skiplist, (void *)buf); // encode end - } -int indexCacheDel(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { - +int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + IndexCache *pCache = cache; + return 0; } -int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) { - +int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result) { + return 0; } - - - - -