diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index adeb52bb8c989332ce8284d24d2369685c147323..d31ea01c37bcd253b4885d7a0929d88d48b42a25 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,36 +15,46 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ +#include "tarray.h" + #ifdef __cplusplus extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void*)buf, (void*)key->mem, len); \ + memcpy((void *)buf, (void *)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(type) == sizeof(c)); \ - memcpy((void*)buf, (void*)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(type) == sizeof(c)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void*)buf, (void*)var, len); \ + memcpy((void *)buf, (void *)var, len); \ buf += len; \ } while (0) +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[4, 5] + */ +void iIntersection(SArray *interResults, SArray *finalResult); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9287a918285df77473b4bd2ae8b1c54fb79d00f8..267d57ab61690a9ad9413388dee578b5277989dc 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -370,22 +370,27 @@ static void indexInterResultsDestroy(SArray* results) { } taosArrayDestroy(results); } + static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray* first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); - taosArrayRemoveDuplicate(first, uidCompare, NULL); + + for (int i = 0; i < taosArrayGetSize(interResults); i--) { + SArray* t = taosArrayGetP(interResults, i); + taosArraySort(t, uidCompare); + taosArrayRemoveDuplicate(t, uidCompare, NULL); + } if (oType == MUST) { + iIntersection(interResults, fResults); // just one column index, enhance later - taosArrayAddAll(fResults, first); + // taosArrayAddAll(fResults, interResults); } else if (oType == SHOULD) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // tag1 condistion || tag2 condition } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // not use currently } return 0; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index aff976e52bb46836d42c1910cd913e96386afcb8..282059df2933d1ce635b08a2f4cbf3566c87a939 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { return ret; } - IndexTFile* pTfile = (IndexTFile*)tfile; + IndexTFile* pTfile = tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c new file mode 100644 index 0000000000000000000000000000000000000000..f03d19838e58f0f9e1cbf6b026f485bf77954822 --- /dev/null +++ b/source/libs/index/src/index_util.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "index_util.h" +#include "index.h" +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t *)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void iIntersection(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray *base = taosArrayGetP(inters, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t *)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(inters); j++) { + SArray *oth = taosArrayGetP(inters, j); + int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t *)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(final, &tgt); + } + } + tfree(mi); +} diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 665dfd7318e0636d446365b42ae9c1d78de06f76..bed42be3e58af3277c8f6f5aada5b7673d63fe7d 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") +add_executable(UtilUT "") target_sources(indexTest PRIVATE @@ -15,6 +16,11 @@ target_sources(fstUT PRIVATE "fstUT.cc" ) +target_sources(UtilUT + PRIVATE + "utilUT.cc" +) + target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -31,6 +37,12 @@ target_include_directories ( fstUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( UtilUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -53,6 +65,14 @@ target_link_libraries (fstUT index ) +target_link_libraries (UtilUT + os + util + common + gtest_main + index +) + #add_test( # NAME index_test diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc new file mode 100644 index 0000000000000000000000000000000000000000..a286965e20afd07c981b4018d2749584ac220508 --- /dev/null +++ b/source/libs/index/test/utilUT.cc @@ -0,0 +1,149 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +class UtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + src = (SArray *)taosArrayInit(2, sizeof(void *)); + for (int i = 0; i < 3; i++) { + SArray *m = taosArrayInit(10, sizeof(uint64_t)); + taosArrayPush(src, &m); + } + + rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + for (int i = 0; i < taosArrayGetSize(src); i++) { + SArray *m = (SArray *)taosArrayGetP(src, i); + taosArrayDestroy(m); + } + taosArrayDestroy(src); + } + + SArray *src; + SArray *rslt; +}; + +static void clearSourceArray(SArray *p) { + for (int i = 0; i < taosArrayGetSize(p); i++) { + SArray *m = (SArray *)taosArrayGetP(p, i); + taosArrayClear(m); + } +} +static void clearFinalArray(SArray *p) { taosArrayClear(p); } +TEST_F(UtilEnv, intersectionSimpleResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 10); + + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectMultiEmptyResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 10; i < 20; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 20; i < 30; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + // empty source + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectSimpleEmpty) { + clearSourceArray(src); + clearFinalArray(rslt); + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, intersect01) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {2, 3, 4, 5}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {1, 2, 3, 5}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {3, 5, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 2); +} +TEST_F(UtilEnv, intersect02) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {8, 10, 12, 13}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {9, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +}