Merge pull request #10390 from taosdata/feature/indexUpdate

add multi tag query

source/libs/index/inc/index_util.h

@@ -15,36 +15,46 @@
 #ifndef __INDEX_UTIL_H__
 #define __INDEX_UTIL_H__
 
+#include "tarray.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define SERIALIZE_MEM_TO_BUF(buf, key, mem)                   \
-  do {                                                        \
-    memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
-    buf += sizeof(key->mem);                                  \
+#define SERIALIZE_MEM_TO_BUF(buf, key, mem)                     \
+  do {                                                          \
+    memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
+    buf += sizeof(key->mem);                                    \
   } while (0)
 
 #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
   do {                                               \
-    memcpy((void*)buf, (void*)key->mem, len);        \
+    memcpy((void *)buf, (void *)key->mem, len);      \
     buf += len;                                      \
   } while (0)
 
-#define SERIALIZE_VAR_TO_BUF(buf, var, type)  \
-  do {                                        \
-    type c = var;                             \
-    assert(sizeof(type) == sizeof(c));        \
-    memcpy((void*)buf, (void*)&c, sizeof(c)); \
-    buf += sizeof(c);                         \
+#define SERIALIZE_VAR_TO_BUF(buf, var, type)    \
+  do {                                          \
+    type c = var;                               \
+    assert(sizeof(type) == sizeof(c));          \
+    memcpy((void *)buf, (void *)&c, sizeof(c)); \
+    buf += sizeof(c);                           \
   } while (0)
 
 #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
   do {                                          \
-    memcpy((void*)buf, (void*)var, len);        \
+    memcpy((void *)buf, (void *)var, len);      \
     buf += len;                                 \
   } while (0)
 
+/* multi sorted result intersection
+ * input: [1, 2, 4, 5]
+ *        [2, 3, 4, 5]
+ *        [1, 4, 5]
+ * output:[4, 5]
+ */
+void iIntersection(SArray *interResults, SArray *finalResult);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/index/src/index.c

@@ -370,22 +370,27 @@ static void indexInterResultsDestroy(SArray* results) {
   }
   taosArrayDestroy(results);
 }
+
 static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
   // refactor, merge interResults into fResults by oType
-  SArray* first = taosArrayGetP(interResults, 0);
-  taosArraySort(first, uidCompare);
-  taosArrayRemoveDuplicate(first, uidCompare, NULL);
+
+  for (int i = 0; i < taosArrayGetSize(interResults); i--) {
+    SArray* t = taosArrayGetP(interResults, i);
+    taosArraySort(t, uidCompare);
+    taosArrayRemoveDuplicate(t, uidCompare, NULL);
+  }
 
   if (oType == MUST) {
+    iIntersection(interResults, fResults);
     // just one column index, enhance later
-    taosArrayAddAll(fResults, first);
+    // taosArrayAddAll(fResults, interResults);
   } else if (oType == SHOULD) {
     // just one column index, enhance later
-    taosArrayAddAll(fResults, first);
+    taosArrayAddAll(fResults, interResults);
     // tag1 condistion || tag2 condition
   } else if (oType == NOT) {
     // just one column index, enhance later
-    taosArrayAddAll(fResults, first);
+    taosArrayAddAll(fResults, interResults);
     // not use currently
   }
   return 0;

source/libs/index/src/index_tfile.c

@@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
     return ret;
   }
 
-  IndexTFile* pTfile = (IndexTFile*)tfile;
+  IndexTFile* pTfile = tfile;
 
   SIndexTerm* term = query->term;
   ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};

source/libs/index/src/index_util.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "index_util.h"
+#include "index.h"
+typedef struct MergeIndex {
+  int idx;
+  int len;
+} MergeIndex;
+
+static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
+  uint64_t v;
+  int32_t  m;
+  while (s <= e) {
+    m = s + (e - s) / 2;
+    v = *(uint64_t *)taosArrayGet(arr, m);
+    if (v >= k) {
+      e = m - 1;
+    } else {
+      s = m + 1;
+    }
+  }
+  return s;
+}
+
+void iIntersection(SArray *inters, SArray *final) {
+  int32_t sz = taosArrayGetSize(inters);
+  if (sz <= 0) {
+    return;
+  }
+  MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
+  for (int i = 0; i < sz; i++) {
+    SArray *t = taosArrayGetP(inters, i);
+
+    mi[i].len = taosArrayGetSize(t);
+    mi[i].idx = 0;
+  }
+
+  SArray *base = taosArrayGetP(inters, 0);
+  for (int i = 0; i < taosArrayGetSize(base); i++) {
+    uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
+    bool     has = true;
+    for (int j = 1; j < taosArrayGetSize(inters); j++) {
+      SArray *oth = taosArrayGetP(inters, j);
+      int     mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
+      if (mid >= 0 && mid < mi[j].len) {
+        uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
+        has = (val == tgt ? true : false);
+        mi[j].idx = mid;
+      } else {
+        has = false;
+      }
+    }
+    if (has == true) {
+      taosArrayPush(final, &tgt);
+    }
+  }
+  tfree(mi);
+}

source/libs/index/test/CMakeLists.txt

 add_executable(indexTest "")
 add_executable(fstTest "")
 add_executable(fstUT "")
+add_executable(UtilUT "")
 
 target_sources(indexTest
   PRIVATE 
@@ -15,6 +16,11 @@ target_sources(fstUT
   PRIVATE 
   "fstUT.cc" 
 )
+target_sources(UtilUT
+  PRIVATE 
+  "utilUT.cc" 
+)
+
 target_include_directories ( indexTest
   PUBLIC
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
@@ -31,6 +37,12 @@ target_include_directories ( fstUT
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
+
+target_include_directories ( UtilUT 
+  PUBLIC
+  "${CMAKE_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+) 
 target_link_libraries (indexTest
   os  
   util
@@ -53,6 +65,14 @@ target_link_libraries (fstUT
   index
 )
 
+target_link_libraries (UtilUT
+  os  
+  util
+  common
+  gtest_main
+  index
+)
+
 
 #add_test(
 #  NAME index_test

source/libs/index/test/utilUT.cc

+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexInt.h"
+#include "index_cache.h"
+#include "index_fst.h"
+#include "index_fst_counting_writer.h"
+#include "index_fst_util.h"
+#include "index_tfile.h"
+#include "index_util.h"
+#include "tglobal.h"
+#include "tskiplist.h"
+#include "tutil.h"
+
+class UtilEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    src = (SArray *)taosArrayInit(2, sizeof(void *));
+    for (int i = 0; i < 3; i++) {
+      SArray *m = taosArrayInit(10, sizeof(uint64_t));
+      taosArrayPush(src, &m);
+    }
+
+    rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
+  }
+  virtual void TearDown() {
+    for (int i = 0; i < taosArrayGetSize(src); i++) {
+      SArray *m = (SArray *)taosArrayGetP(src, i);
+      taosArrayDestroy(m);
+    }
+    taosArrayDestroy(src);
+  }
+
+  SArray *src;
+  SArray *rslt;
+};
+
+static void clearSourceArray(SArray *p) {
+  for (int i = 0; i < taosArrayGetSize(p); i++) {
+    SArray *m = (SArray *)taosArrayGetP(p, i);
+    taosArrayClear(m);
+  }
+}
+static void clearFinalArray(SArray *p) { taosArrayClear(p); }
+TEST_F(UtilEnv, intersectionSimpleResult) {
+  SArray *f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 10);
+
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+}
+TEST_F(UtilEnv, intersectMultiEmptyResult) {
+  SArray *f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 10; i < 20; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 20; i < 30; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  // empty source
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+}
+TEST_F(UtilEnv, intersectSimpleEmpty) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+}
+TEST_F(UtilEnv, intersect01) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {2, 3, 4, 5};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {1, 2, 3, 5};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  uint64_t arr3[] = {3, 5, 10, 11};
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
+    taosArrayPush(f, &arr3[i]);
+  }
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 2);
+}
+TEST_F(UtilEnv, intersect02) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {13, 14, 15};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {8, 10, 12, 13};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  uint64_t arr3[] = {9, 10, 11};
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
+    taosArrayPush(f, &arr3[i]);
+  }
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+}