indexTests.cc 33.4 KB
Newer Older
dengyihao's avatar
dengyihao 已提交
1 2 3 4
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
5 6
 * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
 * Software Foundation.
dengyihao's avatar
dengyihao 已提交
7 8 9 10 11 12 13 14
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
dengyihao's avatar
dengyihao 已提交
15
#include <gtest/gtest.h>
dengyihao's avatar
dengyihao 已提交
16
#include <algorithm>
dengyihao's avatar
dengyihao 已提交
17
#include <iostream>
dengyihao's avatar
dengyihao 已提交
18
#include <string>
dengyihao's avatar
dengyihao 已提交
19
#include <thread>
dengyihao's avatar
dengyihao 已提交
20
#include "index.h"
dengyihao's avatar
dengyihao 已提交
21 22 23 24
#include "indexCache.h"
#include "indexFst.h"
#include "indexFstCountingWriter.h"
#include "indexFstUtil.h"
dengyihao's avatar
dengyihao 已提交
25
#include "indexInt.h"
dengyihao's avatar
dengyihao 已提交
26 27
#include "indexTfile.h"
#include "indexUtil.h"
dengyihao's avatar
dengyihao 已提交
28
#include "tskiplist.h"
dengyihao's avatar
dengyihao 已提交
29
#include "tutil.h"
dengyihao's avatar
dengyihao 已提交
30
using namespace std;
dengyihao's avatar
dengyihao 已提交
31

dengyihao's avatar
dengyihao 已提交
32
#define NUM_OF_THREAD 10
dengyihao's avatar
dengyihao 已提交
33

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
class DebugInfo {
 public:
  DebugInfo(const char* str) : info(str) {
    std::cout << "------------" << info << "\t"
              << "begin"
              << "-------------" << std::endl;
  }
  ~DebugInfo() {
    std::cout << "-----------" << info << "\t"
              << "end"
              << "--------------" << std::endl;
  }

 private:
  std::string info;
};
dengyihao's avatar
dengyihao 已提交
50

dengyihao's avatar
dengyihao 已提交
51
class FstWriter {
dengyihao's avatar
dengyihao 已提交
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
 public:
  FstWriter() {
    _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024);
    _b = fstBuilderCreate(NULL, 0);
  }
  bool Put(const std::string& key, uint64_t val) {
    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
    bool     ok = fstBuilderInsert(_b, skey, val);
    fstSliceDestroy(&skey);
    return ok;
  }
  ~FstWriter() {
    fstBuilderFinish(_b);
    fstBuilderDestroy(_b);

dengyihao's avatar
dengyihao 已提交
67
    writerCtxDestroy(_wc, false);
dengyihao's avatar
dengyihao 已提交
68 69 70 71 72
  }

 private:
  FstBuilder* _b;
  WriterCtx*  _wc;
dengyihao's avatar
dengyihao 已提交
73 74 75
};

class FstReadMemory {
dengyihao's avatar
dengyihao 已提交
76 77 78 79 80 81 82 83
 public:
  FstReadMemory(size_t size) {
    _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024);
    _w = fstCountingWriterCreate(_wc);
    _size = size;
    memset((void*)&_s, 0, sizeof(_s));
  }
  bool init() {
wafwerar's avatar
wafwerar 已提交
84
    char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size);
dengyihao's avatar
dengyihao 已提交
85
    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
dengyihao's avatar
dengyihao 已提交
86 87 88
    if (nRead <= 0) {
      return false;
    }
dengyihao's avatar
dengyihao 已提交
89 90 91
    _size = nRead;
    _s = fstSliceCreate((uint8_t*)buf, _size);
    _fst = fstCreate(&_s);
wafwerar's avatar
wafwerar 已提交
92
    taosMemoryFree(buf);
dengyihao's avatar
dengyihao 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
    return _fst != NULL;
  }
  bool Get(const std::string& key, uint64_t* val) {
    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
    bool     ok = fstGet(_fst, &skey, val);
    fstSliceDestroy(&skey);
    return ok;
  }
  bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
    int64_t s = taosGetTimestampUs();
    bool    ok = this->Get(key, val);
    int64_t e = taosGetTimestampUs();
    *elapse = e - s;
    return ok;
  }
  // add later
  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
    StreamWithState*       st = streamBuilderIntoStream(sb);
    StreamWithStateResult* rt = NULL;

dengyihao's avatar
dengyihao 已提交
114 115 116
    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
      result.push_back((uint64_t)(rt->out.out));
    }
dengyihao's avatar
dengyihao 已提交
117 118 119 120 121 122 123 124 125 126
    return true;
  }
  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
    int64_t s = taosGetTimestampUs();
    bool    ok = this->Search(ctx, result);
    int64_t e = taosGetTimestampUs();
    return ok;
  }

  ~FstReadMemory() {
dengyihao's avatar
dengyihao 已提交
127
    fstCountingWriterDestroy(_w);
dengyihao's avatar
dengyihao 已提交
128
    fstDestroy(_fst);
dengyihao's avatar
dengyihao 已提交
129
    fstSliceDestroy(&_s);
dengyihao's avatar
dengyihao 已提交
130
    writerCtxDestroy(_wc, true);
dengyihao's avatar
dengyihao 已提交
131 132 133 134 135 136 137 138 139 140
  }

 private:
  FstCountingWriter* _w;
  Fst*               _fst;
  FstSlice           _s;
  WriterCtx*         _wc;
  size_t             _size;
};

dengyihao's avatar
dengyihao 已提交
141 142 143
#define L 100
#define M 100
#define N 100
dengyihao's avatar
dengyihao 已提交
144

dengyihao's avatar
dengyihao 已提交
145 146
int Performance_fstWriteRecords(FstWriter* b) {
  std::string str("aa");
dengyihao's avatar
dengyihao 已提交
147
  for (int i = 0; i < L; i++) {
dengyihao's avatar
dengyihao 已提交
148
    str[0] = 'a' + i;
dengyihao's avatar
dengyihao 已提交
149 150
    str.resize(2);
    for (int j = 0; j < M; j++) {
dengyihao's avatar
dengyihao 已提交
151 152
      str[1] = 'a' + j;
      str.resize(2);
dengyihao's avatar
dengyihao 已提交
153
      for (int k = 0; k < N; k++) {
dengyihao's avatar
dengyihao 已提交
154 155
        str.push_back('a');
        b->Put(str, k);
dengyihao's avatar
dengyihao 已提交
156
        printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
dengyihao's avatar
dengyihao 已提交
157
      }
dengyihao's avatar
dengyihao 已提交
158
    }
dengyihao's avatar
dengyihao 已提交
159
  }
dengyihao's avatar
dengyihao 已提交
160
  return L * M * N;
dengyihao's avatar
dengyihao 已提交
161
}
dengyihao's avatar
dengyihao 已提交
162
void Performance_fstReadRecords(FstReadMemory* m) {
dengyihao's avatar
dengyihao 已提交
163 164 165
  std::string str("aa");
  for (int i = 0; i < M; i++) {
    str[0] = 'a' + i;
dengyihao's avatar
dengyihao 已提交
166 167
    str.resize(2);
    for (int j = 0; j < N; j++) {
dengyihao's avatar
dengyihao 已提交
168 169 170 171
      str[1] = 'a' + j;
      str.resize(2);
      for (int k = 0; k < L; k++) {
        str.push_back('a');
dengyihao's avatar
dengyihao 已提交
172
        uint64_t val, cost;
dengyihao's avatar
dengyihao 已提交
173
        if (m->GetWithTimeCostUs(str, &val, &cost)) {
dengyihao's avatar
dengyihao 已提交
174
          printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
dengyihao's avatar
dengyihao 已提交
175 176 177 178
        } else {
          printf("failed to get key: %s\n", str.c_str());
        }
      }
dengyihao's avatar
dengyihao 已提交
179
    }
dengyihao's avatar
dengyihao 已提交
180
  }
dengyihao's avatar
dengyihao 已提交
181
}
dengyihao's avatar
dengyihao 已提交
182
void checkFstPerf() {
dengyihao's avatar
dengyihao 已提交
183 184
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
dengyihao's avatar
dengyihao 已提交
185

dengyihao's avatar
dengyihao 已提交
186
  int     num = Performance_fstWriteRecords(fw);
dengyihao's avatar
dengyihao 已提交
187
  int64_t e = taosGetTimestampUs();
dengyihao's avatar
dengyihao 已提交
188
  printf("write %d record cost %" PRId64 "us\n", num, e - s);
dengyihao's avatar
dengyihao 已提交
189 190
  delete fw;

dengyihao's avatar
dengyihao 已提交
191
  FstReadMemory* m = new FstReadMemory(1024 * 64);
dengyihao's avatar
dengyihao 已提交
192 193 194
  if (m->init()) {
    printf("success to init fst read");
  }
dengyihao's avatar
dengyihao 已提交
195
  Performance_fstReadRecords(m);
dengyihao's avatar
dengyihao 已提交
196
  delete m;
dengyihao's avatar
dengyihao 已提交
197
}
dengyihao's avatar
dengyihao 已提交
198
void checkFstPrefixSearch() {
dengyihao's avatar
dengyihao 已提交
199 200 201
  FstWriter*  fw = new FstWriter;
  int64_t     s = taosGetTimestampUs();
  int         count = 2;
dengyihao's avatar
dengyihao 已提交
202
  std::string key("ab");
dengyihao's avatar
dengyihao 已提交
203

dengyihao's avatar
dengyihao 已提交
204
  for (int i = 0; i < count; i++) {
dengyihao's avatar
dengyihao 已提交
205 206
    key[1] = key[1] + i;
    fw->Put(key, i);
dengyihao's avatar
dengyihao 已提交
207 208
  }
  int64_t e = taosGetTimestampUs();
dengyihao's avatar
dengyihao 已提交
209

dengyihao's avatar
dengyihao 已提交
210 211
  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
  delete fw;
dengyihao's avatar
dengyihao 已提交
212

dengyihao's avatar
dengyihao 已提交
213
  FstReadMemory* m = new FstReadMemory(1024 * 64);
dengyihao's avatar
dengyihao 已提交
214
  if (m->init() == false) {
dengyihao's avatar
dengyihao 已提交
215
    std::cout << "init readMemory failed" << std::endl;
dengyihao's avatar
dengyihao 已提交
216 217 218
    delete m;
    return;
  }
dengyihao's avatar
dengyihao 已提交
219 220

  // prefix search
dengyihao's avatar
dengyihao 已提交
221
  std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
222 223 224 225

  AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
  m->Search(ctx, result);
  assert(result.size() == count);
dengyihao's avatar
dengyihao 已提交
226
  for (int i = 0; i < result.size(); i++) {
dengyihao's avatar
dengyihao 已提交
227
    assert(result[i] == i);  // check result
dengyihao's avatar
dengyihao 已提交
228 229
  }

wafwerar's avatar
wafwerar 已提交
230
  taosMemoryFree(ctx);
dengyihao's avatar
dengyihao 已提交
231
  delete m;
dengyihao's avatar
dengyihao 已提交
232
}
dengyihao's avatar
dengyihao 已提交
233
void validateFst() {
dengyihao's avatar
dengyihao 已提交
234 235 236 237
  int        val = 100;
  int        count = 100;
  FstWriter* fw = new FstWriter;
  // write
dengyihao's avatar
dengyihao 已提交
238
  {
dengyihao's avatar
dengyihao 已提交
239
    std::string key("ab");
dengyihao's avatar
dengyihao 已提交
240
    for (int i = 0; i < count; i++) {
dengyihao's avatar
dengyihao 已提交
241
      key.push_back('a' + i);
dengyihao's avatar
dengyihao 已提交
242
      fw->Put(key, val - i);
dengyihao's avatar
dengyihao 已提交
243
    }
dengyihao's avatar
dengyihao 已提交
244 245
  }
  delete fw;
dengyihao's avatar
dengyihao 已提交
246

dengyihao's avatar
dengyihao 已提交
247
  // read
dengyihao's avatar
dengyihao 已提交
248 249 250
  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
dengyihao's avatar
dengyihao 已提交
251 252
    delete m;
    return;
dengyihao's avatar
dengyihao 已提交
253
  }
dengyihao's avatar
dengyihao 已提交
254 255

  {
dengyihao's avatar
dengyihao 已提交
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
    std::string key("ab");
    uint64_t    out;
    if (m->Get(key, &out)) {
      printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
    } else {
      printf("failed to get(%s)\n", key.c_str());
    }
    for (int i = 0; i < count; i++) {
      key.push_back('a' + i);
      if (m->Get(key, &out)) {
        assert(val - i == out);
        printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
      } else {
        printf("failed to get(%s)\n", key.c_str());
      }
dengyihao's avatar
dengyihao 已提交
271
    }
dengyihao's avatar
dengyihao 已提交
272
  }
dengyihao's avatar
dengyihao 已提交
273
  delete m;
dengyihao's avatar
dengyihao 已提交
274
}
dengyihao's avatar
dengyihao 已提交
275
class IndexEnv : public ::testing::Test {
dengyihao's avatar
dengyihao 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
 protected:
  virtual void SetUp() {
    taosRemoveDir(path);
    opts = indexOptsCreate();
    int ret = indexOpen(opts, path, &index);
    assert(ret == 0);
  }
  virtual void TearDown() {
    indexClose(index);
    indexOptsDestroy(opts);
  }

  const char* path = "/tmp/tindex";
  SIndexOpts* opts;
  SIndex*     index;
dengyihao's avatar
dengyihao 已提交
291 292
};

293 294 295 296 297
/// TEST_F(IndexEnv, testPut) {
//  /  // single index column
//      / {
//    / std::string colName("tag1"), colVal("Hello world");
//    / SIndexTerm* term =
298 299
//        indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), /
//        colVal.size());
300 301 302 303 304 305 306
//    SIndexMultiTerm* terms = indexMultiTermCreate();
//    indexMultiTermAdd(terms, term);
//    / / for (size_t i = 0; i < 100; i++) {
//      / int tableId = i;
//      / int ret = indexPut(index, terms, tableId);
//      / assert(ret == 0);
//      /
dengyihao's avatar
dengyihao 已提交
307
//    }
308 309
//    / indexMultiTermDestroy(terms);
//    /
dengyihao's avatar
dengyihao 已提交
310
//  }
311 312 313 314 315 316
//  /  // multi index column
//      / {
//    / SIndexMultiTerm* terms = indexMultiTermCreate();
//    / {
//      / std::string colName("tag1"), colVal("Hello world");
//      / SIndexTerm* term =
317 318
//          / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
//          colVal.size());
319 320
//      / indexMultiTermAdd(terms, term);
//      /
dengyihao's avatar
dengyihao 已提交
321
//    }
322 323 324
//    / {
//      / std::string colName("tag2"), colVal("Hello world");
//      / SIndexTerm* term =
325 326
//          / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
//          colVal.size());
327 328
//      / indexMultiTermAdd(terms, term);
//      /
dengyihao's avatar
dengyihao 已提交
329
//    }
330 331 332 333 334
//    / / for (int i = 0; i < 100; i++) {
//      / int tableId = i;
//      / int ret = indexPut(index, terms, tableId);
//      / assert(ret == 0);
//      /
dengyihao's avatar
dengyihao 已提交
335
//    }
336 337
//    / indexMultiTermDestroy(terms);
//    /
dengyihao's avatar
dengyihao 已提交
338
//  }
339 340
//  /  //
//      /
dengyihao's avatar
dengyihao 已提交
341 342
//}

dengyihao's avatar
dengyihao 已提交
343 344
class TFileObj {
 public:
345 346
  TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage")
      : path_(path), colName_(colName) {
dengyihao's avatar
dengyihao 已提交
347
    colId_ = 10;
dengyihao's avatar
dengyihao 已提交
348 349
    reader_ = NULL;
    writer_ = NULL;
dengyihao's avatar
dengyihao 已提交
350 351 352 353 354 355 356 357
    // Do Nothing
    //
  }
  int Put(SArray* tv) {
    if (reader_ != NULL) {
      tfileReaderDestroy(reader_);
      reader_ = NULL;
    }
dengyihao's avatar
dengyihao 已提交
358 359 360
    if (writer_ == NULL) {
      InitWriter();
    }
dengyihao's avatar
dengyihao 已提交
361
    return tfileWriterPut(writer_, tv, false);
dengyihao's avatar
dengyihao 已提交
362 363 364
  }
  bool InitWriter() {
    TFileHeader header;
dengyihao's avatar
dengyihao 已提交
365 366
    header.suid = 1;
    header.version = 1;
dengyihao's avatar
dengyihao 已提交
367
    memcpy(header.colName, colName_.c_str(), colName_.size());
dengyihao's avatar
dengyihao 已提交
368 369
    header.colType = TSDB_DATA_TYPE_BINARY;

dengyihao's avatar
dengyihao 已提交
370
    std::string path(path_);
dengyihao's avatar
dengyihao 已提交
371 372
    int         colId = 2;
    char        buf[64] = {0};
dengyihao's avatar
dengyihao 已提交
373
    sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId_, header.version);
dengyihao's avatar
dengyihao 已提交
374 375
    path.append("/").append(buf);

dengyihao's avatar
dengyihao 已提交
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
    fileName_ = path;

    WriterCtx* ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);

    writer_ = tfileWriterCreate(ctx, &header);
    return writer_ != NULL ? true : false;
  }
  bool InitReader() {
    WriterCtx* ctx = writerCtxCreate(TFile, fileName_.c_str(), true, 64 * 1024 * 1024);
    reader_ = tfileReaderCreate(ctx);
    return reader_ != NULL ? true : false;
  }
  int Get(SIndexTermQuery* query, SArray* result) {
    if (writer_ != NULL) {
      tfileWriterDestroy(writer_);
      writer_ = NULL;
    }
    if (reader_ == NULL && InitReader()) {
      //
      //
    }
dengyihao's avatar
dengyihao 已提交
397 398 399 400 401 402 403
    SIdxTempResult* tr = sIdxTempResultCreate();

    int ret = tfileReaderSearch(reader_, query, tr);

    sIdxTempResultMergeTo(result, tr);
    sIdxTempResultDestroy(tr);
    return ret;
dengyihao's avatar
dengyihao 已提交
404 405
  }
  ~TFileObj() {
dengyihao's avatar
dengyihao 已提交
406 407 408 409 410 411
    if (writer_) {
      tfileWriterDestroy(writer_);
    }
    if (reader_) {
      tfileReaderDestroy(reader_);
    }
dengyihao's avatar
dengyihao 已提交
412 413 414 415 416 417 418 419 420 421 422 423
  }

 private:
  std::string path_;
  std::string colName_;
  std::string fileName_;

  TFileWriter* writer_;
  TFileReader* reader_;

  int colId_;
};
dengyihao's avatar
dengyihao 已提交
424

dengyihao's avatar
dengyihao 已提交
425 426 427 428 429 430
class IndexTFileEnv : public ::testing::Test {
 protected:
  virtual void SetUp() {
    taosRemoveDir(dir.c_str());
    taosMkDir(dir.c_str());
    fObj = new TFileObj(dir, colName);
dengyihao's avatar
dengyihao 已提交
431 432 433 434 435
  }

  virtual void TearDown() {
    // indexClose(index);
    // indexeptsDestroy(opts);
dengyihao's avatar
dengyihao 已提交
436 437
    delete fObj;
    // tfileWriterDestroy(twrite);
dengyihao's avatar
dengyihao 已提交
438
  }
dengyihao's avatar
dengyihao 已提交
439 440 441 442 443 444 445
  TFileObj*   fObj;
  std::string dir = "/tmp/tindex";
  std::string colName = "voltage";

  int coldId = 2;
  int version = 1;
  int colType = TSDB_DATA_TYPE_BINARY;
dengyihao's avatar
dengyihao 已提交
446 447 448
};

static TFileValue* genTFileValue(const char* val) {
wafwerar's avatar
wafwerar 已提交
449
  TFileValue* tv = (TFileValue*)taosMemoryCalloc(1, sizeof(TFileValue));
dengyihao's avatar
dengyihao 已提交
450
  int32_t     vlen = strlen(val) + 1;
wafwerar's avatar
wafwerar 已提交
451
  tv->colVal = (char*)taosMemoryCalloc(1, vlen);
dengyihao's avatar
dengyihao 已提交
452 453 454
  memcpy(tv->colVal, val, vlen);

  tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t));
dengyihao's avatar
dengyihao 已提交
455
  for (size_t i = 0; i < 200; i++) {
dengyihao's avatar
dengyihao 已提交
456 457 458 459 460 461 462
    uint64_t v = i;
    taosArrayPush(tv->tableId, &v);
  }
  return tv;
}
static void destroyTFileValue(void* val) {
  TFileValue* tv = (TFileValue*)val;
wafwerar's avatar
wafwerar 已提交
463
  taosMemoryFree(tv->colVal);
dengyihao's avatar
dengyihao 已提交
464
  taosArrayDestroy(tv->tableId);
wafwerar's avatar
wafwerar 已提交
465
  taosMemoryFree(tv);
dengyihao's avatar
dengyihao 已提交
466
}
dengyihao's avatar
dengyihao 已提交
467
TEST_F(IndexTFileEnv, test_tfile_write) {
dengyihao's avatar
dengyihao 已提交
468
  TFileValue* v1 = genTFileValue("ab");
dengyihao's avatar
dengyihao 已提交
469

dengyihao's avatar
dengyihao 已提交
470
  SArray* data = (SArray*)taosArrayInit(4, sizeof(void*));
dengyihao's avatar
dengyihao 已提交
471

dengyihao's avatar
dengyihao 已提交
472
  taosArrayPush(data, &v1);
dengyihao's avatar
dengyihao 已提交
473 474 475
  // taosArrayPush(data, &v2);
  // taosArrayPush(data, &v3);
  // taosArrayPush(data, &v4);
dengyihao's avatar
dengyihao 已提交
476

dengyihao's avatar
dengyihao 已提交
477
  fObj->Put(data);
478 479 480 481
  for (size_t i = 0; i < taosArrayGetSize(data); i++) {
    // data
    destroyTFileValue(taosArrayGetP(data, i));
  }
dengyihao's avatar
dengyihao 已提交
482
  taosArrayDestroy(data);
dengyihao's avatar
dengyihao 已提交
483

484 485 486 487
  std::string     colName("voltage");
  std::string     colVal("ab");
  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                     colVal.c_str(), colVal.size());
dengyihao's avatar
dengyihao 已提交
488 489 490 491
  SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};

  SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
  fObj->Get(&query, result);
dengyihao's avatar
dengyihao 已提交
492
  assert(taosArrayGetSize(result) == 200);
dengyihao's avatar
dengyihao 已提交
493
  indexTermDestroy(term);
494
  taosArrayDestroy(result);
dengyihao's avatar
dengyihao 已提交
495 496

  // tfileWriterDestroy(twrite);
dengyihao's avatar
dengyihao 已提交
497
}
dengyihao's avatar
dengyihao 已提交
498 499 500 501
class CacheObj {
 public:
  CacheObj() {
    // TODO
dengyihao's avatar
dengyihao 已提交
502
    cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY);
dengyihao's avatar
dengyihao 已提交
503 504
  }
  int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
dengyihao's avatar
dengyihao 已提交
505
    int ret = indexCachePut(cache, term, uid);
dengyihao's avatar
dengyihao 已提交
506 507 508 509 510 511
    if (ret != 0) {
      //
      std::cout << "failed to put into cache: " << ret << std::endl;
    }
    return ret;
  }
dengyihao's avatar
dengyihao 已提交
512 513 514 515
  void Debug() {
    //
    indexCacheDebug(cache);
  }
dengyihao's avatar
dengyihao 已提交
516
  int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
dengyihao's avatar
dengyihao 已提交
517 518 519 520 521 522
    SIdxTempResult* tr = sIdxTempResultCreate();

    int ret = indexCacheSearch(cache, query, tr, s);
    sIdxTempResultMergeTo(result, tr);
    sIdxTempResultDestroy(tr);

dengyihao's avatar
dengyihao 已提交
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
    if (ret != 0) {
      std::cout << "failed to get from cache:" << ret << std::endl;
    }
    return ret;
  }
  ~CacheObj() {
    // TODO
    indexCacheDestroy(cache);
  }

 private:
  IndexCache* cache = NULL;
};

class IndexCacheEnv : public ::testing::Test {
 protected:
  virtual void SetUp() {
    // TODO
    coj = new CacheObj();
  }
  virtual void TearDown() {
    delete coj;
    // formate
  }
  CacheObj* coj;
};

550
#define MAX_TERM_KEY_LEN 128
dengyihao's avatar
dengyihao 已提交
551
TEST_F(IndexCacheEnv, cache_test) {
552 553
  int     version = 0;
  int16_t colId = 0;
dengyihao's avatar
dengyihao 已提交
554
  int16_t othColId = 10;
dengyihao's avatar
dengyihao 已提交
555

556
  uint64_t    suid = 0;
dengyihao's avatar
dengyihao 已提交
557 558
  std::string colName("voltage");
  {
559
    std::string colVal("v1");
560 561
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
562
    coj->Put(term, colId, version++, suid++);
563
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
564
    // indexTermDestry(term);
dengyihao's avatar
dengyihao 已提交
565 566
  }
  {
567
    std::string colVal("v3");
568 569
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
570
    coj->Put(term, colId, version++, suid++);
571
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
572 573
  }
  {
574
    std::string colVal("v2");
575 576
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
577
    coj->Put(term, colId, version++, suid++);
578
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
579 580
  }
  {
581
    std::string colVal("v3");
582 583
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
584
    coj->Put(term, colId, version++, suid++);
585
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
586 587
  }
  {
588
    std::string colVal("v3");
589 590
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
591
    coj->Put(term, colId, version++, suid++);
592
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
593
  }
dengyihao's avatar
dengyihao 已提交
594 595
  coj->Debug();
  std::cout << "--------first----------" << std::endl;
dengyihao's avatar
dengyihao 已提交
596 597
  {
    std::string colVal("v3");
598 599
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
dengyihao's avatar
dengyihao 已提交
600
    coj->Put(term, othColId, version++, suid++);
601
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
602 603 604
  }
  {
    std::string colVal("v4");
605 606
    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
dengyihao's avatar
dengyihao 已提交
607
    coj->Put(term, othColId, version++, suid++);
608
    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
609
  }
dengyihao's avatar
dengyihao 已提交
610 611
  coj->Debug();
  std::cout << "--------second----------" << std::endl;
dengyihao's avatar
dengyihao 已提交
612
  {
613
    std::string colVal("v4");
dengyihao's avatar
dengyihao 已提交
614
    for (size_t i = 0; i < 10; i++) {
615
      colVal[colVal.size() - 1] = 'a' + i;
616 617
      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                         colVal.c_str(), colVal.size());
618
      coj->Put(term, colId, version++, suid++);
619
      indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
620 621
    }
  }
dengyihao's avatar
dengyihao 已提交
622 623
  coj->Debug();
  // begin query
dengyihao's avatar
dengyihao 已提交
624
  {
625 626 627
    std::string     colVal("v3");
    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
628 629 630 631 632
    SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
    SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
    STermValueType  valType;

    coj->Get(&query, colId, 10000, ret, &valType);
dengyihao's avatar
dengyihao 已提交
633
    std::cout << "size : " << taosArrayGetSize(ret) << std::endl;
dengyihao's avatar
dengyihao 已提交
634
    assert(taosArrayGetSize(ret) == 4);
635 636 637
    taosArrayDestroy(ret);

    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
638
  }
639
  {
640 641 642
    std::string     colVal("v2");
    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
643 644 645
    SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
    SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
    STermValueType  valType;
dengyihao's avatar
dengyihao 已提交
646

647 648
    coj->Get(&query, colId, 10000, ret, &valType);
    assert(taosArrayGetSize(ret) == 1);
649 650 651
    taosArrayDestroy(ret);

    indexTermDestroy(term);
dengyihao's avatar
dengyihao 已提交
652 653
  }
}
dengyihao's avatar
dengyihao 已提交
654 655 656 657 658 659 660 661 662
class IndexObj {
 public:
  IndexObj() {
    // opt
    numOfWrite = 0;
    numOfRead = 0;
    indexInit();
  }
  int Init(const std::string& dir) {
dengyihao's avatar
dengyihao 已提交
663
    taosRemoveDir(dir.c_str());
dengyihao's avatar
dengyihao 已提交
664 665 666 667 668 669 670 671
    taosMkDir(dir.c_str());
    int ret = indexOpen(&opts, dir.c_str(), &idx);
    if (ret != 0) {
      // opt
      std::cout << "failed to open index: %s" << dir << std::endl;
    }
    return ret;
  }
dengyihao's avatar
dengyihao 已提交
672 673 674 675 676 677 678 679
  void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    SIndexMultiTerm* terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
    Put(terms, uid);
    indexMultiTermDestroy(terms);
  }
dengyihao's avatar
dengyihao 已提交
680 681 682 683 684 685 686 687 688 689 690 691 692
  int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                      size_t numOfTable = 100 * 10000) {
    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    SIndexMultiTerm* terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
    for (size_t i = 0; i < numOfTable; i++) {
      int ret = Put(terms, i);
      assert(ret == 0);
    }
    indexMultiTermDestroy(terms);
    return numOfTable;
  }
dengyihao's avatar
dengyihao 已提交
693 694 695
  int WriteMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                           size_t numOfTable = 100 * 10000) {
    std::string tColVal = colVal;
dengyihao's avatar
dengyihao 已提交
696
    size_t      colValSize = tColVal.size();
dengyihao's avatar
dengyihao 已提交
697 698
    int         skip = 100;
    numOfTable /= skip;
dengyihao's avatar
dengyihao 已提交
699
    for (int i = 0; i < numOfTable; i++) {
dengyihao's avatar
dengyihao 已提交
700 701
      for (int k = 0; k < 10 && k < colVal.size(); k++) {
        // opt
wafwerar's avatar
wafwerar 已提交
702
        tColVal[taosRand() % colValSize] = 'a' + k % 26;
dengyihao's avatar
dengyihao 已提交
703
      }
dengyihao's avatar
dengyihao 已提交
704
      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
dengyihao's avatar
dengyihao 已提交
705
                                         tColVal.c_str(), tColVal.size());
dengyihao's avatar
dengyihao 已提交
706 707
      SIndexMultiTerm* terms = indexMultiTermCreate();
      indexMultiTermAdd(terms, term);
dengyihao's avatar
dengyihao 已提交
708 709
      for (size_t j = 0; j < skip; j++) {
        int ret = Put(terms, j);
dengyihao's avatar
dengyihao 已提交
710 711 712 713 714 715
        assert(ret == 0);
      }
      indexMultiTermDestroy(terms);
    }
    return numOfTable;
  }
dengyihao's avatar
dengyihao 已提交
716 717 718 719 720 721 722 723 724 725 726
  int ReadMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                          size_t numOfTable = 100 * 10000) {
    std::string tColVal = colVal;

    int colValSize = tColVal.size();
    for (int i = 0; i < numOfTable; i++) {
      tColVal[i % colValSize] = 'a' + i % 26;
      SearchOne(colName, tColVal);
    }
    return 0;
  }
dengyihao's avatar
dengyihao 已提交
727

dengyihao's avatar
dengyihao 已提交
728 729 730 731 732 733 734 735 736 737
  int Put(SIndexMultiTerm* fvs, uint64_t uid) {
    numOfWrite += taosArrayGetSize(fvs);
    return indexPut(idx, fvs, uid);
  }
  int Search(SIndexMultiTermQuery* multiQ, SArray* result) {
    SArray* query = multiQ->query;
    numOfRead = taosArrayGetSize(query);
    return indexSearch(idx, multiQ, result);
  }

dengyihao's avatar
dengyihao 已提交
738 739 740 741 742 743 744
  int SearchOne(const std::string& colName, const std::string& colVal) {
    SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    indexMultiTermQueryAdd(mq, term, QUERY_TERM);

    SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
dengyihao's avatar
dengyihao 已提交
745

dengyihao's avatar
dengyihao 已提交
746 747 748 749 750 751
    int64_t s = taosGetTimestampUs();
    if (Search(mq, result) == 0) {
      int64_t e = taosGetTimestampUs();
      std::cout << "search and time cost:" << e - s << "\tquery col:" << colName << "\t val: " << colVal
                << "\t size:" << taosArrayGetSize(result) << std::endl;
    } else {
dengyihao's avatar
dengyihao 已提交
752
      return -1;
dengyihao's avatar
dengyihao 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
    }
    int sz = taosArrayGetSize(result);
    indexMultiTermQueryDestroy(mq);
    taosArrayDestroy(result);
    return sz;
    // assert(taosArrayGetSize(result) == targetSize);
  }
  int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
    SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    indexMultiTermQueryAdd(mq, term, QUERY_TERM);

    SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));

dengyihao's avatar
dengyihao 已提交
768 769 770
    int64_t s = taosGetTimestampUs();
    if (Search(mq, result) == 0) {
      int64_t e = taosGetTimestampUs();
dengyihao's avatar
add UT  
dengyihao 已提交
771
      std::cout << "search one successfully and time cost:" << e - s << "us\tquery col:" << colName
dengyihao's avatar
dengyihao 已提交
772
                << "\t val: " << colVal << "\t size:" << taosArrayGetSize(result) << std::endl;
dengyihao's avatar
dengyihao 已提交
773 774
    } else {
    }
775 776 777
    int sz = taosArrayGetSize(result);
    indexMultiTermQueryDestroy(mq);
    taosArrayDestroy(result);
dengyihao's avatar
dengyihao 已提交
778 779 780 781
    assert(sz == 1);
    uint64_t* ret = (uint64_t*)taosArrayGet(result, 0);
    assert(val = *ret);

782
    return sz;
dengyihao's avatar
dengyihao 已提交
783
  }
dengyihao's avatar
dengyihao 已提交
784

dengyihao's avatar
dengyihao 已提交
785
  void PutOne(const std::string& colName, const std::string& colVal) {
786
    SIndexMultiTerm* terms = indexMultiTermCreate();
dengyihao's avatar
dengyihao 已提交
787 788 789 790 791 792
    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    indexMultiTermAdd(terms, term);
    Put(terms, 10);
    indexMultiTermDestroy(terms);
  }
dengyihao's avatar
dengyihao 已提交
793 794 795 796 797 798 799 800
  void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
    SIndexMultiTerm* terms = indexMultiTermCreate();
    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    indexMultiTermAdd(terms, term);
    Put(terms, val);
    indexMultiTermDestroy(terms);
  }
dengyihao's avatar
dengyihao 已提交
801 802 803 804 805 806 807
  void Debug() {
    std::cout << "numOfWrite:" << numOfWrite << std::endl;
    std::cout << "numOfRead:" << numOfRead << std::endl;
  }

  ~IndexObj() {
    indexCleanUp();
dengyihao's avatar
dengyihao 已提交
808
    indexClose(idx);
dengyihao's avatar
dengyihao 已提交
809 810 811 812 813 814 815 816 817 818 819
  }

 private:
  SIndexOpts opts;
  SIndex*    idx;
  int        numOfWrite;
  int        numOfRead;
};

class IndexEnv2 : public ::testing::Test {
 protected:
dengyihao's avatar
dengyihao 已提交
820 821 822
  virtual void SetUp() { index = new IndexObj(); }
  virtual void TearDown() { delete index; }
  IndexObj*    index;
dengyihao's avatar
dengyihao 已提交
823 824
};
TEST_F(IndexEnv2, testIndexOpen) {
dengyihao's avatar
dengyihao 已提交
825
  std::string path = "/tmp/test";
826 827 828 829 830
  if (index->Init(path) != 0) {
    std::cout << "failed to init index" << std::endl;
    exit(1);
  }

dengyihao's avatar
dengyihao 已提交
831
  int targetSize = 200;
832
  {
dengyihao's avatar
dengyihao 已提交
833
    std::string colName("tag1"), colVal("Hello");
834 835 836 837 838 839 840 841 842 843 844 845 846

    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    SIndexMultiTerm* terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
    for (size_t i = 0; i < targetSize; i++) {
      int tableId = i;
      int ret = index->Put(terms, tableId);
      assert(ret == 0);
    }
    indexMultiTermDestroy(terms);
  }
  {
dengyihao's avatar
dengyihao 已提交
847 848
    size_t      size = 200;
    std::string colName("tag1"), colVal("hello");
849 850 851 852 853 854 855 856 857 858 859 860 861

    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    SIndexMultiTerm* terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
    for (size_t i = 0; i < size; i++) {
      int tableId = i;
      int ret = index->Put(terms, tableId);
      assert(ret == 0);
    }
    indexMultiTermDestroy(terms);
  }
  {
dengyihao's avatar
dengyihao 已提交
862
    size_t      size = 200;
dengyihao's avatar
dengyihao 已提交
863
    std::string colName("tag1"), colVal("Hello");
864

dengyihao's avatar
dengyihao 已提交
865 866 867 868 869 870 871 872 873 874 875 876 877 878
    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    SIndexMultiTerm* terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
    for (size_t i = size * 3; i < size * 4; i++) {
      int tableId = i;
      int ret = index->Put(terms, tableId);
      assert(ret == 0);
    }
    indexMultiTermDestroy(terms);
  }

  {
    std::string           colName("tag1"), colVal("Hello");
879 880 881 882 883 884 885
    SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                       colVal.c_str(), colVal.size());
    indexMultiTermQueryAdd(mq, term, QUERY_TERM);

    SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
    index->Search(mq, result);
dengyihao's avatar
dengyihao 已提交
886
    std::cout << "target size: " << taosArrayGetSize(result) << std::endl;
dengyihao's avatar
dengyihao 已提交
887
    assert(taosArrayGetSize(result) == 400);
888 889
    taosArrayDestroy(result);
    indexMultiTermQueryDestroy(mq);
dengyihao's avatar
dengyihao 已提交
890 891
  }
}
dengyihao's avatar
dengyihao 已提交
892 893

TEST_F(IndexEnv2, testIndex_TrigeFlush) {
dengyihao's avatar
dengyihao 已提交
894
  std::string path = "/tmp/testxxx";
dengyihao's avatar
dengyihao 已提交
895 896 897 898
  if (index->Init(path) != 0) {
    // r
    std::cout << "failed to init" << std::endl;
  }
dengyihao's avatar
dengyihao 已提交
899
  int numOfTable = 100 * 10000;
dengyihao's avatar
dengyihao 已提交
900 901 902
  index->WriteMillonData("tag1", "Hello Wolrd", numOfTable);
  int target = index->SearchOne("tag1", "Hello Wolrd");
  std::cout << "Get Index: " << target << std::endl;
dengyihao's avatar
dengyihao 已提交
903
  assert(numOfTable == target);
dengyihao's avatar
dengyihao 已提交
904
}
dengyihao's avatar
dengyihao 已提交
905

dengyihao's avatar
dengyihao 已提交
906
static void single_write_and_search(IndexObj* idx) {
dengyihao's avatar
dengyihao 已提交
907
  int target = idx->SearchOne("tag1", "Hello");
dengyihao's avatar
dengyihao 已提交
908
  target = idx->SearchOne("tag2", "Test");
dengyihao's avatar
dengyihao 已提交
909 910 911 912
}
static void multi_write_and_search(IndexObj* idx) {
  int target = idx->SearchOne("tag1", "Hello");
  target = idx->SearchOne("tag2", "Test");
dengyihao's avatar
dengyihao 已提交
913 914
  idx->WriteMultiMillonData("tag1", "hello world test", 100 * 10000);
  idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10000);
dengyihao's avatar
dengyihao 已提交
915
}
dengyihao's avatar
dengyihao 已提交
916
TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
dengyihao's avatar
dengyihao 已提交
917 918 919 920
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
    // opt
  }
dengyihao's avatar
dengyihao 已提交
921 922
  index->PutOne("tag1", "Hello");
  index->PutOne("tag2", "Test");
dengyihao's avatar
dengyihao 已提交
923 924
  index->WriteMultiMillonData("tag1", "Hello", 100 * 10000);
  index->WriteMultiMillonData("tag2", "Test", 100 * 10000);
dengyihao's avatar
dengyihao 已提交
925 926 927 928
  std::thread threads[NUM_OF_THREAD];

  for (int i = 0; i < NUM_OF_THREAD; i++) {
    //
dengyihao's avatar
dengyihao 已提交
929 930 931 932 933 934 935 936 937
    threads[i] = std::thread(single_write_and_search, index);
  }
  for (int i = 0; i < NUM_OF_THREAD; i++) {
    // TOD
    threads[i].join();
  }
}
TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
  std::string path = "/tmp/cache_and_tfile";
dengyihao's avatar
dengyihao 已提交
938 939
  if (index->Init(path) != 0) {
  }
dengyihao's avatar
dengyihao 已提交
940 941 942 943 944

  std::thread threads[NUM_OF_THREAD];
  for (int i = 0; i < NUM_OF_THREAD; i++) {
    //
    threads[i] = std::thread(multi_write_and_search, index);
dengyihao's avatar
dengyihao 已提交
945 946 947 948 949
  }
  for (int i = 0; i < NUM_OF_THREAD; i++) {
    // TOD
    threads[i].join();
  }
dengyihao's avatar
dengyihao 已提交
950 951
}

dengyihao's avatar
dengyihao 已提交
952
TEST_F(IndexEnv2, testIndex_restart) {
dengyihao's avatar
dengyihao 已提交
953
  std::string path = "/tmp/cache_and_tfile";
dengyihao's avatar
dengyihao 已提交
954 955 956 957 958 959 960 961 962 963
  if (index->Init(path) != 0) {
  }
  index->SearchOneTarget("tag1", "Hello", 10);
  index->SearchOneTarget("tag2", "Test", 10);
}
TEST_F(IndexEnv2, testIndex_restart1) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  index->ReadMultiMillonData("tag1", "coding");
dengyihao's avatar
dengyihao 已提交
964 965
  index->SearchOneTarget("tag1", "Hello", 10);
  index->SearchOneTarget("tag2", "Test", 10);
dengyihao's avatar
dengyihao 已提交
966
}
dengyihao's avatar
dengyihao 已提交
967

dengyihao's avatar
dengyihao 已提交
968 969
TEST_F(IndexEnv2, testIndex_read_performance) {
  std::string path = "/tmp/cache_and_tfile";
dengyihao's avatar
dengyihao 已提交
970 971
  if (index->Init(path) != 0) {
  }
dengyihao's avatar
dengyihao 已提交
972 973
  index->PutOneTarge("tag1", "Hello", 12);
  index->PutOneTarge("tag1", "Hello", 15);
dengyihao's avatar
dengyihao 已提交
974
  index->ReadMultiMillonData("tag1", "Hello");
dengyihao's avatar
dengyihao 已提交
975
  std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl;
dengyihao's avatar
dengyihao 已提交
976
  assert(3 == index->SearchOne("tag1", "Hello"));
dengyihao's avatar
dengyihao 已提交
977
}
dengyihao's avatar
dengyihao 已提交
978
TEST_F(IndexEnv2, testIndexMultiTag) {
dengyihao's avatar
dengyihao 已提交
979
  std::string path = "/tmp/multi_tag";
dengyihao's avatar
dengyihao 已提交
980 981
  if (index->Init(path) != 0) {
  }
dengyihao's avatar
dengyihao 已提交
982 983 984 985 986
  int64_t st = taosGetTimestampUs();
  int32_t num = 1000 * 10000;
  index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num);
  std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl;
  // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000);
dengyihao's avatar
dengyihao 已提交
987
}
dengyihao's avatar
dengyihao 已提交
988
TEST_F(IndexEnv2, testLongComVal1) {
dengyihao's avatar
dengyihao 已提交
989
  std::string path = "/tmp/long_colVal";
dengyihao's avatar
dengyihao 已提交
990 991
  if (index->Init(path) != 0) {
  }
dengyihao's avatar
dengyihao 已提交
992 993 994
  // gen colVal by randstr
  std::string randstr = "xxxxxxxxxxxxxxxxx";
  index->WriteMultiMillonData("tag1", randstr, 100 * 10000);
dengyihao's avatar
dengyihao 已提交
995
}
dengyihao's avatar
dengyihao 已提交
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060

TEST_F(IndexEnv2, testLongComVal2) {
  std::string path = "/tmp/long_colVal";
  if (index->Init(path) != 0) {
  }
  // gen colVal by randstr
  std::string randstr = "abcccc fdadfafdafda";
  index->WriteMultiMillonData("tag1", randstr, 100 * 10000);
}
TEST_F(IndexEnv2, testLongComVal3) {
  std::string path = "/tmp/long_colVal";
  if (index->Init(path) != 0) {
  }
  // gen colVal by randstr
  std::string randstr = "Yes, coding and coding and coding";
  index->WriteMultiMillonData("tag1", randstr, 100 * 10000);
}
TEST_F(IndexEnv2, testLongComVal4) {
  std::string path = "/tmp/long_colVal";
  if (index->Init(path) != 0) {
  }
  // gen colVal by randstr
  std::string randstr = "111111 bac fdadfa";
  index->WriteMultiMillonData("tag1", randstr, 100 * 10000);
}
TEST_F(IndexEnv2, testIndex_read_performance1) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  index->PutOneTarge("tag1", "Hello", 12);
  index->PutOneTarge("tag1", "Hello", 15);
  index->ReadMultiMillonData("tag1", "Hello", 1000);
  std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl;
  assert(3 == index->SearchOne("tag1", "Hello"));
}
TEST_F(IndexEnv2, testIndex_read_performance2) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  index->PutOneTarge("tag1", "Hello", 12);
  index->PutOneTarge("tag1", "Hello", 15);
  index->ReadMultiMillonData("tag1", "Hello", 1000 * 10);
  std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl;
  assert(3 == index->SearchOne("tag1", "Hello"));
}
TEST_F(IndexEnv2, testIndex_read_performance3) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  index->PutOneTarge("tag1", "Hello", 12);
  index->PutOneTarge("tag1", "Hello", 15);
  index->ReadMultiMillonData("tag1", "Hello", 1000 * 100);
  std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl;
  assert(3 == index->SearchOne("tag1", "Hello"));
}
TEST_F(IndexEnv2, testIndex_read_performance4) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  index->PutOneTarge("tag10", "Hello", 12);
  index->PutOneTarge("tag12", "Hello", 15);
  index->ReadMultiMillonData("tag10", "Hello", 1000 * 100);
  std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl;
  assert(3 == index->SearchOne("tag10", "Hello"));
}
dengyihao's avatar
test  
dengyihao 已提交
1061
TEST_F(IndexEnv2, testIndex_cache_del) {
dengyihao's avatar
dengyihao 已提交
1062 1063 1064 1065 1066 1067 1068 1069 1070
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  for (int i = 0; i < 100; i++) {
    index->PutOneTarge("tag10", "Hello", i);
  }
  index->Del("tag10", "Hello", 12);
  index->Del("tag10", "Hello", 11);

dengyihao's avatar
test  
dengyihao 已提交
1071 1072 1073
  // index->WriteMultiMillonData("tag10", "xxxxxxxxxxxxxx", 100 * 10000);
  index->Del("tag10", "Hello", 17);
  EXPECT_EQ(97, index->SearchOne("tag10", "Hello"));
dengyihao's avatar
dengyihao 已提交
1074

dengyihao's avatar
test  
dengyihao 已提交
1075
  index->PutOneTarge("tag10", "Hello", 17);  // add again
dengyihao's avatar
dengyihao 已提交
1076
  EXPECT_EQ(98, index->SearchOne("tag10", "Hello"));
dengyihao's avatar
test  
dengyihao 已提交
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108

  // del all
  for (int i = 0; i < 200; i++) {
    index->Del("tag10", "Hello", i);
  }
  EXPECT_EQ(0, index->SearchOne("tag10", "Hello"));

  // add other item
  for (int i = 0; i < 2000; i++) {
    index->PutOneTarge("tag10", "World", i);
  }

  for (int i = 0; i < 2000; i++) {
    index->PutOneTarge("tag10", "Hello", i);
  }
  EXPECT_EQ(2000, index->SearchOne("tag10", "Hello"));

  for (int i = 0; i < 2000; i++) {
    index->Del("tag10", "Hello", i);
  }
  EXPECT_EQ(0, index->SearchOne("tag10", "Hello"));
}

TEST_F(IndexEnv2, testIndex_del) {
  std::string path = "/tmp/cache_and_tfile";
  if (index->Init(path) != 0) {
  }
  for (int i = 0; i < 100; i++) {
    index->PutOneTarge("tag10", "Hello", i);
  }
  index->Del("tag10", "Hello", 12);
  index->Del("tag10", "Hello", 11);
dengyihao's avatar
add UT  
dengyihao 已提交
1109
  EXPECT_EQ(98, index->SearchOne("tag10", "Hello"));
dengyihao's avatar
test  
dengyihao 已提交
1110 1111 1112 1113

  index->WriteMultiMillonData("tag10", "xxxxxxxxxxxxxx", 100 * 10000);
  index->Del("tag10", "Hello", 17);
  EXPECT_EQ(97, index->SearchOne("tag10", "Hello"));
dengyihao's avatar
dengyihao 已提交
1114
}