fstTest.cc 17.7 KB
Newer Older
dengyihao's avatar
dengyihao 已提交
1

dengyihao's avatar
dengyihao 已提交
2
#include <algorithm>
dengyihao's avatar
dengyihao 已提交
3 4
#include <iostream>
#include <string>
dengyihao's avatar
dengyihao 已提交
5
#include <thread>
dengyihao's avatar
dengyihao 已提交
6 7
#include <vector>
#include "index.h"
dengyihao's avatar
dengyihao 已提交
8 9 10 11
#include "indexCache.h"
#include "indexFst.h"
#include "indexFstCountingWriter.h"
#include "indexFstUtil.h"
dengyihao's avatar
dengyihao 已提交
12
#include "indexInt.h"
dengyihao's avatar
dengyihao 已提交
13
#include "indexTfile.h"
dengyihao's avatar
dengyihao 已提交
14 15
#include "tskiplist.h"
#include "tutil.h"
dengyihao's avatar
dengyihao 已提交
16
void* callback(void* s) { return s; }
dengyihao's avatar
dengyihao 已提交
17 18 19 20 21

static std::string fileName = "/tmp/tindex.tindex";
class FstWriter {
 public:
  FstWriter() {
22
    taosRemoveFile(fileName.c_str());
dengyihao's avatar
dengyihao 已提交
23 24 25 26
    _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024);
    _b = fstBuilderCreate(_wc, 0);
  }
  bool Put(const std::string& key, uint64_t val) {
dengyihao's avatar
dengyihao 已提交
27 28 29 30
    // char buf[128] = {0};
    // int  len = 0;
    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
dengyihao's avatar
dengyihao 已提交
31 32
    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
    bool     ok = fstBuilderInsert(_b, skey, val);
dengyihao's avatar
dengyihao 已提交
33

dengyihao's avatar
dengyihao 已提交
34 35 36 37 38 39 40
    fstSliceDestroy(&skey);
    return ok;
  }
  ~FstWriter() {
    fstBuilderFinish(_b);
    fstBuilderDestroy(_b);

dengyihao's avatar
dengyihao 已提交
41
    writerCtxDestroy(_wc, false);
dengyihao's avatar
dengyihao 已提交
42 43 44 45 46 47 48 49 50
  }

 private:
  FstBuilder* _b;
  WriterCtx*  _wc;
};

class FstReadMemory {
 public:
dengyihao's avatar
dengyihao 已提交
51
  FstReadMemory(int32_t size, const std::string& fileName = "/tmp/tindex.tindex") {
dengyihao's avatar
dengyihao 已提交
52 53 54 55 56 57
    _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024);
    _w = fstCountingWriterCreate(_wc);
    _size = size;
    memset((void*)&_s, 0, sizeof(_s));
  }
  bool init() {
wafwerar's avatar
wafwerar 已提交
58
    char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size);
dengyihao's avatar
dengyihao 已提交
59
    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
dengyihao's avatar
dengyihao 已提交
60 61 62
    if (nRead <= 0) {
      return false;
    }
dengyihao's avatar
dengyihao 已提交
63 64 65
    _size = nRead;
    _s = fstSliceCreate((uint8_t*)buf, _size);
    _fst = fstCreate(&_s);
wafwerar's avatar
wafwerar 已提交
66
    taosMemoryFree(buf);
dengyihao's avatar
dengyihao 已提交
67 68 69
    return _fst != NULL;
  }
  bool Get(const std::string& key, uint64_t* val) {
dengyihao's avatar
dengyihao 已提交
70 71 72 73 74
    // char buf[128] = {0};
    // int  len = 0;
    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);

dengyihao's avatar
dengyihao 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
    bool     ok = fstGet(_fst, &skey, val);
    fstSliceDestroy(&skey);
    return ok;
  }
  bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
    int64_t s = taosGetTimestampUs();
    bool    ok = this->Get(key, val);
    int64_t e = taosGetTimestampUs();
    *elapse = e - s;
    return ok;
  }
  // add later
  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
    StreamWithState*       st = streamBuilderIntoStream(sb);
    StreamWithStateResult* rt = NULL;
    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
      // result.push_back((uint64_t)(rt->out.out));
      FstSlice*   s = &rt->data;
      int32_t     sz = 0;
      char*       ch = (char*)fstSliceData(s, &sz);
      std::string key(ch, sz);
      printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
dengyihao's avatar
dengyihao 已提交
99
      result.push_back(rt->out.out);
dengyihao's avatar
dengyihao 已提交
100
      swsResultDestroy(rt);
dengyihao's avatar
dengyihao 已提交
101
    }
dengyihao's avatar
dengyihao 已提交
102 103
    streamWithStateDestroy(st);
    fstStreamBuilderDestroy(sb);
dengyihao's avatar
dengyihao 已提交
104 105
    return true;
  }
dengyihao's avatar
dengyihao 已提交
106 107
  bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
                   RangeType highType, std::vector<uint64_t>& result) {
dengyihao's avatar
dengyihao 已提交
108 109 110 111 112 113
    FstStreamBuilder* sb = fstSearch(_fst, ctx);

    FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
    FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());

    // range [low, high);
dengyihao's avatar
dengyihao 已提交
114 115
    fstStreamBuilderSetRange(sb, &l, lowType);
    fstStreamBuilderSetRange(sb, &h, highType);
dengyihao's avatar
dengyihao 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

    fstSliceDestroy(&l);
    fstSliceDestroy(&h);

    StreamWithState*       st = streamBuilderIntoStream(sb);
    StreamWithStateResult* rt = NULL;
    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
      // result.push_back((uint64_t)(rt->out.out));
      FstSlice*   s = &rt->data;
      int32_t     sz = 0;
      char*       ch = (char*)fstSliceData(s, &sz);
      std::string key(ch, sz);
      printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
      result.push_back(rt->out.out);
      swsResultDestroy(rt);
dengyihao's avatar
dengyihao 已提交
131
    }
dengyihao's avatar
dengyihao 已提交
132 133
    streamWithStateDestroy(st);
    fstStreamBuilderDestroy(sb);
dengyihao's avatar
dengyihao 已提交
134 135 136 137 138 139 140 141 142 143 144 145 146
    return true;
  }
  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
    int64_t s = taosGetTimestampUs();
    bool    ok = this->Search(ctx, result);
    int64_t e = taosGetTimestampUs();
    return ok;
  }

  ~FstReadMemory() {
    fstCountingWriterDestroy(_w);
    fstDestroy(_fst);
    fstSliceDestroy(&_s);
dengyihao's avatar
dengyihao 已提交
147
    writerCtxDestroy(_wc, false);
dengyihao's avatar
dengyihao 已提交
148 149 150 151 152 153 154
  }

 private:
  FstCountingWriter* _w;
  Fst*               _fst;
  FstSlice           _s;
  WriterCtx*         _wc;
dengyihao's avatar
dengyihao 已提交
155
  int32_t            _size;
dengyihao's avatar
dengyihao 已提交
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
};

#define L 100
#define M 100
#define N 100

int Performance_fstWriteRecords(FstWriter* b) {
  std::string str("aa");
  for (int i = 0; i < L; i++) {
    str[0] = 'a' + i;
    str.resize(2);
    for (int j = 0; j < M; j++) {
      str[1] = 'a' + j;
      str.resize(2);
      for (int k = 0; k < N; k++) {
        str.push_back('a');
        b->Put(str, k);
        printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
      }
    }
  }
  return L * M * N;
}
dengyihao's avatar
dengyihao 已提交
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
void Performance_fstReadRecords(FstReadMemory* m) {
  std::string str("aa");
  for (int i = 0; i < M; i++) {
    str[0] = 'a' + i;
    str.resize(2);
    for (int j = 0; j < N; j++) {
      str[1] = 'a' + j;
      str.resize(2);
      for (int k = 0; k < L; k++) {
        str.push_back('a');
        uint64_t val, cost;
        if (m->GetWithTimeCostUs(str, &val, &cost)) {
          printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
        } else {
          printf("failed to get key: %s\n", str.c_str());
        }
      }
    }
  }
}

void checkMillonWriteAndReadOfFst() {
  FstWriter* fw = new FstWriter;
  Performance_fstWriteRecords(fw);
  delete fw;
  FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);

dengyihao's avatar
dengyihao 已提交
206 207 208
  if (fr->init()) {
    printf("success to init fst read");
  }
dengyihao's avatar
dengyihao 已提交
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262

  Performance_fstReadRecords(fr);
  delete fr;
}
void checkFstLongTerm() {
  FstWriter* fw = new FstWriter;
  // Performance_fstWriteRecords(fw);

  fw->Put("A B", 1);
  fw->Put("C", 2);
  fw->Put("a", 3);
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }
  {
    uint64_t val = 0;
    if (m->Get("A B", &val)) {
      std::cout << "success to Get: " << val << std::endl;
    } else {
      std::cout << "failed to Get:" << val << std::endl;
    }
  }
  {
    uint64_t val = 0;
    if (m->Get("C", &val)) {
      std::cout << "success to Get: " << val << std::endl;
    } else {
      std::cout << "failed to Get:" << val << std::endl;
    }
  }
  {
    uint64_t val = 0;
    if (m->Get("a", &val)) {
      std::cout << "success to Get: " << val << std::endl;
    } else {
      std::cout << "failed to Get:" << val << std::endl;
    }
  }

  // prefix search
  // std::vector<uint64_t> result;

  // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
  // m->Search(ctx, result);
  // std::cout << "size: " << result.size() << std::endl;
  // assert(result.size() == count);
  // for (int i = 0; i < result.size(); i++) {
  // assert(result[i] == i);  // check result
  //}
wafwerar's avatar
wafwerar 已提交
263
  // taosMemoryFree(ctx);
dengyihao's avatar
dengyihao 已提交
264 265
  // delete m;
}
dengyihao's avatar
dengyihao 已提交
266
void checkFstCheckIterator1() {
dengyihao's avatar
dengyihao 已提交
267 268 269
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
dengyihao's avatar
dengyihao 已提交
270
  // Performance_fstWriteRecords(fw);
dengyihao's avatar
dengyihao 已提交
271 272 273
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
dengyihao's avatar
dengyihao 已提交
274

275 276
  fw->Put("test1&^D&10", 1);
  fw->Put("test2&^D&10", 2);
dengyihao's avatar
dengyihao 已提交
277 278 279 280 281 282 283 284 285 286 287 288
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }

  // prefix search
  std::vector<uint64_t> result;

dengyihao's avatar
dengyihao 已提交
289 290 291 292 293 294 295
  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
  m->Search(ctx, result);
  std::cout << "size: " << result.size() << std::endl;
  // assert(result.size() == count);
  for (int i = 0; i < result.size(); i++) {
    // assert(result[i] == i);  // check result
  }
dengyihao's avatar
dengyihao 已提交
296
  automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
297 298 299

  delete m;
}
dengyihao's avatar
dengyihao 已提交
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
void checkFstCheckIterator2() {
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
  // Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;

  fw->Put("a", 1);
  fw->Put("b", 2);
  fw->Put("c", 4);
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }

  // prefix search
  std::vector<uint64_t> result;

  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
  m->Search(ctx, result);
  std::cout << "size: " << result.size() << std::endl;
  // assert(result.size() == count);
  for (int i = 0; i < result.size(); i++) {
    // assert(result[i] == i);  // check result
  }
dengyihao's avatar
dengyihao 已提交
331
  automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
332 333 334

  delete m;
}
dengyihao's avatar
dengyihao 已提交
335 336 337 338 339 340 341 342 343 344 345 346
void checkFstCheckIteratorPrefix() {
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
  // Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;

  fw->Put("Hello world", 1);
  fw->Put("Hello worle", 2);
  fw->Put("hello worlf", 4);
dengyihao's avatar
dengyihao 已提交
347 348 349 350 351
  fw->Put("ja", 4);
  fw->Put("jb", 4);
  fw->Put("jc", 4);
  fw->Put("jddddddddd", 4);
  fw->Put("jefffffff", 4);
dengyihao's avatar
dengyihao 已提交
352 353 354 355 356 357 358 359
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }
dengyihao's avatar
dengyihao 已提交
360 361 362
  {
    // prefix search
    std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
363

dengyihao's avatar
dengyihao 已提交
364 365 366
    AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
    m->Search(ctx, result);
    assert(result.size() == 1);
dengyihao's avatar
dengyihao 已提交
367
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
368 369 370 371
  }
  {
    // prefix search
    std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
372

dengyihao's avatar
dengyihao 已提交
373 374 375
    AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
    m->Search(ctx, result);
    assert(result.size() == 2);
dengyihao's avatar
dengyihao 已提交
376
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
377
  }
dengyihao's avatar
dengyihao 已提交
378 379
  {
    std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
380

dengyihao's avatar
dengyihao 已提交
381 382 383
    AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
    m->Search(ctx, result);
    assert(result.size() == 1);
dengyihao's avatar
dengyihao 已提交
384
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
385
  }
dengyihao's avatar
dengyihao 已提交
386 387
  delete m;
}
dengyihao's avatar
dengyihao 已提交
388
void checkFstCheckIteratorRange1() {
dengyihao's avatar
dengyihao 已提交
389 390 391 392 393 394 395 396 397 398 399 400 401
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
  // Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;

  fw->Put("a", 1);
  fw->Put("b", 2);
  fw->Put("c", 3);
  fw->Put("d", 4);
  fw->Put("e", 5);
dengyihao's avatar
dengyihao 已提交
402 403
  fw->Put("f", 5);
  fw->Put("G", 5);
dengyihao's avatar
dengyihao 已提交
404 405 406 407 408 409 410 411 412 413 414
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }
  {
    // prefix search
    std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
415
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
dengyihao's avatar
dengyihao 已提交
416
    // [b, e)
dengyihao's avatar
dengyihao 已提交
417
    m->SearchRange(ctx, "b", GE, "e", LT, result);
dengyihao's avatar
dengyihao 已提交
418
    assert(result.size() == 3);
dengyihao's avatar
dengyihao 已提交
419
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
420
  }
dengyihao's avatar
dengyihao 已提交
421 422 423 424 425 426 427
  {
    // prefix search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GT, "e", LT, result);
    assert(result.size() == 2);
dengyihao's avatar
dengyihao 已提交
428
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
429 430 431 432 433 434 435 436
  }
  {
    // prefix search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GT, "e", LE, result);
    assert(result.size() == 3);
dengyihao's avatar
dengyihao 已提交
437
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
438 439 440 441 442 443 444 445
  }
  {
    // prefix search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GE, "e", LE, result);
    assert(result.size() == 4);
dengyihao's avatar
dengyihao 已提交
446
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
447
  }
dengyihao's avatar
dengyihao 已提交
448
  delete m;
dengyihao's avatar
dengyihao 已提交
449 450 451 452 453 454 455 456 457 458 459
}
void checkFstCheckIteratorRange2() {
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
  // Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;

  fw->Put("ab", 1);
dengyihao's avatar
dengyihao 已提交
460
  fw->Put("b", 2);
dengyihao's avatar
dengyihao 已提交
461 462 463 464 465 466 467 468 469 470 471 472 473
  fw->Put("cdd", 3);
  fw->Put("cde", 3);
  fw->Put("ddd", 4);
  fw->Put("ed", 5);
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }
  {
dengyihao's avatar
dengyihao 已提交
474
    // range  search
dengyihao's avatar
dengyihao 已提交
475
    std::vector<uint64_t> result;
dengyihao's avatar
dengyihao 已提交
476
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
dengyihao's avatar
dengyihao 已提交
477
    // [b, e)
dengyihao's avatar
dengyihao 已提交
478
    m->SearchRange(ctx, "b", GE, "ed", LT, result);
dengyihao's avatar
dengyihao 已提交
479
    assert(result.size() == 4);
dengyihao's avatar
dengyihao 已提交
480
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
481
  }
dengyihao's avatar
dengyihao 已提交
482 483 484 485 486 487 488 489 490
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "bb", GE, "ed", LT, result);
    assert(result.size() == 3);
    automCtxDestroy(ctx);
  }
dengyihao's avatar
dengyihao 已提交
491 492 493 494 495 496 497
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GE, "ed", LE, result);
    assert(result.size() == 5);
dengyihao's avatar
dengyihao 已提交
498 499
    automCtxDestroy(ctx);
    // taosMemoryFree(ctx);
dengyihao's avatar
dengyihao 已提交
500 501 502 503 504 505 506 507
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GT, "ed", LE, result);
    assert(result.size() == 4);
dengyihao's avatar
dengyihao 已提交
508
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
509 510 511 512 513 514 515 516
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GT, "ed", LT, result);
    assert(result.size() == 3);
dengyihao's avatar
dengyihao 已提交
517
    automCtxDestroy(ctx);
dengyihao's avatar
dengyihao 已提交
518
  }
dengyihao's avatar
dengyihao 已提交
519
  delete m;
dengyihao's avatar
dengyihao 已提交
520
}
dengyihao's avatar
dengyihao 已提交
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
void checkFstCheckIteratorRange3() {
  FstWriter* fw = new FstWriter;
  int64_t    s = taosGetTimestampUs();
  int        count = 2;
  // Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();

  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;

  fw->Put("ab", 1);
  fw->Put("b", 2);
  fw->Put("cdd", 3);
  fw->Put("cde", 3);
  fw->Put("ddd", 4);
  fw->Put("ed", 5);
  delete fw;

  FstReadMemory* m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl;
    delete m;
    return;
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "b", GE, "", (RangeType)10, result);
    assert(result.size() == 5);
    automCtxDestroy(ctx);
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result);
    assert(result.size() == 1);
    automCtxDestroy(ctx);
    // taosMemoryFree(ctx);
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result);
    assert(result.size() == 0);
    automCtxDestroy(ctx);
  }
  {
    // range  search
    std::vector<uint64_t> result;
    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
    // [b, e)
    m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result);
    assert(result.size() == 0);
    automCtxDestroy(ctx);
  }
  delete m;
}
dengyihao's avatar
dengyihao 已提交
583 584 585

void fst_get(Fst* fst) {
  for (int i = 0; i < 10000; i++) {
dengyihao's avatar
dengyihao 已提交
586
    std::string term = "Hello World";
dengyihao's avatar
dengyihao 已提交
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601
    FstSlice    key = fstSliceCreate((uint8_t*)term.c_str(), term.size());
    uint64_t    offset = 0;
    bool        ret = fstGet(fst, &key, &offset);
    if (ret == false) {
      std::cout << "not found" << std::endl;
    } else {
      std::cout << "found value:" << offset << std::endl;
    }
  }
}

#define NUM_OF_THREAD 10
void validateTFile(char* arg) {
  std::thread threads[NUM_OF_THREAD];
  // std::vector<std::thread> threads;
dengyihao's avatar
dengyihao 已提交
602
  TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1");
dengyihao's avatar
dengyihao 已提交
603 604 605 606 607 608 609 610 611 612 613

  for (int i = 0; i < NUM_OF_THREAD; i++) {
    threads[i] = std::thread(fst_get, reader->fst);
    // threads.push_back(fst_get, reader->fst);
    // std::thread t(fst_get, reader->fst);
  }
  for (int i = 0; i < NUM_OF_THREAD; i++) {
    // wait join
    threads[i].join();
  }
}
dengyihao's avatar
dengyihao 已提交
614

dengyihao's avatar
add UT  
dengyihao 已提交
615
void iterTFileReader(char* path, char* uid, char* colName, char* ver) {
dengyihao's avatar
dengyihao 已提交
616
  // tfInit();
dengyihao's avatar
dengyihao 已提交
617

dengyihao's avatar
add UT  
dengyihao 已提交
618 619 620 621 622 623 624 625 626
  uint64_t suid = atoi(uid);
  int      version = atoi(ver);

  TFileReader* reader = tfileReaderOpen(path, suid, version, colName);

  Iterate* iter = tfileIteratorCreate(reader);
  bool     tn = iter ? iter->next(iter) : false;
  int      count = 0;
  int      termCount = 0;
dengyihao's avatar
dengyihao 已提交
627 628 629 630 631 632 633 634 635 636 637 638
  while (tn == true) {
    count++;
    IterateValue* cv = iter->getValue(iter);
    termCount += (int)taosArrayGetSize(cv->val);
    printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val));
    tn = iter->next(iter);
  }
  printf("total size: %d\n term count: %d\n", count, termCount);

  tfileIteratorDestroy(iter);
}

dengyihao's avatar
dengyihao 已提交
639
int main(int argc, char* argv[]) {
dengyihao's avatar
dengyihao 已提交
640
  // tool to check all kind of fst test
dengyihao's avatar
dengyihao 已提交
641
  // if (argc > 1) { validateTFile(argv[1]); }
dengyihao's avatar
dengyihao 已提交
642 643 644 645
  // if (argc > 4) {
  // path suid colName ver
  // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
  //}
dengyihao's avatar
dengyihao 已提交
646
  checkFstCheckIterator1();
647 648 649 650 651
  // checkFstCheckIterator2();
  // checkFstCheckIteratorPrefix();
  // checkFstCheckIteratorRange1();
  // checkFstCheckIteratorRange2();
  // checkFstCheckIteratorRange3();
dengyihao's avatar
dengyihao 已提交
652
  // checkFstLongTerm();
dengyihao's avatar
dengyihao 已提交
653
  // checkFstPrefixSearch();
dengyihao's avatar
dengyihao 已提交
654

dengyihao's avatar
dengyihao 已提交
655 656
  // checkMillonWriteAndReadOfFst();

dengyihao's avatar
dengyihao 已提交
657 658
  return 1;
}