indexTests.cc

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
#include <gtest/gtest.h>
#include <string>
#include <iostream>
#include "index.h"
#include "tutil.h"
#include "indexInt.h"
#include "index_fst.h"
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"


class FstWriter {
  public:
    FstWriter() {
      _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0); 
      _b = fstBuilderCreate(NULL, 0);
    }  
   bool Put(const std::string &key, uint64_t val) {
      FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());   
      bool ok = fstBuilderInsert(_b, skey, val);
      fstSliceDestroy(&skey);
      return ok;
   }
   ~FstWriter() {
     fstBuilderFinish(_b);
     fstBuilderDestroy(_b);

     writerCtxDestroy(_wc);
   }
  private:
    FstBuilder *_b; 
    WriterCtx *_wc;
};

class FstReadMemory {
  public:
   FstReadMemory(size_t size) {
     _wc   = writerCtxCreate(TFile, "/tmp/tindex", true, 0);   
     _w    = fstCountingWriterCreate(_wc); 
     _size = size; 
     memset((void *)&_s, 0, sizeof(_s));
   }
   bool init() {
     char *buf = (char *)calloc(1,  sizeof(char) * _size);
     int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size); 
     if (nRead <= 0) { return false; } 
      _size = nRead;
     _s   = fstSliceCreate((uint8_t *)buf, _size);  
     _fst = fstCreate(&_s); 
     free(buf);
     return _fst != NULL;
   }
   bool Get(const std::string &key, uint64_t *val) {
     FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());   
     bool ok = fstGet(_fst, &skey, val); 
     fstSliceDestroy(&skey);
     return ok;
   }
   bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) {
     int64_t s = taosGetTimestampUs();
     bool ok = this->Get(key, val); 
     int64_t e = taosGetTimestampUs();
     *elapse = e - s;
     return ok; 
   }
   // add later
   bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) { 
      FstStreamBuilder *sb = fstSearch(_fst, ctx);
      StreamWithState  *st = streamBuilderIntoStream(sb);  
      StreamWithStateResult *rt = NULL;  
      
      while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
        result.push_back((uint64_t)(rt->out.out));
      }
      return true;
   }
   bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
     int64_t s = taosGetTimestampUs();
     bool ok = this->Search(ctx, result); 
     int64_t e = taosGetTimestampUs();
     return ok;
   }
    
   ~FstReadMemory() {
    fstCountingWriterDestroy(_w);
    fstDestroy(_fst);
    fstSliceDestroy(&_s);
    writerCtxDestroy(_wc);
  } 
  
  private:
   FstCountingWriter *_w; 
   Fst *_fst;
   FstSlice _s;  
   WriterCtx *_wc;
   size_t _size;
   
}; 

//TEST(IndexTest, index_create_test) {
//  SIndexOpts *opts = indexOptsCreate();
//  SIndex *index = indexOpen(opts, "./test");
//  if (index == NULL) {
//    std::cout << "index open failed" << std::endl; 
//  }
//
//  
//  // write   
//  for (int i = 0; i < 100000; i++) {
//    SIndexMultiTerm* terms = indexMultiTermCreate();
//    std::string val = "field";    
//
//    indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
//
//    val.append(std::to_string(i)); 
//    indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
//
//    val.insert(0, std::to_string(i));
//    indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
//
//    val.append("const");    
//    indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
//
//     
//    indexPut(index, terms, i);
//    indexMultiTermDestroy(terms);
//  } 
// 
//
//  // query
//  SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); 
//  
//  indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
//  indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
//
//  SArray *result = (SArray *)taosArrayInit(10, sizeof(int));   
//  indexSearch(index, multiQuery, result);
//
//  std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
//  for (int i = 0;  i < taosArrayGetSize(result); i++) {
//    int *v = (int *)taosArrayGet(result, i);
//    std::cout << "value --->" << *v  << std::endl;
//  }
//  // add more test case 
//  indexMultiTermQueryDestroy(multiQuery);
//
//  indexOptsDestroy(opts); 
//  indexClose(index); 
//  //
//}


#define L 100
#define M 100
#define N 100

int Performance_fstWriteRecords(FstWriter *b) {
  std::string str("aa"); 
  for (int i = 0; i < L; i++) {
    str[0] = 'a' + i;
    str.resize(2); 
    for(int j = 0; j < M; j++) {
      str[1] = 'a' + j;
      str.resize(2);
      for (int k = 0; k < N; k++) {
        str.push_back('a');
        b->Put(str, k);
        printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
      }
    } 
  }
  return L * M * N;
}

void Performance_fstReadRecords(FstReadMemory *m) {
  std::string str("aa");
  for (int i = 0; i < M; i++) {
    str[0] = 'a' + i;
    str.resize(2); 
    for(int j = 0; j < N; j++) {
      str[1] = 'a' + j;
      str.resize(2);
      for (int k = 0; k < L; k++) {
        str.push_back('a');
        uint64_t val, cost; 
        if (m->GetWithTimeCostUs(str, &val, &cost)) {
          printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
        } else {
          printf("failed to get key: %s\n", str.c_str());
        }
      }
    } 
  }
}
void checkFstPerf() {
  FstWriter *fw = new FstWriter;
  int64_t s = taosGetTimestampUs();

  int num = Performance_fstWriteRecords(fw);
  int64_t e = taosGetTimestampUs();
  printf("write %d record cost %" PRId64"us\n", num,  e - s);
  delete fw;

  FstReadMemory *m = new FstReadMemory(1024 * 64);
  if (m->init()) {
    printf("success to init fst read");  
  }  
  Performance_fstReadRecords(m); 
  delete m;
} 

void checkFstPrefixSearch() {
  FstWriter *fw = new FstWriter;
  int64_t s = taosGetTimestampUs();
  int count = 2;
  std::string key("ab");
  
  for (int i = 0; i < count; i++) {
    key[1] = key[1] + i;   
    fw->Put(key, i); 
  }
  int64_t e = taosGetTimestampUs();
  
  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
  delete fw;

  FstReadMemory *m = new FstReadMemory(1024 * 64);
  if (m->init() == false) {
    std::cout << "init readMemory failed" << std::endl; 
    delete m;
    return;
  }
  
  // prefix search 
  std::vector<uint64_t> result;
  AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX); 
  m->Search(ctx, result);   
  assert(result.size() == count); 
  for (int i = 0; i < result.size(); i++) {
    assert(result[i] == i); // check result
  }

  free(ctx);
  delete m;
}  
void validateFst() {
  int val = 100;
  int count = 100;
  FstWriter *fw = new FstWriter;
  // write 
  {
    std::string key("ab");
    for (int i = 0; i < count; i++) {
      key.push_back('a' + i);
      fw->Put(key, val - i);
    }
  }
  delete fw;

  // read
  FstReadMemory *m = new FstReadMemory(1024 * 64);
  if (m->init() == false) { 
    std::cout << "init readMemory failed" << std::endl; 
    delete m;
    return;
  }

  {
   std::string key("ab");
   uint64_t out;
   if (m->Get(key, &out)) {
     printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
   } else {
     printf("failed to get(%s)\n", key.c_str());
   }
   for (int i = 0; i < count; i++) {
     key.push_back('a' + i);
     if (m->Get(key, &out) ) {
       assert(val - i ==  out);
       printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
     } else {
       printf("failed to get(%s)\n", key.c_str());
    }
   }
  } 
  delete m;
} 

class IndexEnv : public ::testing::Test {
  protected:
    virtual void SetUp() {
      taosRemoveDir(path);  
      opts = indexOptsCreate(); 
      int ret = indexOpen(opts, path, &index); 
      assert(ret == 0);
    } 
    virtual void TearDown() {
      indexClose(index);
      indexOptsDestroy(opts);
    }
    
    const char *path = "/tmp/tindex";
    SIndexOpts *opts;  
    SIndex *index; 
};

TEST_F(IndexEnv, testPut) {

   // single index column 
   {
     
    std::string colName("tag1"), colVal("Hello world");
    SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
    SIndexMultiTerm *terms = indexMultiTermCreate();
    indexMultiTermAdd(terms, term);
   
    for (size_t i = 0; i < 100; i++) {
      int tableId = i;
      int ret = indexPut(index, terms, tableId);
      assert(ret == 0); 
    }
    indexMultiTermDestroy(terms);
   }
   // multi index column
   {
    
    SIndexMultiTerm *terms = indexMultiTermCreate();
    {
      std::string colName("tag1"), colVal("Hello world");
      SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
      indexMultiTermAdd(terms, term);
    }
    {
      std::string colName("tag2"), colVal("Hello world");
      SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
      indexMultiTermAdd(terms, term);
    }
    
    for (int i = 0; i < 100; i++) {
      int tableId = i;
      int ret = indexPut(index, terms, tableId);
      assert(ret == 0); 
    } 
    indexMultiTermDestroy(terms);
   }
   //   
} 

TEST_F(IndexEnv, testDel) {
     
}