#include #include #include #include "index.h" #include "tutil.h" #include "indexInt.h" #include "index_fst.h" #include "index_fst_util.h" #include "index_fst_counting_writer.h" class FstWriter { public: FstWriter() { _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string &key, uint64_t val) { FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); bool ok = fstBuilderInsert(_b, skey, val); fstSliceDestroy(&skey); return ok; } ~FstWriter() { fstBuilderFinish(_b); fstBuilderDestroy(_b); } private: FstBuilder *_b; }; class FstReadMemory { public: FstReadMemory(size_t size) { _w = fstCountingWriterCreate(NULL, true); _size = size; memset((void *)&_s, 0, sizeof(_s)); } bool init() { char *buf = (char *)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size); if (nRead <= 0) { return false; } _size = nRead; _s = fstSliceCreate((uint8_t *)buf, _size); _fst = fstCreate(&_s); free(buf); return _fst != NULL; } bool Get(const std::string &key, uint64_t *val) { FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); bool ok = fstGet(_fst, &skey, val); fstSliceDestroy(&skey); return ok; } bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) { int64_t s = taosGetTimestampUs(); bool ok = this->Get(key, val); int64_t e = taosGetTimestampUs(); *elapse = e - s; return ok; } // add later bool Search(const std::string &key, std::vector &result) { return true; } ~FstReadMemory() { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); } private: FstCountingWriter *_w; Fst *_fst; FstSlice _s; size_t _size; }; //TEST(IndexTest, index_create_test) { // SIndexOpts *opts = indexOptsCreate(); // SIndex *index = indexOpen(opts, "./test"); // if (index == NULL) { // std::cout << "index open failed" << std::endl; // } // // // // write // for (int i = 0; i < 100000; i++) { // SIndexMultiTerm* terms = indexMultiTermCreate(); // std::string val = "field"; // // indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); // // val.append(std::to_string(i)); // indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); // // val.insert(0, std::to_string(i)); // indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); // // val.append("const"); // indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); // // // indexPut(index, terms, i); // indexMultiTermDestroy(terms); // } // // // // query // SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); // // indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); // indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); // // SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); // indexSearch(index, multiQuery, result); // // std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; // for (int i = 0; i < taosArrayGetSize(result); i++) { // int *v = (int *)taosArrayGet(result, i); // std::cout << "value --->" << *v << std::endl; // } // // add more test case // indexMultiTermQueryDestroy(multiQuery); // // indexOptsDestroy(opts); // indexClose(index); // // //} #define L 100 #define M 100 #define N 100 int Performance_fstWriteRecords(FstWriter *b) { std::string str("aa"); for (int i = 0; i < L; i++) { str[0] = 'a' + i; str.resize(2); for(int j = 0; j < M; j++) { str[1] = 'a' + j; str.resize(2); for (int k = 0; k < N; k++) { str.push_back('a'); b->Put(str, k); printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str()); } } } return L * M * N; } void Performance_fstReadRecords(FstReadMemory *m) { std::string str("aa"); for (int i = 0; i < M; i++) { str[0] = 'a' + i; str.resize(2); for(int j = 0; j < N; j++) { str[1] = 'a' + j; str.resize(2); for (int k = 0; k < L; k++) { str.push_back('a'); uint64_t val, cost; if (m->GetWithTimeCostUs(str, &val, &cost)) { printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost); } else { printf("failed to get key: %s\n", str.c_str()); } } } } } void checkFstPerf() { FstWriter *fw = new FstWriter; int64_t s = taosGetTimestampUs(); int num = Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); printf("write %d record cost %" PRId64"us\n", num, e - s); delete fw; FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init()) { printf("success to init fst read"); } Performance_fstReadRecords(m); delete m; } void validateFst() { int val = 100; int count = 100; FstWriter *fw = new FstWriter; // write { std::string key("ab"); for (int i = 0; i < count; i++) { key.push_back('a' + i); fw->Put(key, val - i); } } delete fw; // read FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init() == false) { std::cout << "init readMemory failed" << std::endl; } { std::string key("ab"); uint64_t out; if (m->Get(key, &out)) { printf("success to get (%s, %" PRId64")\n", key.c_str(), out); } else { printf("failed to get(%s)\n", key.c_str()); } for (int i = 0; i < count; i++) { key.push_back('a' + i); if (m->Get(key, &out) ) { assert(val - i == out); printf("success to get (%s, %" PRId64")\n", key.c_str(), out); } else { printf("failed to get(%s)\n", key.c_str()); } } } delete m; } int main(int argc, char** argv) { checkFstPerf(); return 1; } //TEST(IndexFstBuilder, IndexFstInput) { // //}