diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index b19b887e9f86a482634b8c0877c1e4c079ac9f44..416b10bd14c63ebea97b40d80175f033e0e3d7f8 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -42,7 +42,12 @@ typedef struct TFileHeader { #define TFILE_HEADER_SIZE (sizeof(TFileHeader)) #define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t)) -//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) + +typedef struct TFileValue { + char* colVal; // null terminated + SArray* tableId; + int32_t offset; +} TFileValue; typedef struct TFileCacheKey { uint64_t suid; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e88cecfd623c86d39ae28c9906c976f60c27927f..f0546afaf591f0f815cb4802f70f73e85ae010c7 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -41,7 +41,7 @@ static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); -static int indexMergeCacheIntoTindex(SIndex* sIdx); +static int indexFlushCacheToTindex(SIndex* sIdx); static void indexInterResultsDestroy(SArray* results); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); @@ -49,9 +49,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -131,9 +129,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { int32_t colId = fi->colId; int32_t version = index->cVersion; int ret = indexCachePut(index->cache, p, colId, version, uid); - if (ret != 0) { - return ret; - } + if (ret != 0) { return ret; } } #endif @@ -221,9 +217,7 @@ void indexOptsDestroy(SIndexOpts* opts){ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { - return NULL; - } + if (p == NULL) { return NULL; } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -250,9 +244,7 @@ SIndexTerm* indexTermCreate(int64_t suid, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { - return NULL; - } + if (t == NULL) { return NULL; } t->suid = suid; t->operType = oper; @@ -332,9 +324,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { - return; - } + if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -363,10 +353,10 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType } return 0; } -static int indexMergeCacheIntoTindex(SIndex* sIdx) { - if (sIdx == NULL) { - return -1; - } +static int indexFlushCacheToTindex(SIndex* sIdx) { + if (sIdx == NULL) { return -1; } + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + return 0; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 6170642707434b4daaf22bc14c9f8058f841e611..bb6a5e048af60a4822716395baa094a1c9cec600 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -151,8 +151,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t EIndexQueryType qtype = query->qType; int32_t keyLen = CACHE_KEY_LEN(term); - - char* buf = calloc(1, keyLen); + char* buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { // } else if (qtype == QUERY_PREFIX) { diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 962973bb5cd6fa47c0635ac5a4e5d8fa708dbfa5..c8fbdd7690bc3ec0243a01dab89fb53d959854f2 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -69,9 +69,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(path); } else { - ctx->file.fd = tfOpenReadWrite(tmpFile); + ctx->file.fd = tfOpenReadWrite(path); } if (ctx->file.fd < 0) { indexError("open file error %d", errno); @@ -93,6 +93,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int END: if (ctx->type == TMemory) { free(ctx->mem.buf); } free(ctx); + return NULL; } void writerCtxDestroy(WriterCtx* ctx) { if (ctx->type == TMemory) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index ecacdcd13c463ef4ffc689534d13737d69f15e1b..e02a3e03276e659e7decd0ff17d94b7a8e4d9162 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -25,12 +25,7 @@ #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) -typedef struct TFileValue { - char* colVal; // null terminated - SArray* tableId; - int32_t offset; -} TFileValue; - +static int tfileStrCompare(const void* a, const void* b); static int tfileValueCompare(const void* a, const void* b, const void* param); static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); @@ -49,6 +44,7 @@ static int tfileRmExpireFile(SArray* result); static void tfileDestroyFileName(void* elem); static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); +static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); TFileCache* tfileCacheCreate(const char* path) { @@ -209,16 +205,28 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { tw->ctx = ctx; tw->header = *header; tfileWriteHeader(tw); + tw->fb = fstBuilderCreate(ctx, 0); + if (tw->fb == NULL) { + tfileWriterDestroy(tw); + return NULL; + } return tw; } int tfileWriterPut(TFileWriter* tw, void* data) { // sort by coltype and write to tindex - __compar_fn_t fn = getComparFunc(tw->header.colType, 0); + __compar_fn_t fn; + + int8_t colType = tw->header.colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + fn = tfileStrCompare; + } else { + fn = getComparFunc(colType, 0); + } taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); int32_t bufLimit = 4096, offset = 0; - char* buf = calloc(1, sizeof(bufLimit)); + char* buf = calloc(1, sizeof(char) * bufLimit); char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; @@ -267,6 +275,9 @@ int tfileWriterPut(TFileWriter* tw, void* data) { // } } + fstBuilderFinish(tw->fb); + fstBuilderDestroy(tw->fb); + tw->fb = NULL; return 0; } void tfileWriterDestroy(TFileWriter* tw) { @@ -305,6 +316,12 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { return 0; } +static int tfileStrCompare(const void* a, const void* b) { + int ret = strcmp((char*)a, (char*)b); + if (ret == 0) { return ret; } + return ret < 0 ? -1 : 1; +} + static int tfileValueCompare(const void* a, const void* b, const void* param) { __compar_fn_t fn = *(__compar_fn_t*)param; @@ -445,6 +462,10 @@ static int tfileCompare(const void* a, const void* b) { return strncmp(aName, bName, aLen > bLen ? aLen : bLen); } // tfile name suid-colId-version.tindex +static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) { + sprintf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version); + return; +} static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) { if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { // read suid & colid & version success diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 85d76bb716a887bf7e7b07d54afde9ae67f431de..f8dae787b898706b5b194858337613e48df0b93f 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -2,8 +2,7 @@ * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -13,141 +12,141 @@ * along with this program. If not, see . */ #include -#include #include +#include #include "index.h" -#include "tutil.h" #include "indexInt.h" #include "index_fst.h" -#include "index_fst_util.h" #include "index_fst_counting_writer.h" - +#include "index_fst_util.h" +#include "index_tfile.h" +#include "tutil.h" class FstWriter { - public: - FstWriter() { - _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0); - _b = fstBuilderCreate(NULL, 0); - } - bool Put(const std::string &key, uint64_t val) { - FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); - bool ok = fstBuilderInsert(_b, skey, val); - fstSliceDestroy(&skey); - return ok; - } - ~FstWriter() { - fstBuilderFinish(_b); - fstBuilderDestroy(_b); - - writerCtxDestroy(_wc); - } - private: - FstBuilder *_b; - WriterCtx *_wc; + public: + FstWriter() { + _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024); + _b = fstBuilderCreate(NULL, 0); + } + bool Put(const std::string& key, uint64_t val) { + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + + writerCtxDestroy(_wc); + } + + private: + FstBuilder* _b; + WriterCtx* _wc; }; class FstReadMemory { - public: - FstReadMemory(size_t size) { - _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 0); - _w = fstCountingWriterCreate(_wc); - _size = size; - memset((void *)&_s, 0, sizeof(_s)); - } - bool init() { - char *buf = (char *)calloc(1, sizeof(char) * _size); - int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size); - if (nRead <= 0) { return false; } - _size = nRead; - _s = fstSliceCreate((uint8_t *)buf, _size); - _fst = fstCreate(&_s); - free(buf); - return _fst != NULL; - } - bool Get(const std::string &key, uint64_t *val) { - FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); - bool ok = fstGet(_fst, &skey, val); - fstSliceDestroy(&skey); - return ok; - } - bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) { - int64_t s = taosGetTimestampUs(); - bool ok = this->Get(key, val); - int64_t e = taosGetTimestampUs(); - *elapse = e - s; - return ok; - } - // add later - bool Search(AutomationCtx *ctx, std::vector &result) { - FstStreamBuilder *sb = fstSearch(_fst, ctx); - StreamWithState *st = streamBuilderIntoStream(sb); - StreamWithStateResult *rt = NULL; - - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - result.push_back((uint64_t)(rt->out.out)); - } - return true; - } - bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector &result) { - int64_t s = taosGetTimestampUs(); - bool ok = this->Search(ctx, result); - int64_t e = taosGetTimestampUs(); - return ok; - } - - ~FstReadMemory() { + public: + FstReadMemory(size_t size) { + _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024); + _w = fstCountingWriterCreate(_wc); + _size = size; + memset((void*)&_s, 0, sizeof(_s)); + } + bool init() { + char* buf = (char*)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); + if (nRead <= 0) { return false; } + _size = nRead; + _s = fstSliceCreate((uint8_t*)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string& key, uint64_t* val) { + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(AutomationCtx* ctx, std::vector& result) { + FstStreamBuilder* sb = fstSearch(_fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + result.push_back((uint64_t)(rt->out.out)); + } + return true; + } + bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } + + ~FstReadMemory() { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc); - } - - private: - FstCountingWriter *_w; - Fst *_fst; - FstSlice _s; - WriterCtx *_wc; - size_t _size; - -}; - -//TEST(IndexTest, index_create_test) { + } + + private: + FstCountingWriter* _w; + Fst* _fst; + FstSlice _s; + WriterCtx* _wc; + size_t _size; +}; + +// TEST(IndexTest, index_create_test) { // SIndexOpts *opts = indexOptsCreate(); // SIndex *index = indexOpen(opts, "./test"); // if (index == NULL) { -// std::cout << "index open failed" << std::endl; +// std::cout << "index open failed" << std::endl; // } // -// -// // write +// +// // write // for (int i = 0; i < 100000; i++) { // SIndexMultiTerm* terms = indexMultiTermCreate(); -// std::string val = "field"; +// std::string val = "field"; // // indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); // -// val.append(std::to_string(i)); +// val.append(std::to_string(i)); // indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); // // val.insert(0, std::to_string(i)); // indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); // -// val.append("const"); +// val.append("const"); // indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); // -// +// // indexPut(index, terms, i); // indexMultiTermDestroy(terms); -// } -// +// } +// // // // query -// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); -// +// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); +// // indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); // indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); // -// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); +// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); // indexSearch(index, multiQuery, result); // // std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; @@ -155,25 +154,24 @@ class FstReadMemory { // int *v = (int *)taosArrayGet(result, i); // std::cout << "value --->" << *v << std::endl; // } -// // add more test case +// // add more test case // indexMultiTermQueryDestroy(multiQuery); // -// indexOptsDestroy(opts); -// indexClose(index); +// indexOptsDestroy(opts); +// indexClose(index); // // //} - #define L 100 #define M 100 #define N 100 -int Performance_fstWriteRecords(FstWriter *b) { - std::string str("aa"); +int Performance_fstWriteRecords(FstWriter* b) { + std::string str("aa"); for (int i = 0; i < L; i++) { str[0] = 'a' + i; - str.resize(2); - for(int j = 0; j < M; j++) { + str.resize(2); + for (int j = 0; j < M; j++) { str[1] = 'a' + j; str.resize(2); for (int k = 0; k < N; k++) { @@ -181,87 +179,86 @@ int Performance_fstWriteRecords(FstWriter *b) { b->Put(str, k); printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str()); } - } + } } return L * M * N; } -void Performance_fstReadRecords(FstReadMemory *m) { +void Performance_fstReadRecords(FstReadMemory* m) { std::string str("aa"); for (int i = 0; i < M; i++) { str[0] = 'a' + i; - str.resize(2); - for(int j = 0; j < N; j++) { + str.resize(2); + for (int j = 0; j < N; j++) { str[1] = 'a' + j; str.resize(2); for (int k = 0; k < L; k++) { str.push_back('a'); - uint64_t val, cost; + uint64_t val, cost; if (m->GetWithTimeCostUs(str, &val, &cost)) { - printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost); + printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost); } else { printf("failed to get key: %s\n", str.c_str()); } } - } + } } } void checkFstPerf() { - FstWriter *fw = new FstWriter; - int64_t s = taosGetTimestampUs(); + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); - int num = Performance_fstWriteRecords(fw); + int num = Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); - printf("write %d record cost %" PRId64"us\n", num, e - s); + printf("write %d record cost %" PRId64 "us\n", num, e - s); delete fw; - FstReadMemory *m = new FstReadMemory(1024 * 64); - if (m->init()) { - printf("success to init fst read"); - } - Performance_fstReadRecords(m); + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init()) { printf("success to init fst read"); } + Performance_fstReadRecords(m); delete m; -} +} void checkFstPrefixSearch() { - FstWriter *fw = new FstWriter; - int64_t s = taosGetTimestampUs(); - int count = 2; + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; std::string key("ab"); - + for (int i = 0; i < count; i++) { - key[1] = key[1] + i; - fw->Put(key, i); + key[1] = key[1] + i; + fw->Put(key, i); } int64_t e = taosGetTimestampUs(); - + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; delete fw; - FstReadMemory *m = new FstReadMemory(1024 * 64); + FstReadMemory* m = new FstReadMemory(1024 * 64); if (m->init() == false) { - std::cout << "init readMemory failed" << std::endl; + std::cout << "init readMemory failed" << std::endl; delete m; return; } - - // prefix search + + // prefix search std::vector result; - AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX); - m->Search(ctx, result); - assert(result.size() == count); + + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX); + m->Search(ctx, result); + assert(result.size() == count); for (int i = 0; i < result.size(); i++) { - assert(result[i] == i); // check result + assert(result[i] == i); // check result } free(ctx); delete m; -} +} void validateFst() { - int val = 100; - int count = 100; - FstWriter *fw = new FstWriter; - // write + int val = 100; + int count = 100; + FstWriter* fw = new FstWriter; + // write { std::string key("ab"); for (int i = 0; i < count; i++) { @@ -272,99 +269,161 @@ void validateFst() { delete fw; // read - FstReadMemory *m = new FstReadMemory(1024 * 64); - if (m->init() == false) { - std::cout << "init readMemory failed" << std::endl; + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; delete m; return; } { - std::string key("ab"); - uint64_t out; - if (m->Get(key, &out)) { - printf("success to get (%s, %" PRId64")\n", key.c_str(), out); - } else { - printf("failed to get(%s)\n", key.c_str()); - } - for (int i = 0; i < count; i++) { - key.push_back('a' + i); - if (m->Get(key, &out) ) { - assert(val - i == out); - printf("success to get (%s, %" PRId64")\n", key.c_str(), out); - } else { - printf("failed to get(%s)\n", key.c_str()); + std::string key("ab"); + uint64_t out; + if (m->Get(key, &out)) { + printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); + } + for (int i = 0; i < count; i++) { + key.push_back('a' + i); + if (m->Get(key, &out)) { + assert(val - i == out); + printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); + } } - } - } + } delete m; -} +} class IndexEnv : public ::testing::Test { - protected: - virtual void SetUp() { - taosRemoveDir(path); - opts = indexOptsCreate(); - int ret = indexOpen(opts, path, &index); - assert(ret == 0); - } - virtual void TearDown() { - indexClose(index); - indexOptsDestroy(opts); - } - - const char *path = "/tmp/tindex"; - SIndexOpts *opts; - SIndex *index; + protected: + virtual void SetUp() { + taosRemoveDir(path); + opts = indexOptsCreate(); + int ret = indexOpen(opts, path, &index); + assert(ret == 0); + } + virtual void TearDown() { + indexClose(index); + indexOptsDestroy(opts); + } + + const char* path = "/tmp/tindex"; + SIndexOpts* opts; + SIndex* index; }; -TEST_F(IndexEnv, testPut) { - - // single index column - { - - std::string colName("tag1"), colVal("Hello world"); - SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexMultiTerm *terms = indexMultiTermCreate(); - indexMultiTermAdd(terms, term); - - for (size_t i = 0; i < 100; i++) { - int tableId = i; - int ret = indexPut(index, terms, tableId); - assert(ret == 0); - } - indexMultiTermDestroy(terms); - } - // multi index column - { - - SIndexMultiTerm *terms = indexMultiTermCreate(); - { - std::string colName("tag1"), colVal("Hello world"); - SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - indexMultiTermAdd(terms, term); - } - { - std::string colName("tag2"), colVal("Hello world"); - SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - indexMultiTermAdd(terms, term); - } - - for (int i = 0; i < 100; i++) { - int tableId = i; - int ret = indexPut(index, terms, tableId); - assert(ret == 0); - } - indexMultiTermDestroy(terms); - } - // -} - -TEST_F(IndexEnv, testDel) { - +// TEST_F(IndexEnv, testPut) { +// // single index column +// { +// std::string colName("tag1"), colVal("Hello world"); +// SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), +// colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); +// +// for (size_t i = 0; i < 100; i++) { +// int tableId = i; +// int ret = indexPut(index, terms, tableId); +// assert(ret == 0); +// } +// indexMultiTermDestroy(terms); +// } +// // multi index column +// { +// SIndexMultiTerm* terms = indexMultiTermCreate(); +// { +// std::string colName("tag1"), colVal("Hello world"); +// SIndexTerm* term = +// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); +// indexMultiTermAdd(terms, term); +// } +// { +// std::string colName("tag2"), colVal("Hello world"); +// SIndexTerm* term = +// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); +// indexMultiTermAdd(terms, term); +// } +// +// for (int i = 0; i < 100; i++) { +// int tableId = i; +// int ret = indexPut(index, terms, tableId); +// assert(ret == 0); +// } +// indexMultiTermDestroy(terms); +// } +// // +//} + +class IndexTFileEnv : public ::testing::Test { + protected: + virtual void SetUp() { + taosRemoveDir(dir); + taosMkDir(dir); + tfInit(); + std::string colName("voltage"); + header.suid = 1; + header.version = 1; + memcpy(header.colName, colName.c_str(), colName.size()); + header.colType = TSDB_DATA_TYPE_BINARY; + + std::string path(dir); + int colId = 2; + char buf[64] = {0}; + sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version); + path.append("/").append(buf); + + ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024); + + twrite = tfileWriterCreate(ctx, &header); + } + + virtual void TearDown() { + // indexClose(index); + // indexeptsDestroy(opts); + tfCleanup(); + tfileWriterDestroy(twrite); + } + const char* dir = "/tmp/tindex"; + WriterCtx* ctx = NULL; + TFileHeader header; + TFileWriter* twrite = NULL; +}; + +static TFileValue* genTFileValue(const char* val) { + TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); + int32_t vlen = strlen(val) + 1; + tv->colVal = (char*)calloc(1, vlen); + memcpy(tv->colVal, val, vlen); + + tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); + for (size_t i = 0; i < 10; i++) { + uint64_t v = i; + taosArrayPush(tv->tableId, &v); + } + return tv; +} +static void destroyTFileValue(void* val) { + TFileValue* tv = (TFileValue*)val; + free(tv->colVal); + taosArrayDestroy(tv->tableId); + free(tv); } +TEST_F(IndexTFileEnv, test_tfile_write) { + TFileValue* v1 = genTFileValue("c"); + TFileValue* v2 = genTFileValue("a"); + SArray* data = (SArray*)taosArrayInit(4, sizeof(void*)); + taosArrayPush(data, &v1); + taosArrayPush(data, &v2); + tfileWriterPut(twrite, data); + // tfileWriterDestroy(twrite); + for (size_t i = 0; i < taosArrayGetSize(data); i++) { + destroyTFileValue(taosArrayGetP(data, i)); + } + taosArrayDestroy(data); +}