diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 495c4d44770f03e5eb250b8510024a6072e2b29c..6b7d8a40825dd0d122c5ce5b3b9520595d07978d 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -527,9 +527,12 @@ static int tfileReaderLoadFst(TFileReader* reader) { if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; - int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); - indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d ", nread, reader->header.fstOffset, ctx->file.buf, - ctx->file.size); + + int64_t ts = taosGetTimestampUs(); + int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); + int64_t cost = taosGetTimestampUs() - ts; + indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread < FST_MAX_SIZE); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 70671a5f3efffc530b1874b03fc2ddfb96196c50..a2c0046f9a3db12d0c2dbe464c2eb961e04f1bc4 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,4 +1,5 @@ +#include #include #include #include @@ -12,7 +13,6 @@ #include "index_tfile.h" #include "tskiplist.h" #include "tutil.h" - void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; @@ -293,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -306,13 +306,41 @@ void validateTFile(char* arg) { } tfCleanup(); } + +void iterTFileReader(char* path, char* ver) { + tfInit(); + + int version = atoi(ver); + TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); + Iterate* iter = tfileIteratorCreate(reader); + bool tn = iter ? iter->next(iter) : false; + int count = 0; + int termCount = 0; + while (tn == true) { + count++; + IterateValue* cv = iter->getValue(iter); + termCount += (int)taosArrayGetSize(cv->val); + printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val)); + tn = iter->next(iter); + } + printf("total size: %d\n term count: %d\n", count, termCount); + + tfileIteratorDestroy(iter); + tfCleanup(); +} + int main(int argc, char* argv[]) { // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } + if (argc > 2) { + // opt + iterTFileReader(argv[1], argv[2]); + } // checkFstCheckIterator(); // checkFstLongTerm(); // checkFstPrefixSearch(); - checkMillonWriteAndReadOfFst(); + // checkMillonWriteAndReadOfFst(); + return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5438f88b7688e8c5f6c0c6251400ad73b4296f5f..4f3330b7b3f0b14fb77f9e79f3432863dedc1ae1 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -665,14 +665,19 @@ class IndexObj { size_t numOfTable = 100 * 10000) { std::string tColVal = colVal; size_t colValSize = tColVal.size(); + int skip = 100; + numOfTable /= skip; for (int i = 0; i < numOfTable; i++) { - tColVal[i % colValSize] = 'a' + i % 26; + for (int k = 0; k < 10 && k < colVal.size(); k++) { + // opt + tColVal[rand() % colValSize] = 'a' + k % 26; + } SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); - for (size_t i = 0; i < 10; i++) { - int ret = Put(terms, i); + for (size_t j = 0; j < skip; j++) { + int ret = Put(terms, j); assert(ret == 0); } indexMultiTermDestroy(terms); @@ -939,10 +944,11 @@ TEST_F(IndexEnv2, testIndex_read_performance) { TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/multi_tag"; if (index->Init(path) != 0) {} - index->WriteMultiMillonData("tag1", "Hello", 100 * 10000); - index->WriteMultiMillonData("tag2", "Test", 100 * 10000); - index->WriteMultiMillonData("tag3", "Test", 100 * 10000); - index->WriteMultiMillonData("tag4", "Test", 100 * 10000); + int64_t st = taosGetTimestampUs(); + int32_t num = 1000 * 10000; + index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num); + std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl; + // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); } TEST_F(IndexEnv2, testLongComVal) { std::string path = "/tmp/long_colVal";