add tfile analysis tool

source/libs/index/src/index_tfile.c

@@ -527,9 +527,12 @@ static int tfileReaderLoadFst(TFileReader* reader) {
   if (buf == NULL) { return -1; }
 
   WriterCtx* ctx = reader->ctx;
-  int32_t    nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
-  indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d ", nread, reader->header.fstOffset, ctx->file.buf,
-            ctx->file.size);
+
+  int64_t ts = taosGetTimestampUs();
+  int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
+  int64_t cost = taosGetTimestampUs() - ts;
+  indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
+            reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost);
   // we assuse fst size less than FST_MAX_SIZE
   assert(nread > 0 && nread < FST_MAX_SIZE);
 

source/libs/index/test/fstTest.cc

 
+#include <algorithm>
 #include <iostream>
 #include <string>
 #include <thread>
@@ -12,7 +13,6 @@
 #include "index_tfile.h"
 #include "tskiplist.h"
 #include "tutil.h"
-
 void* callback(void* s) { return s; }
 
 static std::string fileName = "/tmp/tindex.tindex";
@@ -293,7 +293,7 @@ void validateTFile(char* arg) {
 
   std::thread threads[NUM_OF_THREAD];
   // std::vector<std::thread> threads;
-  TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1");
+  TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1");
 
   for (int i = 0; i < NUM_OF_THREAD; i++) {
     threads[i] = std::thread(fst_get, reader->fst);
@@ -306,13 +306,41 @@ void validateTFile(char* arg) {
   }
   tfCleanup();
 }
+
+void iterTFileReader(char* path, char* ver) {
+  tfInit();
+
+  int          version = atoi(ver);
+  TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1");
+  Iterate*     iter = tfileIteratorCreate(reader);
+  bool         tn = iter ? iter->next(iter) : false;
+  int          count = 0;
+  int          termCount = 0;
+  while (tn == true) {
+    count++;
+    IterateValue* cv = iter->getValue(iter);
+    termCount += (int)taosArrayGetSize(cv->val);
+    printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val));
+    tn = iter->next(iter);
+  }
+  printf("total size: %d\n term count: %d\n", count, termCount);
+
+  tfileIteratorDestroy(iter);
+  tfCleanup();
+}
+
 int main(int argc, char* argv[]) {
   // tool to check all kind of fst test
   // if (argc > 1) { validateTFile(argv[1]); }
+  if (argc > 2) {
+    // opt
+    iterTFileReader(argv[1], argv[2]);
+  }
   // checkFstCheckIterator();
   // checkFstLongTerm();
   // checkFstPrefixSearch();
 
-  checkMillonWriteAndReadOfFst();
+  // checkMillonWriteAndReadOfFst();
+
   return 1;
 }

source/libs/index/test/indexTests.cc

@@ -665,14 +665,19 @@ class IndexObj {
                            size_t numOfTable = 100 * 10000) {
     std::string tColVal = colVal;
     size_t      colValSize = tColVal.size();
+    int         skip = 100;
+    numOfTable /= skip;
     for (int i = 0; i < numOfTable; i++) {
-      tColVal[i % colValSize] = 'a' + i % 26;
+      for (int k = 0; k < 10 && k < colVal.size(); k++) {
+        // opt
+        tColVal[rand() % colValSize] = 'a' + k % 26;
+      }
       SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      for (size_t i = 0; i < 10; i++) {
-        int ret = Put(terms, i);
+      for (size_t j = 0; j < skip; j++) {
+        int ret = Put(terms, j);
         assert(ret == 0);
       }
       indexMultiTermDestroy(terms);
@@ -939,10 +944,11 @@ TEST_F(IndexEnv2, testIndex_read_performance) {
 TEST_F(IndexEnv2, testIndexMultiTag) {
   std::string path = "/tmp/multi_tag";
   if (index->Init(path) != 0) {}
-  index->WriteMultiMillonData("tag1", "Hello", 100 * 10000);
-  index->WriteMultiMillonData("tag2", "Test", 100 * 10000);
-  index->WriteMultiMillonData("tag3", "Test", 100 * 10000);
-  index->WriteMultiMillonData("tag4", "Test", 100 * 10000);
+  int64_t st = taosGetTimestampUs();
+  int32_t num = 1000 * 10000;
+  index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num);
+  std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl;
+  // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000);
 }
 TEST_F(IndexEnv2, testLongComVal) {
   std::string path = "/tmp/long_colVal";