index_tfile.c 21.1 KB
Newer Older
dengyihao's avatar
dengyihao 已提交
1 2
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
dengyihao's avatar
dengyihao 已提交
3
p *
dengyihao's avatar
dengyihao 已提交
4 5 6 7 8 9 10 11 12 13 14 15
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

dengyihao's avatar
dengyihao 已提交
16 17
//#include <sys/types.h>
//#include <dirent.h>
dengyihao's avatar
dengyihao 已提交
18
#include "index_tfile.h"
19
#include "index.h"
dengyihao's avatar
dengyihao 已提交
20
#include "index_fst.h"
21
#include "index_fst_counting_writer.h"
dengyihao's avatar
dengyihao 已提交
22
#include "index_util.h"
dengyihao's avatar
dengyihao 已提交
23
#include "taosdef.h"
dengyihao's avatar
dengyihao 已提交
24
#include "tcompare.h"
dengyihao's avatar
dengyihao 已提交
25

dengyihao's avatar
dengyihao 已提交
26 27 28 29 30 31 32
typedef struct TFileFstIter {
  FstStreamBuilder* fb;
  StreamWithState*  st;
  AutomationCtx*    ctx;
  TFileReader*      rdr;
} TFileFstIter;

dengyihao's avatar
dengyihao 已提交
33 34
#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))

dengyihao's avatar
dengyihao 已提交
35
static int  tfileUidCompare(const void* a, const void* b);
dengyihao's avatar
dengyihao 已提交
36
static int  tfileStrCompare(const void* a, const void* b);
dengyihao's avatar
dengyihao 已提交
37 38
static int  tfileValueCompare(const void* a, const void* b, const void* param);
static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds);
dengyihao's avatar
dengyihao 已提交
39

dengyihao's avatar
dengyihao 已提交
40
static int tfileWriteHeader(TFileWriter* writer);
dengyihao's avatar
dengyihao 已提交
41
static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset);
dengyihao's avatar
dengyihao 已提交
42
static int tfileWriteData(TFileWriter* write, TFileValue* tval);
dengyihao's avatar
dengyihao 已提交
43

dengyihao's avatar
dengyihao 已提交
44 45 46
static int tfileReaderLoadHeader(TFileReader* reader);
static int tfileReaderLoadFst(TFileReader* reader);
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result);
dengyihao's avatar
dengyihao 已提交
47

dengyihao's avatar
dengyihao 已提交
48 49 50 51 52 53 54
static SArray* tfileGetFileList(const char* path);
static int     tfileRmExpireFile(SArray* result);
static void    tfileDestroyFileName(void* elem);
static int     tfileCompare(const void* a, const void* b);
static int     tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version);
static void    tfileGenFileName(char* filename, uint64_t suid, const char* col, int version);
static void    tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version);
dengyihao's avatar
dengyihao 已提交
55

dengyihao's avatar
dengyihao 已提交
56 57
TFileCache* tfileCacheCreate(const char* path) {
  TFileCache* tcache = calloc(1, sizeof(TFileCache));
dengyihao's avatar
dengyihao 已提交
58
  if (tcache == NULL) { return NULL; }
59 60 61 62

  tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
  tcache->capacity = 64;

dengyihao's avatar
dengyihao 已提交
63
  SArray* files = tfileGetFileList(path);
dengyihao's avatar
dengyihao 已提交
64

dengyihao's avatar
dengyihao 已提交
65 66
  uint64_t suid;
  int32_t  colId, version;
dengyihao's avatar
dengyihao 已提交
67
  for (size_t i = 0; i < taosArrayGetSize(files); i++) {
dengyihao's avatar
dengyihao 已提交
68
    char* file = taosArrayGetP(files, i);
dengyihao's avatar
dengyihao 已提交
69 70 71

    // refactor later, use colname and version info
    char colName[256] = {0};
dengyihao's avatar
dengyihao 已提交
72
    if (0 != tfileParseFileName(file, &suid, colName, (int*)&version)) {
dengyihao's avatar
dengyihao 已提交
73
      indexInfo("try parse invalid file:  %s, skip it", file);
74 75
      continue;
    }
dengyihao's avatar
dengyihao 已提交
76

dengyihao's avatar
dengyihao 已提交
77 78
    char fullName[256] = {0};
    sprintf(fullName, "%s/%s", path, file);
dengyihao's avatar
dengyihao 已提交
79

dengyihao's avatar
dengyihao 已提交
80
    WriterCtx* wc = writerCtxCreate(TFile, fullName, true, 1024 * 1024 * 64);
81
    if (wc == NULL) {
dengyihao's avatar
dengyihao 已提交
82
      indexError("failed to open index:%s", file);
83
      goto End;
dengyihao's avatar
dengyihao 已提交
84
    }
dengyihao's avatar
dengyihao 已提交
85

dengyihao's avatar
dengyihao 已提交
86 87 88 89 90 91 92 93 94 95 96
    char         buf[128] = {0};
    TFileReader* reader = tfileReaderCreate(wc);
    TFileHeader* header = &reader->header;
    ICacheKey    key = {.suid = header->suid,
                     .colName = header->colName,
                     .nColName = strlen(header->colName),
                     .colType = header->colType};

    int32_t sz = indexSerialCacheKey(&key, buf);
    assert(sz < sizeof(buf));
    taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
dengyihao's avatar
dengyihao 已提交
97
    tfileReaderRef(reader);
dengyihao's avatar
dengyihao 已提交
98
  }
dengyihao's avatar
dengyihao 已提交
99
  taosArrayDestroyEx(files, tfileDestroyFileName);
dengyihao's avatar
dengyihao 已提交
100
  return tcache;
dengyihao's avatar
dengyihao 已提交
101
End:
102
  tfileCacheDestroy(tcache);
dengyihao's avatar
dengyihao 已提交
103
  taosArrayDestroyEx(files, tfileDestroyFileName);
104
  return NULL;
dengyihao's avatar
dengyihao 已提交
105
}
dengyihao's avatar
dengyihao 已提交
106
void tfileCacheDestroy(TFileCache* tcache) {
dengyihao's avatar
dengyihao 已提交
107
  if (tcache == NULL) { return; }
108 109

  // free table cache
dengyihao's avatar
dengyihao 已提交
110
  TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
111
  while (reader) {
dengyihao's avatar
dengyihao 已提交
112
    TFileReader* p = *reader;
113 114
    indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
              p->header.colType);
dengyihao's avatar
dengyihao 已提交
115

dengyihao's avatar
dengyihao 已提交
116
    tfileReaderUnRef(p);
117 118 119 120
    reader = taosHashIterate(tcache->tableCache, reader);
  }
  taosHashCleanup(tcache->tableCache);
  free(tcache);
dengyihao's avatar
dengyihao 已提交
121 122
}

dengyihao's avatar
dengyihao 已提交
123 124 125 126 127
TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
  char    buf[128] = {0};
  int32_t sz = indexSerialCacheKey(key, buf);
  assert(sz < sizeof(buf));
  TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
dengyihao's avatar
dengyihao 已提交
128
  if (reader == NULL) { return NULL; }
129
  tfileReaderRef(*reader);
dengyihao's avatar
dengyihao 已提交
130

131
  return *reader;
dengyihao's avatar
dengyihao 已提交
132
}
dengyihao's avatar
dengyihao 已提交
133 134 135
void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
  char    buf[128] = {0};
  int32_t sz = indexSerialCacheKey(key, buf);
dengyihao's avatar
dengyihao 已提交
136
  // remove last version index reader
dengyihao's avatar
dengyihao 已提交
137
  TFileReader** p = taosHashGet(tcache->tableCache, buf, sz);
dengyihao's avatar
dengyihao 已提交
138
  if (p != NULL) {
dengyihao's avatar
dengyihao 已提交
139
    TFileReader* oldReader = *p;
dengyihao's avatar
dengyihao 已提交
140
    taosHashRemove(tcache->tableCache, buf, sz);
dengyihao's avatar
dengyihao 已提交
141
    oldReader->remove = true;
dengyihao's avatar
dengyihao 已提交
142
    tfileReaderUnRef(oldReader);
dengyihao's avatar
dengyihao 已提交
143
  }
dengyihao's avatar
dengyihao 已提交
144

dengyihao's avatar
dengyihao 已提交
145
  taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
dengyihao's avatar
dengyihao 已提交
146
  tfileReaderRef(reader);
dengyihao's avatar
dengyihao 已提交
147
  return;
148
}
dengyihao's avatar
dengyihao 已提交
149 150
TFileReader* tfileReaderCreate(WriterCtx* ctx) {
  TFileReader* reader = calloc(1, sizeof(TFileReader));
dengyihao's avatar
dengyihao 已提交
151
  if (reader == NULL) { return NULL; }
dengyihao's avatar
dengyihao 已提交
152

153
  // T_REF_INC(reader);
dengyihao's avatar
dengyihao 已提交
154
  reader->ctx = ctx;
dengyihao's avatar
dengyihao 已提交
155 156
  if (0 != tfileReaderLoadHeader(reader)) {
    tfileReaderDestroy(reader);
157 158
    indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid,
               reader->header.colName);
dengyihao's avatar
dengyihao 已提交
159 160 161 162 163 164 165 166 167
    return NULL;
  }

  if (0 != tfileReaderLoadFst(reader)) {
    tfileReaderDestroy(reader);
    indexError("failed to load index fst, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName);
    return NULL;
  }

168
  return reader;
dengyihao's avatar
dengyihao 已提交
169
}
dengyihao's avatar
dengyihao 已提交
170
void tfileReaderDestroy(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
171
  if (reader == NULL) { return; }
172
  // T_REF_INC(reader);
dengyihao's avatar
dengyihao 已提交
173
  fstDestroy(reader->fst);
dengyihao's avatar
dengyihao 已提交
174
  writerCtxDestroy(reader->ctx, reader->remove);
dengyihao's avatar
dengyihao 已提交
175 176 177
  free(reader);
}

dengyihao's avatar
dengyihao 已提交
178
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) {
dengyihao's avatar
dengyihao 已提交
179 180
  SIndexTerm*     term = query->term;
  EIndexQueryType qtype = query->qType;
dengyihao's avatar
dengyihao 已提交
181

dengyihao's avatar
dengyihao 已提交
182
  int ret = -1;
dengyihao's avatar
dengyihao 已提交
183
  // refactor to callback later
dengyihao's avatar
dengyihao 已提交
184
  if (qtype == QUERY_TERM) {
dengyihao's avatar
dengyihao 已提交
185 186 187
    uint64_t offset;
    FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
    if (fstGet(reader->fst, &key, &offset)) {
188 189
      indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName,
                term->colVal);
dengyihao's avatar
dengyihao 已提交
190
      ret = tfileReaderLoadTableIds(reader, offset, result);
dengyihao's avatar
dengyihao 已提交
191
    } else {
192 193
      indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
                term->colVal);
dengyihao's avatar
dengyihao 已提交
194
    }
dengyihao's avatar
dengyihao 已提交
195 196
    fstSliceDestroy(&key);
  } else if (qtype == QUERY_PREFIX) {
dengyihao's avatar
dengyihao 已提交
197
    // handle later
dengyihao's avatar
dengyihao 已提交
198
    //
dengyihao's avatar
dengyihao 已提交
199 200
  } else {
    // handle later
dengyihao's avatar
dengyihao 已提交
201
  }
dengyihao's avatar
dengyihao 已提交
202
  tfileReaderUnRef(reader);
dengyihao's avatar
dengyihao 已提交
203
  return ret;
dengyihao's avatar
dengyihao 已提交
204 205
}

dengyihao's avatar
dengyihao 已提交
206 207
TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) {
  char fullname[256] = {0};
dengyihao's avatar
dengyihao 已提交
208
  tfileGenFileFullName(fullname, path, suid, colName, version);
dengyihao's avatar
dengyihao 已提交
209
  // indexInfo("open write file name %s", fullname);
dengyihao's avatar
dengyihao 已提交
210
  WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
dengyihao's avatar
dengyihao 已提交
211
  if (wcx == NULL) { return NULL; }
dengyihao's avatar
dengyihao 已提交
212 213 214 215 216 217 218 219 220

  TFileHeader tfh = {0};
  tfh.suid = suid;
  tfh.version = version;
  memcpy(tfh.colName, colName, strlen(colName));
  tfh.colType = colType;

  return tfileWriterCreate(wcx, &tfh);
}
dengyihao's avatar
dengyihao 已提交
221 222
TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) {
  char fullname[256] = {0};
dengyihao's avatar
dengyihao 已提交
223 224
  tfileGenFileFullName(fullname, path, suid, colName, version);

dengyihao's avatar
dengyihao 已提交
225
  WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
dengyihao's avatar
dengyihao 已提交
226
  // indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
dengyihao's avatar
dengyihao 已提交
227 228
  if (wc == NULL) { return NULL; }

dengyihao's avatar
dengyihao 已提交
229
  TFileReader* reader = tfileReaderCreate(wc);
dengyihao's avatar
dengyihao 已提交
230 231
  return reader;
}
dengyihao's avatar
dengyihao 已提交
232
TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
dengyihao's avatar
dengyihao 已提交
233 234 235 236 237
  // char pathBuf[128] = {0};
  // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
  // TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
  // memcpy(header.colName, );

dengyihao's avatar
dengyihao 已提交
238 239 240 241 242 243 244
  // char buf[TFILE_HADER_PRE_SIZE];
  // int  len = TFILE_HADER_PRE_SIZE;
  // if (len != ctx->write(ctx, buf, len)) {
  //  indexError("index: %" PRIu64 " failed to write header info", header->suid);
  //  return NULL;
  //}
  TFileWriter* tw = calloc(1, sizeof(TFileWriter));
dengyihao's avatar
dengyihao 已提交
245
  if (tw == NULL) {
dengyihao's avatar
dengyihao 已提交
246
    indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid);
dengyihao's avatar
dengyihao 已提交
247 248
    return NULL;
  }
dengyihao's avatar
dengyihao 已提交
249 250
  tw->ctx = ctx;
  tw->header = *header;
dengyihao's avatar
dengyihao 已提交
251
  tfileWriteHeader(tw);
dengyihao's avatar
dengyihao 已提交
252 253
  return tw;
}
dengyihao's avatar
dengyihao 已提交
254

dengyihao's avatar
dengyihao 已提交
255
int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
dengyihao's avatar
dengyihao 已提交
256
  // sort by coltype and write to tindex
dengyihao's avatar
dengyihao 已提交
257 258 259 260 261 262 263 264 265
  if (order == false) {
    __compar_fn_t fn;
    int8_t        colType = tw->header.colType;
    if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
      fn = tfileStrCompare;
    } else {
      fn = getComparFunc(colType, 0);
    }
    taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
dengyihao's avatar
dengyihao 已提交
266
  }
dengyihao's avatar
dengyihao 已提交
267

dengyihao's avatar
dengyihao 已提交
268 269 270
  int32_t bufLimit = 64 * 4096, offset = 0;
  // char*   buf = calloc(1, sizeof(char) * bufLimit);
  // char*   p = buf;
dengyihao's avatar
dengyihao 已提交
271
  int32_t sz = taosArrayGetSize((SArray*)data);
dengyihao's avatar
dengyihao 已提交
272 273 274 275 276
  int32_t fstOffset = tw->offset;

  // ugly code, refactor later
  for (size_t i = 0; i < sz; i++) {
    TFileValue* v = taosArrayGetP((SArray*)data, i);
dengyihao's avatar
dengyihao 已提交
277
    // taosArrayRemoveDuplicate(v->tablId, tfileUidCompare, NULL);
dengyihao's avatar
dengyihao 已提交
278
    int32_t tbsz = taosArrayGetSize(v->tableId);
dengyihao's avatar
dengyihao 已提交
279
    fstOffset += TF_TABLE_TATOAL_SIZE(tbsz);
dengyihao's avatar
dengyihao 已提交
280 281 282
  }
  tfileWriteFstOffset(tw, fstOffset);

dengyihao's avatar
dengyihao 已提交
283 284 285 286 287 288 289
  for (size_t i = 0; i < sz; i++) {
    TFileValue* v = taosArrayGetP((SArray*)data, i);

    int32_t tbsz = taosArrayGetSize(v->tableId);
    // check buf has enough space or not
    int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);

dengyihao's avatar
dengyihao 已提交
290 291 292 293 294 295 296 297 298 299 300 301 302 303
    // if (offset + ttsz >= bufLimit) {
    //  // batch write
    //  indexInfo("offset: %d, ttsz: %d", offset, ttsz);
    //  // std::cout << "offset: " << offset << std::endl;
    //  // std::cout << "ttsz:" << ttsz < < < std::endl;
    //  tw->ctx->write(tw->ctx, buf, offset);
    //  offset = 0;
    //  memset(buf, 0, bufLimit);
    //  p = buf;
    //}
    // if (ttsz >= bufLimit) {
    //}
    char* buf = calloc(1, ttsz * sizeof(char));
    char* p = buf;
dengyihao's avatar
dengyihao 已提交
304
    tfileSerialTableIdsToBuf(p, v->tableId);
dengyihao's avatar
dengyihao 已提交
305 306 307
    tw->ctx->write(tw->ctx, buf, ttsz);
    // offset += ttsz;
    // p = buf + offset;
dengyihao's avatar
dengyihao 已提交
308
    // set up value offset
dengyihao's avatar
dengyihao 已提交
309 310
    v->offset = tw->offset;
    tw->offset += ttsz;
dengyihao's avatar
dengyihao 已提交
311
    free(buf);
dengyihao's avatar
dengyihao 已提交
312
  }
dengyihao's avatar
dengyihao 已提交
313 314 315 316 317
  // if (offset != 0) {
  // write reversed data in buf to tindex
  // tw->ctx->write(tw->ctx, buf, offset);
  //}
  // tfree(buf);
dengyihao's avatar
dengyihao 已提交
318

dengyihao's avatar
dengyihao 已提交
319 320
  tw->fb = fstBuilderCreate(tw->ctx, 0);
  if (tw->fb == NULL) {
dengyihao's avatar
dengyihao 已提交
321
    tfileWriterClose(tw);
dengyihao's avatar
dengyihao 已提交
322 323
    return -1;
  }
dengyihao's avatar
dengyihao 已提交
324 325

  // write data
dengyihao's avatar
dengyihao 已提交
326 327 328
  for (size_t i = 0; i < sz; i++) {
    // TODO, fst batch write later
    TFileValue* v = taosArrayGetP((SArray*)data, i);
dengyihao's avatar
dengyihao 已提交
329 330 331 332
    if (tfileWriteData(tw, v) != 0) {
      indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
                 (int)taosArrayGetSize(v->tableId));
    } else {
dengyihao's avatar
dengyihao 已提交
333 334
      // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
      //          (int)taosArrayGetSize(v->tableId));
dengyihao's avatar
dengyihao 已提交
335 336
    }
  }
dengyihao's avatar
dengyihao 已提交
337 338 339
  fstBuilderFinish(tw->fb);
  fstBuilderDestroy(tw->fb);
  tw->fb = NULL;
dengyihao's avatar
dengyihao 已提交
340 341
  return 0;
}
dengyihao's avatar
dengyihao 已提交
342 343 344
void tfileWriterClose(TFileWriter* tw) {
  if (tw == NULL) { return; }
  writerCtxDestroy(tw->ctx, false);
dengyihao's avatar
dengyihao 已提交
345 346
  free(tw);
}
dengyihao's avatar
dengyihao 已提交
347
void tfileWriterDestroy(TFileWriter* tw) {
dengyihao's avatar
dengyihao 已提交
348 349
  if (tw == NULL) { return; }
  writerCtxDestroy(tw->ctx, false);
dengyihao's avatar
dengyihao 已提交
350 351
  free(tw);
}
dengyihao's avatar
dengyihao 已提交
352

dengyihao's avatar
dengyihao 已提交
353 354
IndexTFile* indexTFileCreate(const char* path) {
  IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
dengyihao's avatar
dengyihao 已提交
355
  if (tfile == NULL) { return NULL; }
356

dengyihao's avatar
dengyihao 已提交
357
  tfile->cache = tfileCacheCreate(path);
dengyihao's avatar
dengyihao 已提交
358 359
  return tfile;
}
360
void indexTFileDestroy(IndexTFile* tfile) {
dengyihao's avatar
dengyihao 已提交
361
  if (tfile == NULL) { return; }
dengyihao's avatar
dengyihao 已提交
362 363 364
  tfileCacheDestroy(tfile->cache);
  free(tfile);
}
dengyihao's avatar
dengyihao 已提交
365

dengyihao's avatar
dengyihao 已提交
366
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
dengyihao's avatar
dengyihao 已提交
367
  int ret = -1;
dengyihao's avatar
dengyihao 已提交
368
  if (tfile == NULL) { return ret; }
dengyihao's avatar
dengyihao 已提交
369
  IndexTFile* pTfile = (IndexTFile*)tfile;
370

dengyihao's avatar
dengyihao 已提交
371 372
  SIndexTerm* term = query->term;
  ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
373
  TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
dengyihao's avatar
dengyihao 已提交
374
  if (reader == NULL) { return 0; }
dengyihao's avatar
dengyihao 已提交
375

dengyihao's avatar
dengyihao 已提交
376
  return tfileReaderSearch(reader, query, result);
dengyihao's avatar
dengyihao 已提交
377
}
dengyihao's avatar
dengyihao 已提交
378
int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
379 380
  // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName =
  // term->nColName, .version = 1};
dengyihao's avatar
dengyihao 已提交
381

382 383
  return 0;
}
dengyihao's avatar
dengyihao 已提交
384 385
static bool tfileIteratorNext(Iterate* iiter) {
  IterateValue* iv = &iiter->val;
dengyihao's avatar
dengyihao 已提交
386 387 388
  if (iv->colVal != NULL && iv->val != NULL) {
    // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
  }
dengyihao's avatar
dengyihao 已提交
389 390 391 392 393 394 395
  iterateValueDestroy(iv, false);

  char*    colVal = NULL;
  uint64_t offset = 0;

  TFileFstIter*          tIter = iiter->iter;
  StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
dengyihao's avatar
dengyihao 已提交
396
  if (rt == NULL) { return false; }
dengyihao's avatar
dengyihao 已提交
397 398 399 400 401 402 403 404 405

  int32_t sz = 0;
  char*   ch = (char*)fstSliceData(&rt->data, &sz);
  colVal = calloc(1, sz + 1);
  memcpy(colVal, ch, sz);

  offset = (uint64_t)(rt->out.out);
  swsResultDestroy(rt);
  // set up iterate value
dengyihao's avatar
dengyihao 已提交
406
  if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
dengyihao's avatar
dengyihao 已提交
407 408

  iv->colVal = colVal;
dengyihao's avatar
dengyihao 已提交
409
  return true;
dengyihao's avatar
dengyihao 已提交
410 411 412
  // std::string key(ch, sz);
}

413
static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
dengyihao's avatar
dengyihao 已提交
414 415

static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
416
  TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter));
dengyihao's avatar
dengyihao 已提交
417 418
  if (tIter == NULL) { return NULL; }

dengyihao's avatar
dengyihao 已提交
419 420 421 422 423 424 425 426
  tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
  tIter->fb = fstSearch(reader->fst, tIter->ctx);
  tIter->st = streamBuilderIntoStream(tIter->fb);
  tIter->rdr = reader;
  return tIter;
}

Iterate* tfileIteratorCreate(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
427
  if (reader == NULL) { return NULL; }
dengyihao's avatar
dengyihao 已提交
428

dengyihao's avatar
dengyihao 已提交
429
  Iterate* iter = calloc(1, sizeof(Iterate));
dengyihao's avatar
dengyihao 已提交
430
  iter->iter = tfileFstIteratorCreate(reader);
431
  if (iter->iter == NULL) {
dengyihao's avatar
dengyihao 已提交
432
    free(iter);
433 434
    return NULL;
  }
dengyihao's avatar
dengyihao 已提交
435 436
  iter->next = tfileIteratorNext;
  iter->getValue = tifileIterateGetValue;
dengyihao's avatar
dengyihao 已提交
437
  iter->val.val = taosArrayInit(1, sizeof(uint64_t));
438
  iter->val.colVal = NULL;
dengyihao's avatar
dengyihao 已提交
439 440 441
  return iter;
}
void tfileIteratorDestroy(Iterate* iter) {
dengyihao's avatar
dengyihao 已提交
442 443
  if (iter == NULL) { return; }

dengyihao's avatar
dengyihao 已提交
444 445 446 447 448 449 450
  IterateValue* iv = &iter->val;
  iterateValueDestroy(iv, true);

  TFileFstIter* tIter = iter->iter;
  streamWithStateDestroy(tIter->st);
  fstStreamBuilderDestroy(tIter->fb);
  automCtxDestroy(tIter->ctx);
451
  free(tIter);
dengyihao's avatar
dengyihao 已提交
452 453 454 455

  free(iter);
}

dengyihao's avatar
dengyihao 已提交
456
TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
dengyihao's avatar
dengyihao 已提交
457
  if (tf == NULL) { return NULL; }
dengyihao's avatar
dengyihao 已提交
458
  ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
dengyihao's avatar
dengyihao 已提交
459 460
  return tfileCacheGet(tf->cache, &key);
}
dengyihao's avatar
dengyihao 已提交
461

dengyihao's avatar
dengyihao 已提交
462 463 464 465 466
static int tfileUidCompare(const void* a, const void* b) {
  uint64_t l = *(uint64_t*)a;
  uint64_t r = *(uint64_t*)b;
  return l - r;
}
dengyihao's avatar
dengyihao 已提交
467 468
static int tfileStrCompare(const void* a, const void* b) {
  int ret = strcmp((char*)a, (char*)b);
dengyihao's avatar
dengyihao 已提交
469
  if (ret == 0) { return ret; }
dengyihao's avatar
dengyihao 已提交
470 471 472
  return ret < 0 ? -1 : 1;
}

dengyihao's avatar
dengyihao 已提交
473 474 475 476 477 478 479 480
static int tfileValueCompare(const void* a, const void* b, const void* param) {
  __compar_fn_t fn = *(__compar_fn_t*)param;

  TFileValue* av = (TFileValue*)a;
  TFileValue* bv = (TFileValue*)b;

  return fn(av->colVal, bv->colVal);
}
dengyihao's avatar
dengyihao 已提交
481 482 483

TFileValue* tfileValueCreate(char* val) {
  TFileValue* tf = calloc(1, sizeof(TFileValue));
dengyihao's avatar
dengyihao 已提交
484
  if (tf == NULL) { return NULL; }
485
  tf->colVal = tstrdup(val);
dengyihao's avatar
dengyihao 已提交
486 487 488 489
  tf->tableId = taosArrayInit(32, sizeof(uint64_t));
  return tf;
}
int tfileValuePush(TFileValue* tf, uint64_t val) {
dengyihao's avatar
dengyihao 已提交
490
  if (tf == NULL) { return -1; }
dengyihao's avatar
dengyihao 已提交
491 492 493 494 495
  taosArrayPush(tf->tableId, &val);
  return 0;
}
void tfileValueDestroy(TFileValue* tf) {
  taosArrayDestroy(tf->tableId);
496
  free(tf->colVal);
dengyihao's avatar
dengyihao 已提交
497 498
  free(tf);
}
dengyihao's avatar
dengyihao 已提交
499 500 501 502 503
static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) {
  int sz = taosArrayGetSize(ids);
  SERIALIZE_VAR_TO_BUF(buf, sz, int32_t);
  for (size_t i = 0; i < sz; i++) {
    uint64_t* v = taosArrayGet(ids, i);
dengyihao's avatar
dengyihao 已提交
504 505 506 507 508 509 510
    SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t);
  }
}

static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
  int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
  tw->header.fstOffset = fstOffset;
dengyihao's avatar
dengyihao 已提交
511
  if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; }
dengyihao's avatar
dengyihao 已提交
512
  tw->offset += sizeof(fstOffset);
dengyihao's avatar
dengyihao 已提交
513 514 515
  return 0;
}
static int tfileWriteHeader(TFileWriter* writer) {
dengyihao's avatar
dengyihao 已提交
516
  char buf[TFILE_HEADER_NO_FST] = {0};
dengyihao's avatar
dengyihao 已提交
517 518 519 520 521

  TFileHeader* header = &writer->header;
  memcpy(buf, (char*)header, sizeof(buf));

  int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
dengyihao's avatar
dengyihao 已提交
522
  if (sizeof(buf) != nwrite) { return -1; }
dengyihao's avatar
dengyihao 已提交
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
  writer->offset = nwrite;
  return 0;
}
static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
  TFileHeader* header = &write->header;
  uint8_t      colType = header->colType;
  if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
    FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal));
    if (fstBuilderInsert(write->fb, key, tval->offset)) {
      fstSliceDestroy(&key);
      return 0;
    }
    fstSliceDestroy(&key);
    return -1;
  } else {
    // handle other type later
  }
dengyihao's avatar
dengyihao 已提交
540
  return 0;
dengyihao's avatar
dengyihao 已提交
541
}
dengyihao's avatar
dengyihao 已提交
542
static int tfileReaderLoadHeader(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
543
  // TODO simple tfile header later
dengyihao's avatar
dengyihao 已提交
544
  char buf[TFILE_HEADER_SIZE] = {0};
dengyihao's avatar
dengyihao 已提交
545

dengyihao's avatar
dengyihao 已提交
546
  int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
dengyihao's avatar
dengyihao 已提交
547 548 549 550
  if (nread == -1) {
    //
    indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
               errno, reader->ctx->file.fd, reader->ctx->file.buf);
dengyihao's avatar
dengyihao 已提交
551 552 553
  } else {
    indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
               errno, reader->ctx->file.fd, reader->ctx->file.buf);
dengyihao's avatar
dengyihao 已提交
554 555
  }
  // assert(nread == sizeof(buf));
dengyihao's avatar
dengyihao 已提交
556
  memcpy(&reader->header, buf, sizeof(buf));
dengyihao's avatar
dengyihao 已提交
557

dengyihao's avatar
dengyihao 已提交
558 559
  return 0;
}
dengyihao's avatar
dengyihao 已提交
560
static int tfileReaderLoadFst(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
561
  // current load fst into memory, refactor it later
dengyihao's avatar
dengyihao 已提交
562
  static int FST_MAX_SIZE = 64 * 1024 * 1024;
dengyihao's avatar
dengyihao 已提交
563 564

  char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
dengyihao's avatar
dengyihao 已提交
565
  if (buf == NULL) { return -1; }
dengyihao's avatar
dengyihao 已提交
566 567 568

  WriterCtx* ctx = reader->ctx;
  int32_t    nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
dengyihao's avatar
dengyihao 已提交
569
  indexError("nread = %d, and fst offset=%d, filename: %s ", nread, reader->header.fstOffset, ctx->file.buf);
dengyihao's avatar
dengyihao 已提交
570 571 572 573 574 575 576 577
  // we assuse fst size less than FST_MAX_SIZE
  assert(nread > 0 && nread < FST_MAX_SIZE);

  FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
  reader->fst = fstCreate(&st);
  free(buf);
  fstSliceDestroy(&st);

dengyihao's avatar
dengyihao 已提交
578
  return reader->fst != NULL ? 0 : -1;
dengyihao's avatar
dengyihao 已提交
579
}
dengyihao's avatar
dengyihao 已提交
580
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
dengyihao's avatar
dengyihao 已提交
581 582
  int32_t    nid;
  WriterCtx* ctx = reader->ctx;
dengyihao's avatar
dengyihao 已提交
583 584

  int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset);
dengyihao's avatar
dengyihao 已提交
585 586
  assert(sizeof(nid) == nread);

dengyihao's avatar
dengyihao 已提交
587 588
  int32_t total = sizeof(uint64_t) * nid;
  char*   buf = calloc(1, total);
dengyihao's avatar
dengyihao 已提交
589
  if (buf == NULL) { return -1; }
dengyihao's avatar
dengyihao 已提交
590

dengyihao's avatar
dengyihao 已提交
591
  nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid));
dengyihao's avatar
dengyihao 已提交
592 593
  assert(total == nread);

dengyihao's avatar
dengyihao 已提交
594
  for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); }
dengyihao's avatar
dengyihao 已提交
595 596 597
  free(buf);
  return 0;
}
dengyihao's avatar
dengyihao 已提交
598
void tfileReaderRef(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
599
  if (reader == NULL) { return; }
dengyihao's avatar
dengyihao 已提交
600 601 602 603
  int ref = T_REF_INC(reader);
  UNUSED(ref);
}

dengyihao's avatar
dengyihao 已提交
604
void tfileReaderUnRef(TFileReader* reader) {
dengyihao's avatar
dengyihao 已提交
605
  if (reader == NULL) { return; }
dengyihao's avatar
dengyihao 已提交
606
  int ref = T_REF_DEC(reader);
607
  if (ref == 0) {
dengyihao's avatar
dengyihao 已提交
608
    // do nothing
609 610
    tfileReaderDestroy(reader);
  }
dengyihao's avatar
dengyihao 已提交
611
}
dengyihao's avatar
dengyihao 已提交
612

dengyihao's avatar
dengyihao 已提交
613 614 615
static SArray* tfileGetFileList(const char* path) {
  SArray* files = taosArrayInit(4, sizeof(void*));

dengyihao's avatar
dengyihao 已提交
616
  DIR* dir = opendir(path);
dengyihao's avatar
dengyihao 已提交
617
  if (NULL == dir) { return NULL; }
dengyihao's avatar
dengyihao 已提交
618 619 620

  struct dirent* entry;
  while ((entry = readdir(dir)) != NULL) {
dengyihao's avatar
dengyihao 已提交
621
    if (entry->d_type && DT_DIR) { continue; }
dengyihao's avatar
dengyihao 已提交
622 623 624
    size_t len = strlen(entry->d_name);
    char*  buf = calloc(1, len + 1);
    memcpy(buf, entry->d_name, len);
dengyihao's avatar
dengyihao 已提交
625
    taosArrayPush(files, &buf);
dengyihao's avatar
dengyihao 已提交
626 627
  }
  closedir(dir);
dengyihao's avatar
dengyihao 已提交
628 629 630 631 632

  taosArraySort(files, tfileCompare);
  tfileRmExpireFile(files);

  return files;
dengyihao's avatar
dengyihao 已提交
633
}
dengyihao's avatar
dengyihao 已提交
634 635 636 637
static int tfileRmExpireFile(SArray* result) {
  // TODO(yihao): remove expire tindex after restart
  return 0;
}
dengyihao's avatar
dengyihao 已提交
638 639 640 641 642 643 644 645 646 647 648
static void tfileDestroyFileName(void* elem) {
  char* p = *(char**)elem;
  free(p);
}
static int tfileCompare(const void* a, const void* b) {
  const char* aName = *(char**)a;
  const char* bName = *(char**)b;

  size_t aLen = strlen(aName);
  size_t bLen = strlen(bName);

649
  int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen);
dengyihao's avatar
dengyihao 已提交
650
  if (ret == 0) { return ret; }
651
  return ret < 0 ? -1 : 1;
dengyihao's avatar
dengyihao 已提交
652
}
dengyihao's avatar
dengyihao 已提交
653 654 655

static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) {
  if (3 == sscanf(filename, "%" PRIu64 "-%[^-]-%d.tindex", suid, col, version)) {
dengyihao's avatar
dengyihao 已提交
656 657 658 659 660
    // read suid & colid & version  success
    return 0;
  }
  return -1;
}
dengyihao's avatar
dengyihao 已提交
661 662 663 664 665 666 667 668 669 670
// tfile name suid-colId-version.tindex
static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version) {
  sprintf(filename, "%" PRIu64 "-%s-%d.tindex", suid, col, version);
  return;
}
static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version) {
  char filename[128] = {0};
  tfileGenFileName(filename, suid, col, version);
  sprintf(fullname, "%s/%s", path, filename);
}