add 2.0 tsdb

source/dnode/vnode/tsdb2/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 3.0...3.20)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(tsdb ${SRC})
+TARGET_LINK_LIBRARIES(tsdb tfs common tutil cJson)
+
+IF (TD_TSDB_PLUGINS)
+  TARGET_LINK_LIBRARIES(tsdb tsdbPlugins)
+ENDIF ()
+
+IF (TD_LINUX)
+  # Someone has no gtest directory, so comment it
+  # ADD_SUBDIRECTORY(tests)
+ENDIF ()

source/dnode/vnode/tsdb2/inc/tsdbBuffer.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_BUFFER_H_
+#define _TD_TSDB_BUFFER_H_
+
+typedef struct {
+  int64_t blockId;
+  int     offset;
+  int     remain;
+  char    data[];
+} STsdbBufBlock;
+
+typedef struct {
+  pthread_cond_t poolNotEmpty;
+  int            bufBlockSize;
+  int            tBufBlocks;
+  int            nBufBlocks;
+  int            nRecycleBlocks;
+  int            nElasticBlocks;
+  int64_t        index;
+  SList*         bufBlockList;  
+} STsdbBufPool;
+
+#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
+
+STsdbBufPool* tsdbNewBufPool();
+void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
+int           tsdbOpenBufPool(STsdbRepo* pRepo);
+void          tsdbCloseBufPool(STsdbRepo* pRepo);
+SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
+int           tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks);
+void          tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic);
+
+// health cite
+STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize);
+void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock);
+
+#endif /* _TD_TSDB_BUFFER_H_ */

source/dnode/vnode/tsdb2/inc/tsdbCommit.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_COMMIT_H_
+#define _TD_TSDB_COMMIT_H_
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
+
+void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
+int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
+void *tsdbCommitData(STsdbRepo *pRepo);
+int   tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
+int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx);
+int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
+int   tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
+                         SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf);
+int   tsdbApplyRtn(STsdbRepo *pRepo);
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
+}
+
+#endif /* _TD_TSDB_COMMIT_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbCommitQueue.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_COMMIT_QUEUE_H_
+#define _TD_TSDB_COMMIT_QUEUE_H_
+
+typedef enum { COMMIT_REQ, COMPACT_REQ,COMMIT_CONFIG_REQ } TSDB_REQ_T;
+
+int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req);
+
+#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbCompact.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_TSDB_COMPACT_H_
+#define _TD_TSDB_COMPACT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void *tsdbCompactImpl(STsdbRepo *pRepo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TD_TSDB_COMPACT_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbFS.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_FS_H_
+#define _TD_TSDB_FS_H_
+
+/**
+ * 1. The fileset .head/.data/.last use the same fver 0 before 2021.10.10.
+ * 2. .head fver is 1 when extract aggregate block data from .data/.last file and save to separate .smad/.smal file
+ * since 2021.10.10
+ * // TODO update date and add release version.
+ */
+typedef enum {
+  TSDB_FS_VER_0 = 0,
+  TSDB_FS_VER_1,
+} ETsdbFsVer;
+
+#define TSDB_FVER_TYPE uint32_t
+#define TSDB_LATEST_FVER TSDB_FS_VER_1     // latest version for DFile
+#define TSDB_LATEST_SFS_VER TSDB_FS_VER_1  // latest version for 'current' file
+
+static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest version for DFile
+  switch (fType) {
+    case TSDB_FILE_HEAD:
+      return TSDB_FS_VER_1;
+    default:
+      return TSDB_FS_VER_0;
+  }
+}
+
+// ================== TSDB global config
+extern bool tsdbForceKeepFile;
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SMFile*     pmf;   // meta file pointer
+  SMFile      mf;    // meta file
+  SArray*     df;    // data file array
+} SFSStatus;
+
+typedef struct {
+  pthread_rwlock_t lock;
+
+  SFSStatus* cstatus;         // current status
+  SHashObj*  metaCache;       // meta cache
+  SHashObj*  metaCacheComp;   // meta cache for compact
+  bool       intxn;
+  SFSStatus* nstatus;  // new status
+} STsdbFS;
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS*   pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet* pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdbRepo *pRepo);
+void     tsdbCloseFS(STsdbRepo *pRepo);
+void     tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdbRepo *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_rdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_wrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_unlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+#endif /* _TD_TSDB_FS_H_ */

source/dnode/vnode/tsdb2/inc/tsdbFile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TS_TSDB_FILE_H_
+#define _TS_TSDB_FILE_H_
+
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+#define TSDB_FILE_STATE_OK 0
+#define TSDB_FILE_STATE_BAD 1
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
+#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
+#define TSDB_FILE_FSYNC(tf) taosFsync(TSDB_FILE_FD(tf))
+#define TSDB_FILE_STATE(tf) ((tf)->state)
+#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
+#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
+#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
+#define ASSERT_TSDB_FSET_NFILES_VALID(s)                              \
+  do {                                                                \
+    uint8_t nDFiles = tsdbGetNFiles(s);                               \
+    ASSERT((nDFiles >= TSDB_FILE_MIN) && (nDFiles <= TSDB_FILE_MAX)); \
+  } while (0)
+typedef enum {
+  TSDB_FILE_HEAD = 0,
+  TSDB_FILE_DATA,
+  TSDB_FILE_LAST,
+  TSDB_FILE_SMAD,  // sma for .data
+  TSDB_FILE_SMAL,  // sma for .last
+  TSDB_FILE_MAX,
+  TSDB_FILE_META
+} TSDB_FILE_T;
+
+#define TSDB_FILE_MIN 3U  // min valid number of files in one DFileSet(.head/.data/.last)
+
+// =============== SMFile
+typedef struct {
+  int64_t  size;
+  int64_t  tombSize;
+  int64_t  nRecords;
+  int64_t  nDels;
+  uint32_t magic;
+} SMFInfo;
+
+typedef struct {
+  SMFInfo info;
+  TFILE   f;
+  int     fd;
+  uint8_t state;
+} SMFile;
+
+void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
+void  tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
+int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
+int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
+int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
+int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
+int   tsdbUpdateMFileHeader(SMFile* pMFile);
+int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
+int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
+int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
+void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
+
+static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
+  ASSERT(TSDB_FILE_CLOSED(pMFile));
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags | O_BINARY);
+  if (pMFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
+  if (TSDB_FILE_OPENED(pMFile)) {
+    close(pMFile->fd);
+    TSDB_FILE_SET_CLOSED(pMFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pMFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pMFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
+
+static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+// =============== SDFile
+typedef struct {
+  uint32_t magic;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+  uint32_t fver;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo info;
+  TFILE   f;
+  int     fd;
+  uint8_t state;
+} SDFile;
+
+void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(void* buf, SDFile* pDFile, uint32_t sfver);
+int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader, TSDB_FILE_T ftype);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags | O_BINARY);
+  if (pDFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    close(pDFile->fd);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+typedef struct {
+  int     fid;
+  int     state;
+  uint16_t ver;  // fset version
+  SDFile  files[TSDB_FILE_MAX];
+} SDFileSet;
+
+typedef enum {
+  TSDB_FSET_VER_0 = 0,  // .head/.data/.last
+  TSDB_FSET_VER_1,      // .head/.data/.last/.smad/.smal
+} ETsdbFSetVer;
+
+#define TSDB_LATEST_FSET_VER TSDB_FSET_VER_1
+
+// get nDFiles in SDFileSet
+static FORCE_INLINE uint8_t tsdbGetNFiles(SDFileSet* pSet) {
+  switch (pSet->ver) {
+    case TSDB_FSET_VER_0:
+      return TSDB_FILE_MIN;
+    case TSDB_FSET_VER_1:
+    default:
+      return TSDB_FILE_MAX;
+  }
+}
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                        \
+  do {                                                                            \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < tsdbGetNFiles(s); ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                               \
+    }                                                                             \
+  } while (0);
+
+void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver, uint16_t fsetVer);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet, uint32_t sfver);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSrc);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSrc); ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsTickPerDay[precision];
+  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
+}
+
+static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+#endif /* _TS_TSDB_FILE_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbHealth.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_HEALTH_H_
+#define _TD_TSDB_HEALTH_H_
+
+bool tsdbUrgeQueryFree(STsdbRepo* pRepo);
+int32_t tsdbInsertNewBlock(STsdbRepo* pRepo);
+
+bool tsdbIdleMemEnough();
+bool tsdbAllowNewBlock(STsdbRepo* pRepo);
+
+#endif /* _TD_TSDB_BUFFER_H_ */

source/dnode/vnode/tsdb2/inc/tsdbLog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbMemTable.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMTABLE_H_
+#define _TD_TSDB_MEMTABLE_H_
+
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
+
+typedef struct {
+  STable *           pTable;
+  SSkipListIterator *pIter;
+} SCommitIter;
+
+struct STableData {
+  uint64_t   uid;
+  TSKEY      keyFirst;
+  TSKEY      keyLast;
+  int64_t    numOfRows;
+  SSkipList* pData;
+  T_REF_DECLARE()
+};
+
+enum { TSDB_UPDATE_META, TSDB_DROP_META };
+
+#ifdef WINDOWS
+#pragma pack(push ,1) 
+typedef struct {
+#else
+typedef struct __attribute__((packed)){
+#endif
+  char     act;
+  uint64_t uid;
+} SActObj;
+#ifdef WINDOWS
+#pragma pack(pop) 
+#endif
+
+typedef struct {
+  int  len;
+  char cont[];
+} SActCont;
+
+int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pATable);
+void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot);
+void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
+int   tsdbAsyncCommit(STsdbRepo* pRepo);
+int   tsdbSyncCommitConfig(STsdbRepo* pRepo);
+int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
+                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdbRepo* pRepo);
+
+static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) {
+  if (pIter == NULL) return NULL;
+
+  SSkipListNode* node = tSkipListIterGet(pIter);
+  if (node == NULL) return NULL;
+
+  return (SMemRow)SL_GET_NODE_DATA(node);
+}
+
+static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
+  SMemRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
+
+  return memRowKey(row);
+}
+
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+  SMemRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TKEY_NULL;
+
+  return memRowTKey(row);
+}
+
+#endif /* _TD_TSDB_MEMTABLE_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbMeta.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_META_H_
+#define _TD_TSDB_META_H_
+
+#define TSDB_MAX_TABLE_SCHEMAS 16
+
+#pragma  pack (push,1)
+typedef struct jsonMapValue{
+  void* table;      // STable *
+  int16_t  colId;   // the json col ID.
+}JsonMapValue;
+
+#pragma  pack (pop)
+
+typedef struct STable {
+  STableId       tableId;
+  ETableType     type;
+  tstr*          name;  // NOTE: there a flexible string here
+  uint64_t       suid;
+  struct STable* pSuper;  // super table pointer
+  SArray*        schema;
+  STSchema*      tagSchema;
+  SKVRow         tagVal;
+  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
+  SHashObj*      jsonKeyMap;     // For json tag key  {"key":[t1, t2, t3]}
+  void*          eventHandler;   // TODO
+  void*          streamHandler;  // TODO
+  TSKEY          lastKey;
+  SMemRow        lastRow;
+  char*          sql;
+  void*          cqhandle;
+  SRWLatch       latch;  // TODO: implementa latch functions
+
+  SDataCol      *lastCols;
+  int16_t        maxColNum;
+  int16_t        restoreColumnNum;
+  bool           hasRestoreLastColumn;
+  int            lastColSVersion;
+  int16_t        cacheLastConfigVersion;
+  T_REF_DECLARE()
+} STable;
+
+typedef struct {
+  pthread_rwlock_t rwLock;
+
+  int32_t   nTables;
+  int32_t   maxTables;
+  STable**  tables;
+  SList*    superList;
+  SHashObj* uidMap;
+  int       maxRowBytes;
+  int       maxCols;
+} STsdbMeta;
+
+#define TSDB_INIT_NTABLES 1024
+#define TABLE_TYPE(t) (t)->type
+#define TABLE_NAME(t) (t)->name
+#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
+#define TABLE_UID(t) (t)->tableId.uid
+#define TABLE_TID(t) (t)->tableId.tid
+#define TABLE_SUID(t) (t)->suid
+// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
+#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
+#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
+#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
+#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
+
+STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
+void       tsdbFreeMeta(STsdbMeta* pMeta);
+int        tsdbOpenMeta(STsdbRepo* pRepo);
+int        tsdbCloseMeta(STsdbRepo* pRepo);
+STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
+STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t _version, int8_t rowType);
+int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
+void       tsdbRefTable(STable* pTable);
+void       tsdbUnRefTable(STable* pTable);
+void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
+int        tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen);
+void       tsdbOrgMeta(STsdbRepo* pRepo);
+int        tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema);
+int16_t    tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId);
+int        tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema);
+STSchema*  tsdbGetTableLatestSchema(STable *pTable);
+void       tsdbFreeLastColumns(STable* pTable);
+int        tsdbCompareJsonMapValue(const void* a, const void* b);
+void*      tsdbGetJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId);
+
+static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
+    return -1;
+  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t _version, int8_t rowType) {
+  STable*   pDTable = (pTable->pSuper != NULL) ? pTable->pSuper : pTable;  // for performance purpose
+  STSchema* pSchema = NULL;
+  STSchema* pTSchema = NULL;
+
+  if (lock) TSDB_RLOCK_TABLE(pDTable);
+  if (_version < 0) {  // get the latest version of schema
+    pTSchema = *(STSchema **)taosArrayGetLast(pDTable->schema);
+  } else {  // get the schema with version
+    void* ptr = taosArraySearch(pDTable->schema, &_version, tsdbCompareSchemaVersion, TD_EQ);
+    if (ptr == NULL) {
+      if (rowType == SMEM_ROW_KV) {
+        ptr = taosArrayGetLast(pDTable->schema);
+      } else {
+        terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+        goto _exit;
+      }
+    }
+    pTSchema = *(STSchema**)ptr;
+  }
+
+  ASSERT(pTSchema != NULL);
+
+  if (copy) {
+    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  } else {
+    pSchema = pTSchema;
+  }
+
+_exit:
+  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
+  return pSchema;
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
+  return tsdbGetTableSchemaImpl(pTable, false, false, -1, -1);
+}
+
+static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
+  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
+    STable *pSuper = pTable->pSuper;
+    if (pSuper == NULL) return NULL;
+    return pSuper->tagSchema;
+  } else if (pTable->type == TSDB_SUPER_TABLE) {
+    return pTable->tagSchema;
+  } else {
+    return NULL;
+  }
+}
+
+static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
+  ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow)));
+  return pTable->lastKey;
+}
+
+#endif /* _TD_TSDB_META_H_ */
\ No newline at end of file

source/dnode/vnode/tsdb2/inc/tsdbReadImpl.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_READ_IMPL_H_
+#define _TD_TSDB_READ_IMPL_H_
+
+#include "tfs.h"
+#include "tsdb.h"
+#include "os.h"
+#include "tsdbFile.h"
+#include "tskiplist.h"
+#include "tsdbMeta.h"
+
+typedef struct SReadH SReadH;
+
+typedef struct {
+  int32_t  tid;
+  uint32_t len;
+  uint32_t offset;
+  uint32_t hasLast : 2;
+  uint32_t numOfBlocks : 30;
+  uint64_t uid;
+  TSKEY    maxKey;
+} SBlockIdx;
+
+#if 0
+typedef struct {
+  int64_t last : 1;
+  int64_t offset : 63;
+  int32_t algorithm : 8;
+  int32_t numOfRows : 24;
+  int32_t len;
+  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  int16_t numOfSubBlocks;
+  int16_t numOfCols;  // not including timestamp column
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  } SBlock;
+#endif
+
+/**
+ * keyLen;     // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+ * numOfCols;  // not including timestamp column
+ */
+#define SBlockFieldsP0    \
+  int64_t last : 1;       \
+  int64_t offset : 63;    \
+  int32_t algorithm : 8;  \
+  int32_t numOfRows : 24; \
+  int32_t len;            \
+  int32_t keyLen;         \
+  int16_t numOfSubBlocks; \
+  int16_t numOfCols;      \
+  TSKEY   keyFirst;       \
+  TSKEY   keyLast
+
+/**
+ * aggrStat;   // only valid when blkVer > 0. 0 - no aggr part in .data/.last/.smad/.smal, 1 - has aggr in .smad/.smal
+ * blkVer;     // 0 - original block, 1 - block since importing .smad/.smal
+ * aggrOffset; // only valid when blkVer > 0 and aggrStat > 0
+ */
+#define SBlockFieldsP1   \
+  uint64_t aggrStat : 1; \
+  uint64_t blkVer : 7;   \
+  uint64_t aggrOffset : 56
+
+typedef struct {
+  SBlockFieldsP0;
+} SBlockV0;
+
+typedef struct {
+  SBlockFieldsP0;
+  SBlockFieldsP1;
+} SBlockV1;
+
+typedef enum {
+  TSDB_SBLK_VER_0 = 0,
+  TSDB_SBLK_VER_1,
+} ESBlockVer;
+
+#define SBlockVerLatest TSDB_SBLK_VER_1
+
+#define SBlock SBlockV1      // latest SBlock definition
+
+// lastest SBlockInfo definition
+typedef struct {
+  int32_t  delimiter;  // For recovery usage
+  int32_t  tid;
+  uint64_t uid;
+  SBlock   blocks[];
+} SBlockInfo;
+
+typedef struct {
+  int16_t  colId;
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+  int64_t  sum;
+  int64_t  max;
+  int64_t  min;
+  int16_t  maxIndex;
+  int16_t  minIndex;
+  int16_t  numOfNull;
+  uint8_t  offsetH;
+  char     padding[1];
+} SBlockColV0;
+
+typedef struct {
+  int16_t  colId;
+  uint8_t  offsetH;
+  uint8_t  reserved;  // reserved field, not used
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+} SBlockColV1;
+
+#define SBlockCol SBlockColV1      // latest SBlockCol definition
+
+typedef struct {
+  int16_t colId;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+} SAggrBlkColV1;
+
+#define SAggrBlkCol SAggrBlkColV1  // latest SAggrBlkCol definition
+
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+}
+
+typedef struct {
+  int32_t   delimiter;  // For recovery usage
+  int32_t   numOfCols;  // For recovery usage
+  uint64_t  uid;        // For recovery usage
+  SBlockCol cols[];
+} SBlockData;
+
+typedef void SAggrBlkData;  // SBlockCol cols[];
+
+struct SReadH {
+  STsdbRepo * pRepo;
+  SDFileSet   rSet;     // FSET to read
+  SArray *    aBlkIdx;  // SBlockIdx array
+  STable *    pTable;   // table to read
+  SBlockIdx * pBlkIdx;  // current reading table SBlockIdx
+  int         cidx;
+  SBlockInfo *  pBlkInfo;  // SBlockInfoV#
+  SBlockData *pBlkData;  // Block info
+  SAggrBlkData *pAggrBlkData;  // Aggregate Block info
+  SDataCols * pDCols[2];
+  void *      pBuf;   // buffer
+  void *      pCBuf;  // compression buffer
+  void *      pExBuf;  // extra buffer
+};
+
+#define TSDB_READ_REPO(rh) ((rh)->pRepo)
+#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
+#define TSDB_READ_FSET(rh) (&((rh)->rSet))
+#define TSDB_READ_TABLE(rh) ((rh)->pTable)
+#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
+#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
+#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
+#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
+#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
+#define TSDB_READ_BUF(rh) ((rh)->pBuf)
+#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
+#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf)
+
+#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
+  (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+      return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
+    case TSDB_SBLK_VER_1:
+    default:
+      return TSDB_BLOCK_STATIS_SIZE(nCols, 1);
+  }
+}
+
+#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+      ASSERT(false);
+      return 0;
+    case TSDB_SBLK_VER_1:
+    default:
+      return TSDB_BLOCK_AGGR_SIZE(nCols, 1);
+  }
+}
+
+int   tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo);
+void  tsdbDestroyReadH(SReadH *pReadh);
+int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
+void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
+int   tsdbLoadBlockIdx(SReadH *pReadh);
+int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen);
+int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
+int   tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
+int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
+void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
+
+static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
+  void * pBuf = *ppBuf;
+  size_t tsize = taosTSizeof(pBuf);
+
+  if (tsize < size) {
+    if (tsize == 0) tsize = 1024;
+
+    while (tsize < size) {
+      tsize *= 2;
+    }
+
+    *ppBuf = taosTRealloc(pBuf, tsize);
+    if (*ppBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE SBlockCol *tsdbGetSBlockCol(SBlock *pBlock, SBlockCol **pDestBlkCol, SBlockCol *pBlkCols,
+                                                int colIdx) {
+  if (pBlock->blkVer == SBlockVerLatest) {
+    *pDestBlkCol = pBlkCols + colIdx;
+    return *pDestBlkCol;
+  }
+  if (pBlock->blkVer == TSDB_SBLK_VER_0) {
+    SBlockColV0 *pBlkCol = (SBlockColV0 *)pBlkCols + colIdx;
+    (*pDestBlkCol)->colId = pBlkCol->colId;
+    (*pDestBlkCol)->len = pBlkCol->len;
+    (*pDestBlkCol)->type = pBlkCol->type;
+    (*pDestBlkCol)->offset = pBlkCol->offset;
+    (*pDestBlkCol)->offsetH = pBlkCol->offsetH;
+  }
+  return *pDestBlkCol;
+}
+
+#endif /*_TD_TSDB_READ_IMPL_H_*/

source/dnode/vnode/tsdb2/inc/tsdbRowMergeBuf.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TSDB_ROW_MERGE_BUF_H
+#define TSDB_ROW_MERGE_BUF_H 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tsdb.h"
+#include "tchecksum.h"
+#include "tsdbReadImpl.h"
+
+typedef void* SMergeBuf;
+
+SDataRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
+
+static FORCE_INLINE int tsdbMergeBufMakeSureRoom(SMergeBuf *pBuf, STSchema* pSchema1, STSchema* pSchema2) {
+  size_t len1 = dataRowMaxBytesFromSchema(pSchema1);
+  size_t len2 = dataRowMaxBytesFromSchema(pSchema2);
+  return tsdbMakeRoom(pBuf, MAX(len1, len2));
+}
+
+static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) {
+  taosTZfree(buf);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef TSDB_ROW_MERGE_BUF_H */

source/dnode/vnode/tsdb2/inc/tsdbint.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_INT_H_
+#define _TD_TSDB_INT_H_
+
+// // TODO: remove the include
+// #include <errno.h>
+// #include <fcntl.h>
+// #include <limits.h>
+// #include <inttypes.h>
+// #include <sys/stat.h>
+// #include <sys/types.h>
+// #include <semaphore.h>
+// #include <dirent.h>
+
+#include "os.h"
+#include "tlog.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "tskiplist.h"
+#include "tdataformat.h"
+#include "tcoding.h"
+#include "tscompression.h"
+#include "tlockfree.h"
+#include "tlist.h"
+#include "hash.h"
+#include "tarray.h"
+#include "tfs.h"
+#include "tsocket.h"
+
+#include "tsdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Log
+#include "tsdbLog.h"
+// Meta
+#include "tsdbMeta.h"
+// Buffer
+#include "tsdbBuffer.h"
+// MemTable
+#include "tsdbMemTable.h"
+// File
+#include "tsdbFile.h"
+// FS
+#include "tsdbFS.h"
+// ReadImpl
+#include "tsdbReadImpl.h"
+// Commit
+#include "tsdbCommit.h"
+// Compact
+#include "tsdbCompact.h"
+// Commit Queue
+#include "tsdbCommitQueue.h"
+
+#include "tsdbRowMergeBuf.h"
+// Main definitions
+struct STsdbRepo {
+  uint8_t state;
+
+  STsdbCfg        config;
+
+  STsdbCfg        save_config;    // save apply config
+  bool            config_changed; // config changed flag
+  pthread_mutex_t save_mutex;     // protect save config
+
+  int16_t         cacheLastConfigVersion;
+
+  STsdbAppH       appH;
+  STsdbStat       stat;
+  STsdbMeta*      tsdbMeta;
+  STsdbBufPool*   pPool;
+  SMemTable*      mem;
+  SMemTable*      imem;
+  STsdbFS*        fs;
+  SRtn            rtn;
+  tsem_t          readyToCommit;
+  pthread_mutex_t mutex;
+  bool            repoLocked;
+  int32_t         code;  // Commit code
+
+  SMergeBuf       mergeBuf;  //used when update=2
+  int8_t          compactState;  // compact state: inCompact/noCompact/waitingCompact?
+  pthread_t*      pthread;
+};
+
+#define REPO_ID(r) (r)->config.tsdbId
+#define REPO_CFG(r) (&((r)->config))
+#define REPO_FS(r) ((r)->fs)
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+int        tsdbLockRepo(STsdbRepo* pRepo);
+int        tsdbUnlockRepo(STsdbRepo* pRepo);
+STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo);
+int        tsdbCheckCommit(STsdbRepo* pRepo);
+int        tsdbRestoreInfo(STsdbRepo* pRepo);
+UNUSED_FUNC int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg);
+int32_t    tsdbLoadLastCache(STsdbRepo *pRepo, STable* pTable);
+void       tsdbGetRootDir(int repoid, char dirName[]);
+void       tsdbGetDataDir(int repoid, char dirName[]);
+
+static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
+  ASSERT(pRepo != NULL);
+  if (pRepo->mem == NULL) return NULL;
+
+  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
+  if (pNode == NULL) return NULL;
+
+  STsdbBufBlock* pBufBlock = NULL;
+  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
+
+  return pBufBlock;
+}
+
+static FORCE_INLINE int tsdbGetNextMaxTables(int tid) {
+  ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES);
+  int maxTables = TSDB_INIT_NTABLES;
+  while (true) {
+    maxTables = MIN(maxTables, TSDB_MAX_TABLES);
+    if (tid <= maxTables) break;
+    maxTables *= 2;
+  }
+
+  return maxTables + 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TD_TSDB_INT_H_ */

source/dnode/vnode/tsdb2/src/tsdbBuffer.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+#include "tsdbHealth.h"
+
+#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+STsdbBufPool *tsdbNewBufPool() {
+  STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool));
+  if (pBufPool == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL);
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    goto _err;
+  }
+
+  pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *));
+  if (pBufPool->bufBlockList == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  return pBufPool;
+
+_err:
+  tsdbFreeBufPool(pBufPool);
+  return NULL;
+}
+
+void tsdbFreeBufPool(STsdbBufPool *pBufPool) {
+  if (pBufPool) {
+    if (pBufPool->bufBlockList) {
+      ASSERT(listNEles(pBufPool->bufBlockList) == 0);
+      tdListFree(pBufPool->bufBlockList);
+    }
+
+    pthread_cond_destroy(&pBufPool->poolNotEmpty);
+
+    free(pBufPool);
+  }
+}
+
+int tsdbOpenBufPool(STsdbRepo *pRepo) {
+  STsdbCfg *    pCfg = &(pRepo->config);
+  STsdbBufPool *pPool = pRepo->pPool;
+
+  ASSERT(pPool != NULL);
+  pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB
+  pPool->tBufBlocks = pCfg->totalBlocks;
+  pPool->nBufBlocks = 0;
+  pPool->nElasticBlocks = 0;
+  pPool->index = 0;
+  pPool->nRecycleBlocks = 0;
+
+  for (int i = 0; i < pCfg->totalBlocks; i++) {
+    STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
+    if (pBufBlock == NULL) goto _err;
+
+    if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
+      tsdbFreeBufBlock(pBufBlock);
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      goto _err;
+    }
+
+    pPool->nBufBlocks++;
+  }
+
+  tsdbDebug("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo),
+            pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks);
+
+  return 0;
+
+_err:
+  tsdbCloseBufPool(pRepo);
+  return -1;
+}
+
+void tsdbCloseBufPool(STsdbRepo *pRepo) {
+  if (pRepo == NULL) return;
+
+  STsdbBufPool * pBufPool = pRepo->pPool;
+  STsdbBufBlock *pBufBlock = NULL;
+
+  if (pBufPool) {
+    SListNode *pNode = NULL;
+    while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) {
+      tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+      tsdbFreeBufBlock(pBufBlock);
+      free(pNode);
+    }
+  }
+
+  tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
+}
+
+SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
+  ASSERT(pRepo != NULL && pRepo->pPool != NULL);
+  ASSERT(IS_REPO_LOCKED(pRepo));
+
+  STsdbBufPool *pBufPool = pRepo->pPool;
+
+  while (POOL_IS_EMPTY(pBufPool)) {
+    if(tsDeadLockKillQuery) {
+      // supply new Block 
+      if(tsdbInsertNewBlock(pRepo) > 0) {
+        tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles);
+        break;
+      } else {
+        // no newBlock, kill query free
+        if(!tsdbUrgeQueryFree(pRepo))
+          tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo));
+      }
+    }
+
+    pRepo->repoLocked = false;
+    pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
+    pRepo->repoLocked = true;
+  }
+
+  SListNode *    pNode = tdListPopHead(pBufPool->bufBlockList);
+  ASSERT(pNode != NULL);
+  STsdbBufBlock *pBufBlock = NULL;
+  tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+
+  pBufBlock->blockId = pBufPool->index++;
+  pBufBlock->offset = 0;
+  pBufBlock->remain = pBufPool->bufBlockSize;
+
+  tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
+  return pNode;
+}
+
+// ---------------- LOCAL FUNCTIONS ----------------
+STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) {
+  STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize);
+  if (pBufBlock == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pBufBlock->blockId = 0;
+  pBufBlock->offset = 0;
+  pBufBlock->remain = bufBlockSize;
+
+  return pBufBlock;
+}
+
+ void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
+
+int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) {
+  if (oldTotalBlocks == pRepo->config.totalBlocks) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int err = TSDB_CODE_SUCCESS;
+
+  if (tsdbLockRepo(pRepo) < 0) return terrno;
+  STsdbBufPool* pPool = pRepo->pPool;
+
+  if (pRepo->config.totalBlocks > oldTotalBlocks) {
+    for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) {
+      STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
+      if (pBufBlock == NULL) goto err;
+
+      if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
+        tsdbFreeBufBlock(pBufBlock);
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        err = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        goto err;
+      }
+
+      pPool->nBufBlocks++;      
+    }
+    pthread_cond_signal(&pPool->poolNotEmpty);
+  } else {
+    pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks;
+  } 
+
+err:
+  tsdbUnlockRepo(pRepo);
+  return err;
+}
+
+void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) {
+  STsdbBufBlock *pBufBlock = NULL;
+  tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock));
+  tsdbFreeBufBlock(pBufBlock);
+  free(pNode);
+  if(bELastic)
+  {
+    pPool->nElasticBlocks--;
+    tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles);
+  }
+  else
+    pPool->nBufBlocks--;
+}
\ No newline at end of file

source/dnode/vnode/tsdb2/src/tsdbCommitQueue.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+
+typedef struct {
+  bool            stop;
+  pthread_mutex_t lock;
+  pthread_cond_t  queueNotEmpty;
+  int             nthreads;
+  int             refCount;
+  SList *         queue;
+  pthread_t *     threads;
+} SCommitQueue;
+
+typedef struct {
+  TSDB_REQ_T req;
+  STsdbRepo *pRepo;
+} SReq;
+
+static void *tsdbLoopCommit(void *arg);
+
+static SCommitQueue tsCommitQueue = {0};
+
+int tsdbInitCommitQueue() {
+  int nthreads = tsNumOfCommitThreads;
+  SCommitQueue *pQueue = &tsCommitQueue;
+
+  if (nthreads < 1) nthreads = 1;
+
+  pQueue->stop = false;
+  pQueue->nthreads = nthreads;
+
+  pQueue->queue = tdListNew(0);
+  if (pQueue->queue == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t));
+  if (pQueue->threads == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tdListFree(pQueue->queue);
+    return -1;
+  }
+
+  pthread_mutex_init(&(pQueue->lock), NULL);
+  pthread_cond_init(&(pQueue->queueNotEmpty), NULL);
+
+  for (int i = 0; i < nthreads; i++) {
+    pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL);
+  }
+
+  return 0;
+}
+
+void tsdbDestroyCommitQueue() {
+  SCommitQueue *pQueue = &tsCommitQueue;
+
+  pthread_mutex_lock(&(pQueue->lock));
+
+  if (pQueue->stop) {
+    pthread_mutex_unlock(&(pQueue->lock));
+    return;
+  }
+
+  pQueue->stop = true;
+  pthread_cond_broadcast(&(pQueue->queueNotEmpty));
+
+  pthread_mutex_unlock(&(pQueue->lock));
+
+  for (size_t i = 0; i < pQueue->nthreads; i++) {
+    pthread_join(pQueue->threads[i], NULL);
+  }
+
+  free(pQueue->threads);
+  tdListFree(pQueue->queue);
+  pthread_cond_destroy(&(pQueue->queueNotEmpty));
+  pthread_mutex_destroy(&(pQueue->lock));
+}
+
+int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req) {
+  SCommitQueue *pQueue = &tsCommitQueue;
+
+  SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SReq));
+  if (pNode == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  ((SReq *)pNode->data)->req = req;
+  ((SReq *)pNode->data)->pRepo = pRepo;
+
+  pthread_mutex_lock(&(pQueue->lock));
+
+  // ASSERT(pQueue->stop);
+
+  tdListAppendNode(pQueue->queue, pNode);
+  pthread_cond_signal(&(pQueue->queueNotEmpty));
+
+  pthread_mutex_unlock(&(pQueue->lock));
+  return 0;
+}
+
+static void tsdbApplyRepoConfig(STsdbRepo *pRepo) {
+  pthread_mutex_lock(&pRepo->save_mutex);
+
+  pRepo->config_changed = false;
+  STsdbCfg * pSaveCfg = &pRepo->save_config;
+  STsdbCfg oldCfg;
+  int32_t oldTotalBlocks = pRepo->config.totalBlocks;
+
+  memcpy(&oldCfg, &(pRepo->config), sizeof(STsdbCfg));
+
+  pRepo->config.compression = pRepo->save_config.compression;
+  pRepo->config.keep = pRepo->save_config.keep;
+  pRepo->config.keep1 = pRepo->save_config.keep1;
+  pRepo->config.keep2 = pRepo->save_config.keep2;
+  pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow;
+  pRepo->config.totalBlocks = pRepo->save_config.totalBlocks;
+
+  pthread_mutex_unlock(&pRepo->save_mutex);
+
+  tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d->%d),totalBlocks(%d->%d)",
+    REPO_ID(pRepo),
+    pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2,
+    pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks);
+
+  int err = tsdbExpandPool(pRepo, oldTotalBlocks);
+  if (!TAOS_SUCCEEDED(err)) {
+    tsdbError("vgId:%d expand pool from %d to %d fail,reason:%s",
+      REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err));
+  }
+
+  if (oldCfg.cacheLastRow != pRepo->config.cacheLastRow) {
+    if (tsdbLockRepo(pRepo) < 0) return;
+    // tsdbCacheLastData(pRepo, &oldCfg);
+    // lazy load last cache when query or update
+    ++pRepo->cacheLastConfigVersion;
+    tsdbUnlockRepo(pRepo);
+  }
+
+}
+
+static void *tsdbLoopCommit(void *arg) {
+  SCommitQueue *pQueue = &tsCommitQueue;
+  SListNode *   pNode = NULL;
+  STsdbRepo *   pRepo = NULL;
+  TSDB_REQ_T    req;
+
+  setThreadName("tsdbCommit");
+
+  while (true) {
+    pthread_mutex_lock(&(pQueue->lock));
+
+    while (true) {
+      pNode = tdListPopHead(pQueue->queue);
+      if (pNode == NULL) {
+        if (pQueue->stop && pQueue->refCount <= 0) {
+          pthread_mutex_unlock(&(pQueue->lock));
+          goto _exit;
+        } else {
+          pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock));
+        }
+      } else {
+        break;
+      }
+    }
+
+    pthread_mutex_unlock(&(pQueue->lock));
+
+    req = ((SReq *)pNode->data)->req;
+    pRepo = ((SReq *)pNode->data)->pRepo;
+
+    if (req == COMMIT_REQ) {
+      tsdbCommitData(pRepo);
+    } else if (req == COMPACT_REQ) {
+      tsdbCompactImpl(pRepo);
+    } else if (req == COMMIT_CONFIG_REQ) {        
+      ASSERT(pRepo->config_changed);
+      tsdbApplyRepoConfig(pRepo);
+      tsem_post(&(pRepo->readyToCommit));
+    } else {
+      ASSERT(0);
+    }
+
+    listNodeFree(pNode);
+  }
+
+_exit:
+  return NULL;
+}
+
+void tsdbIncCommitRef(int vgId) {
+  int refCount = atomic_add_fetch_32(&tsCommitQueue.refCount, 1);
+  tsdbDebug("vgId:%d, inc commit queue ref to %d", vgId, refCount);
+}
+
+void tsdbDecCommitRef(int vgId) {
+  int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1);
+  pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty));
+  tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount);
+}

source/dnode/vnode/tsdb2/src/tsdbCompact.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "tsdbint.h"
+
+typedef struct {
+  STable *    pTable;
+  SBlockIdx * pBlkIdx;
+  SBlockIdx   bindex;
+  SBlockInfo *pInfo;
+} STableCompactH;
+
+typedef struct {
+  SRtn       rtn;
+  SFSIter    fsIter;
+  SArray *   tbArray;  // table array to cache table obj and block indexes
+  SReadH     readh;
+  SDFileSet  wSet;
+  SArray *   aBlkIdx;
+  SArray *   aSupBlk;
+  SDataCols *pDataCols;
+} SCompactH;
+
+#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet))
+#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh))
+#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
+#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
+#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
+#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
+#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
+#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh))
+#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh))
+#define TSDB_COMPACT_EXBUF(pComph) TSDB_READ_EXBUF(&((pComph)->readh))
+
+static int  tsdbAsyncCompact(STsdbRepo *pRepo);
+static void tsdbStartCompact(STsdbRepo *pRepo);
+static void tsdbEndCompact(STsdbRepo *pRepo, int eno);
+static int  tsdbCompactMeta(STsdbRepo *pRepo);
+static int  tsdbCompactTSData(STsdbRepo *pRepo);
+static int  tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet);
+static bool tsdbShouldCompact(SCompactH *pComph);
+static int  tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo);
+static void tsdbDestroyCompactH(SCompactH *pComph);
+static int  tsdbInitCompTbArray(SCompactH *pComph);
+static void tsdbDestroyCompTbArray(SCompactH *pComph);
+static int  tsdbCacheFSetIndex(SCompactH *pComph);
+static int  tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet);
+static void tsdbCompactFSetEnd(SCompactH *pComph);
+static int  tsdbCompactFSetImpl(SCompactH *pComph);
+static int  tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
+                                      void **ppCBuf, void **ppExBuf);
+
+enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
+int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); }
+
+void *tsdbCompactImpl(STsdbRepo *pRepo) {
+  // Check if there are files in TSDB FS to compact
+  if (REPO_FS(pRepo)->cstatus->pmf == NULL) {
+    pRepo->compactState = TSDB_NO_COMPACT;
+    tsem_post(&(pRepo->readyToCommit));
+    tsdbInfo("vgId:%d compact over, no file to compact in FS", REPO_ID(pRepo));
+    return NULL;
+  }
+
+  tsdbStartCompact(pRepo);
+
+  if (tsdbCompactMeta(pRepo) < 0) {
+    tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbCompactTSData(pRepo) < 0) {
+    tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS);
+  return NULL;
+
+_err:
+  pRepo->code = terrno;
+  tsdbEndCompact(pRepo, terrno);
+  return NULL;
+}
+
+static int tsdbAsyncCompact(STsdbRepo *pRepo) {
+  if (pRepo->compactState != TSDB_NO_COMPACT) {
+    tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
+    return 0; 
+  } 
+  pRepo->compactState = TSDB_WAITING_COMPACT;   
+  tsem_wait(&(pRepo->readyToCommit));
+  return tsdbScheduleCommit(pRepo, COMPACT_REQ);
+}
+
+static void tsdbStartCompact(STsdbRepo *pRepo) {
+  assert(pRepo->compactState != TSDB_IN_COMPACT);
+  tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
+  tsdbStartFSTxn(pRepo, 0, 0);
+  pRepo->code = TSDB_CODE_SUCCESS;
+  pRepo->compactState = TSDB_IN_COMPACT;
+}
+
+static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
+  if (eno != TSDB_CODE_SUCCESS) {
+    tsdbEndFSTxnWithError(REPO_FS(pRepo));
+  } else {
+    tsdbEndFSTxn(pRepo);
+  }
+  pRepo->compactState = TSDB_NO_COMPACT;
+  tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
+  tsem_post(&(pRepo->readyToCommit));
+}
+
+static int tsdbCompactMeta(STsdbRepo *pRepo) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  tsdbUpdateMFile(pfs, pfs->cstatus->pmf);
+  return 0;
+}
+
+  static int tsdbCompactTSData(STsdbRepo *pRepo) {
+    SCompactH  compactH;
+    SDFileSet *pSet = NULL;
+
+    tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo));
+
+    // If no file, just return 0;
+    if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) {
+      tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo));
+      return 0;
+    }
+
+    if (tsdbInitCompactH(&compactH, pRepo) < 0) {
+      return -1;
+    }
+
+    while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) {
+      // Remove those expired files
+      if (pSet->fid < compactH.rtn.minFid) {
+        tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
+                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+        continue;
+      }
+
+      if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) {
+        tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL);
+        tsdbUpdateDFileSet(REPO_FS(pRepo), pSet);
+        continue;
+      }
+
+      if (tsdbCompactFSet(&compactH, pSet) < 0) {
+        tsdbDestroyCompactH(&compactH);
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        return -1;
+      }
+    }
+
+    tsdbDestroyCompactH(&compactH);
+    tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo));
+    return 0;
+  }
+
+  static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) {
+    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
+    SDiskID    did;
+
+    tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet),
+              TSDB_FSET_ID(pSet));
+
+    if (tsdbCompactFSetInit(pComph, pSet) < 0) {
+      return -1;
+    }
+
+    if (!tsdbShouldCompact(pComph)) {
+      tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid);
+      if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) {
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+    } else {
+      // Create new fset as compacted fset
+      tfsAllocDisk(tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &(did.level), &(did.id));
+      if (did.level == TFS_UNDECIDED_LEVEL) {
+        terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet),
+                      FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_LATEST_FSET_VER);
+      if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) {
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      if (tsdbCompactFSetImpl(pComph) < 0) {
+        tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+        tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+      tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph));
+      tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid);
+    }
+
+    tsdbCompactFSetEnd(pComph);
+    return 0;
+  }
+
+  static bool tsdbShouldCompact(SCompactH *pComph) {
+    if (tsdbForceCompactFile) {
+      return true;
+    }
+    STsdbRepo *     pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg *      pCfg = REPO_CFG(pRepo);
+    SReadH *        pReadh = &(pComph->readh);
+    STableCompactH *pTh;
+    SBlock *        pBlock;
+    int             defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
+    SDFile *        pDataF = TSDB_READ_DATA_FILE(pReadh);
+    SDFile *        pLastF = TSDB_READ_LAST_FILE(pReadh);
+
+    int     tblocks = 0;       // total blocks
+    int     nSubBlocks = 0;    // # of blocks with sub-blocks
+    int     nSmallBlocks = 0;  // # of blocks with rows < defaultRows
+    int64_t tsize = 0;
+
+    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
+      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
+
+      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
+
+      for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) {
+        tblocks++;
+        pBlock = pTh->pInfo->blocks + bidx;
+
+        if (pBlock->numOfRows < defaultRows) {
+          nSmallBlocks++;
+        }
+
+        if (pBlock->numOfSubBlocks > 1) {
+          nSubBlocks++;
+          for (int k = 0; k < pBlock->numOfSubBlocks; k++) {
+            SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k;
+            tsize = tsize + iBlock->len;
+          }
+        } else if (pBlock->numOfSubBlocks == 1) {
+          tsize += pBlock->len;
+        } else {
+          ASSERT(0);
+        }
+      }
+    }
+
+    return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) ||
+            (tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85));
+  }
+
+  static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo) {
+    STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+    memset(pComph, 0, sizeof(*pComph));
+
+    TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph));
+
+    tsdbGetRtnSnap(pRepo, &(pComph->rtn));
+    tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
+
+    if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) {
+      return -1;
+    }
+
+    if (tsdbInitCompTbArray(pComph) < 0) {
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+    if (pComph->aBlkIdx == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
+    if (pComph->aSupBlk == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+    if (pComph->pDataCols == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static void tsdbDestroyCompactH(SCompactH *pComph) {
+    pComph->pDataCols = tdFreeDataCols(pComph->pDataCols);
+    pComph->aSupBlk = taosArrayDestroy(&pComph->aSupBlk);
+    pComph->aBlkIdx = taosArrayDestroy(&pComph->aBlkIdx);
+    tsdbDestroyCompTbArray(pComph);
+    tsdbDestroyReadH(&(pComph->readh));
+    tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+  }
+
+  static int tsdbInitCompTbArray(SCompactH *pComph) {  // Init pComp->tbArray
+    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+    if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
+
+    pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH));
+    if (pComph->tbArray == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbUnlockRepoMeta(pRepo);
+      return -1;
+    }
+
+    // Note here must start from 0
+    for (int i = 0; i < pMeta->maxTables; i++) {
+      STableCompactH ch = {0};
+      if (pMeta->tables[i] != NULL) {
+        tsdbRefTable(pMeta->tables[i]);
+        ch.pTable = pMeta->tables[i];
+      }
+
+      if (taosArrayPush(pComph->tbArray, &ch) == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tsdbUnlockRepoMeta(pRepo);
+        return -1;
+      }
+    }
+
+    if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
+    return 0;
+  }
+
+  static void tsdbDestroyCompTbArray(SCompactH *pComph) {
+    STableCompactH *pTh;
+
+    if (pComph->tbArray == NULL) return;
+
+    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
+      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
+      if (pTh->pTable) {
+        tsdbUnRefTable(pTh->pTable);
+      }
+
+      // pTh->pInfo = taosTZfree(pTh->pInfo);
+      tfree(pTh->pInfo);
+    }
+
+    pComph->tbArray = taosArrayDestroy(&pComph->tbArray);
+  }
+
+  static int tsdbCacheFSetIndex(SCompactH *pComph) {
+    SReadH *pReadH = &(pComph->readh);
+
+    if (tsdbLoadBlockIdx(pReadH) < 0) {
+      return -1;
+    }
+
+    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
+      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
+      pTh->pBlkIdx = NULL;
+
+      if (pTh->pTable == NULL) continue;
+      if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) {
+        return -1;
+      }
+
+      if (pReadH->pBlkIdx == NULL) continue;
+      pTh->bindex = *(pReadH->pBlkIdx);
+      pTh->pBlkIdx = &(pTh->bindex);
+
+      uint32_t originLen = 0;
+      if (tsdbLoadBlockInfo(pReadH, (void **)(&(pTh->pInfo)), &originLen) < 0) {
+        return -1;
+      }
+    }
+
+    return 0;
+  }
+
+  static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) {
+    taosArrayClear(pComph->aBlkIdx);
+    taosArrayClear(pComph->aSupBlk);
+
+    if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) {
+      return -1;
+    }
+
+    if (tsdbCacheFSetIndex(pComph) < 0) {
+      tsdbCloseAndUnsetFSet(&(pComph->readh));
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); }
+
+  static int tsdbCompactFSetImpl(SCompactH *pComph) {
+    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg * pCfg = REPO_CFG(pRepo);
+    SReadH *   pReadh = &(pComph->readh);
+    SBlockIdx  blkIdx;
+    void **    ppBuf = &(TSDB_COMPACT_BUF(pComph));
+    void **    ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph));
+    void **    ppExBuf = &(TSDB_COMPACT_EXBUF(pComph));
+    int        defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
+
+    taosArrayClear(pComph->aBlkIdx);
+
+    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
+      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
+      STSchema *      pSchema;
+
+      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
+
+      pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1, -1);
+      taosArrayClear(pComph->aSupBlk);
+      if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) ||
+          (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tdFreeSchema(pSchema);
+        return -1;
+      }
+      tdFreeSchema(pSchema);
+
+      // Loop to compact each block data
+      for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) {
+        SBlock *pBlock = pTh->pInfo->blocks + i;
+
+        // Load the block data
+        if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) {
+          return -1;
+        }
+
+        // Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file
+        if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) {
+          if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf, ppExBuf) < 0) {
+            return -1;
+          }
+        } else {
+          int ridx = 0;
+
+          while (true) {
+            if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
+            int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
+
+            tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);
+
+            if (pComph->pDataCols->numOfRows < defaultRows) {
+              break;
+            }
+
+            if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf, ppExBuf) < 0) {
+              return -1;
+            }
+            tdResetDataCols(pComph->pDataCols);
+          }
+        }
+      }
+
+      if (pComph->pDataCols->numOfRows > 0 &&
+          tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf, ppExBuf) < 0) {
+        return -1;
+      }
+
+      if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) <
+          0) {
+        return -1;
+      }
+
+      if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        return -1;
+      }
+    }
+
+    if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) {
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
+                                       void **ppCBuf, void **ppExBuf) {
+    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg * pCfg = REPO_CFG(pRepo);
+    SDFile *   pDFile;
+    bool       isLast;
+    SBlock     block;
+
+    ASSERT(pDataCols->numOfRows > 0);
+
+    if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
+      pDFile = TSDB_COMPACT_LAST_FILE(pComph);
+      isLast = true;
+    } else {
+      pDFile = TSDB_COMPACT_DATA_FILE(pComph);
+      isLast = false;
+    }
+
+    if (tsdbWriteBlockImpl(pRepo, pTable, pDFile,
+                           isLast ? TSDB_COMPACT_SMAL_FILE(pComph) : TSDB_COMPACT_SMAD_FILE(pComph), pDataCols, &block,
+                           isLast, true, ppBuf, ppCBuf, ppExBuf) < 0) {
+      return -1;
+    }
+
+    if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    return 0;
+  }
+

source/dnode/vnode/tsdb2/src/tsdbRecover.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/tsdb2/tests/CMakeLists.txt

+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+add_executable(tsdbTests ${SOURCE_LIST})
+target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc)
+
+add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)
\ No newline at end of file