Merge remote-tracking branch 'origin/3.0' into feature/dnode3

CMakeLists.txt

@@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 ## cJson
 cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 
+## leveldb
+option(BUILD_WITH_LEVELDB "If build with leveldb" OFF)
+if(${BUILD_WITH_LEVELDB})
+    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_LEVELDB})
+
+## rocksdb
+option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF)
+if(${BUILD_WITH_ROCKSDB})
+    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_ROCKSDB})
+
 ## download dependencies
 configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

cmake/EnableCMP0048.txt.in

+cmake_policy(SET CMP0048 NEW)
\ No newline at end of file

cmake/leveldb_CMakeLists.txt.in

+
+# leveldb
+ExternalProject_Add(leveldb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
+        GIT_TAG master
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/rocksdb_CMakeLists.txt.in

+
+# rocksdb
+ExternalProject_Add(rocksdb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
+        GIT_TAG v6.23.3
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )
\ No newline at end of file

deps/CMakeLists.txt

+# googletest
 if(${BUILD_TEST})
     add_subdirectory(googletest)
 endif(${BUILD_TEST})
 
+# cJson
+# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
+option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
+option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
+option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
 add_subdirectory(cJson)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
     PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson>
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
+# lz4
 add_subdirectory(lz4/build/cmake)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
 )
 
+# zlib
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
 add_subdirectory(zlib)
 target_include_directories(
     zlib
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
+
+# leveldb
+if(${BUILD_WITH_LEVELDB})
+option(LEVELDB_BUILD_TESTS "" OFF)
+    add_subdirectory(leveldb)
+    target_include_directories(
+        leveldb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
+    )
+endif(${BUILD_WITH_LEVELDB})
+
+# rocksdb
+if(${BUILD_WITH_ROCKSDB})
+    option(WITH_TESTS "" OFF)
+    option(WITH_BENCHMARK_TOOLS "" OFF)
+    option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
+    add_subdirectory(rocksdb)
+    target_include_directories(
+        rocksdb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
+    )
+endif(${BUILD_WITH_ROCKSDB})

include/common/schema.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_COMMON_SCHEMA_H_
-#define _TD_COMMON_SCHEMA_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// ----------------- TSDB COLUMN DEFINITION
-typedef struct {
-  int8_t   type;    // Column type
-  int16_t  colId;   // column ID
-  int16_t  bytes;   // column bytes (restore to int16_t in case of misuse)
-  uint16_t offset;  // point offset in SDataRow after the header part.
-} STColumn;
-
-#define colType(col) ((col)->type)
-#define colColId(col) ((col)->colId)
-#define colBytes(col) ((col)->bytes)
-#define colOffset(col) ((col)->offset)
-
-#define colSetType(col, t) (colType(col) = (t))
-#define colSetColId(col, id) (colColId(col) = (id))
-#define colSetBytes(col, b) (colBytes(col) = (b))
-#define colSetOffset(col, o) (colOffset(col) = (o))
-
-// ----------------- TSDB SCHEMA DEFINITION
-typedef struct {
-  int version;    // version
-  int numOfCols;  // Number of columns appended
-  int tlen;  // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + //
-             // (bytes))
-  uint16_t flen;  // First part length in a SDataRow after the header part
-  uint16_t vlen;  // pure value part length, excluded the overhead (bytes only)
-  STColumn columns[];
-} STSchema;
-
-#define schemaNCols(s) ((s)->numOfCols)
-#define schemaVersion(s) ((s)->version)
-#define schemaTLen(s) ((s)->tlen)
-#define schemaFLen(s) ((s)->flen)
-#define schemaVLen(s) ((s)->vlen)
-#define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s) tfree((s))
-
-STSchema *tdDupSchema(STSchema *pSchema);
-int       tdEncodeSchema(void **buf, STSchema *pSchema);
-void *    tdDecodeSchema(void *buf, STSchema **pRSchema);
-
-static FORCE_INLINE int comparColId(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
-    return 1;
-  } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
-    return -1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
-  void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId);
-  if (ptr == NULL) return NULL;
-  return (STColumn *)ptr;
-}
-
-// ----------------- SCHEMA BUILDER DEFINITION
-typedef struct {
-  int       tCols;
-  int       nCols;
-  int       tlen;
-  uint16_t  flen;
-  uint16_t  vlen;
-  int       version;
-  STColumn *columns;
-} STSchemaBuilder;
-
-int       tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
-void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
-void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
-int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
-STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_COMMON_SCHEMA_H_*/
\ No newline at end of file

include/common/tcompare.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TCOMPARE_H_
+#define _TD_TCOMPARE_H_
+
+#include "compare.h"
+#include "ttypes.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
+int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
+int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TCOMPARE_H_*/
\ No newline at end of file

include/common/tdataformat.h

@@ -87,7 +87,7 @@ typedef struct {
 #define schemaColAt(s, i) ((s)->columns + i)
 #define tdFreeSchema(s) tfree((s))
 
-STSchema *tdDupSchema(STSchema *pSchema);
+STSchema *tdDupSchema(const STSchema *pSchema);
 int       tdEncodeSchema(void **buf, STSchema *pSchema);
 void *    tdDecodeSchema(void *buf, STSchema **pRSchema);
 

include/common/trow.h

@@ -16,10 +16,55 @@
 #ifndef _TD_COMMON_ROW_H_
 #define _TD_COMMON_ROW_H_
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// types
+typedef void *                  SRow;
+typedef struct SRowBatch        SRowBatch;
+typedef struct SRowBuilder      SRowBuilder;
+typedef struct SRowBatchIter    SRowBatchIter;
+typedef struct SRowBatchBuilder SRowBatchBuilder;
+
+// SRow
+#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t))
+#define rowType(r) (*(uint8_t *)(r))                                // row type
+#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t)))  // row length
+#define rowSVer(r) \
+  (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t)))  // row schema version, only for SDataRow
+#define rowNCols(r) rowSVer(r)                                         // only for SKVRow
+#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t)))  // row version
+#define rowCopy(dest, r) memcpy((dest), r, rowLen(r))
+
+static FORCE_INLINE SRow rowDup(SRow row) {
+  SRow r = malloc(rowLen(row));
+  if (r == NULL) {
+    return NULL;
+  }
+
+  rowCopy(r, row);
+
+  return r;
+}
+
+// SRowBatch
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate();
+void         rowBuilderDestroy(SRowBuilder *);
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *);
+void           rowBatchIterDestroy(SRowBatchIter *);
+const SRow     rowBatchIterNext(SRowBatchIter *);
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);
+
 #ifdef __cplusplus
 }
 #endif

include/common/tschema.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_SCHEMA_H_
+#define _TD_COMMON_SCHEMA_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_SCHEMA_H_*/
\ No newline at end of file

include/common/ttypes.h

@@ -6,10 +6,10 @@ extern "C" {
 #endif
 
 #include "taosdef.h"
+#include "types.h"
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
 typedef int32_t  VarDataOffsetT;
-typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
 typedef uint16_t TDRowLenT;    // not including overhead: 0 ~ 65535
 typedef uint32_t TDRowTLenT;   // total length, including overhead
 
@@ -30,11 +30,7 @@ typedef struct {
 } SNCharNullT;
 #pragma pack(pop)
 
-#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
-
-#define varDataLen(v)       ((VarDataLenT *)(v))[0]
 #define varDataTLen(v)      (sizeof(VarDataLenT) + varDataLen(v))
-#define varDataVal(v)       ((void *)((char *)v + VARSTR_HEADER_SIZE))
 #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
 #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))

include/libs/tkv/tkv.h

@@ -16,56 +16,42 @@
 #ifndef _TD_TKV_H_
 #define _TD_TKV_H_
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct tkv_db_s tkv_db_t;
-
-typedef struct {
-  /* data */
-} tkv_key_t;
-
-typedef struct {
-  bool    pinned;
-  int64_t ref;  // TODO: use util library
-                // TODO: add a RW latch here
-  uint64_t offset;
-  void *   pObj;
-} tkv_obj_t;
-
-typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *);
-typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *);
-typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj);
-typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj);
-typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
-typedef void (*tkv_obj_destroy_fn_t)(void *);
-
-typedef struct {
-  uint64_t             memLimit;
-  tkv_get_key_fn_t     getKey;
-  tkv_obj_encode_fn_t  encode;
-  tkv_obj_decode_fn_t  decode;
-  tkv_obj_comp_fn_t    compare;
-  tkv_obj_destroy_fn_t destroy;
-} tkv_db_option_t;
-
-tkv_db_t *       tkvOpenDB(char *dir, tkv_db_option_t *);
-int              tkvCloseDB(tkv_db_t *);
-int              tkvPut(tkv_db_t *, tkv_obj_t *);
-int              tkvPutBatch(tkv_db_t *, tkv_obj_t **, int);  // TODO: use array here
-const tkv_obj_t *tkvGet(tkv_key_t *);
-int              tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **);  // TODO: use array here
-int              tkvDrop(tkv_db_t *, tkv_key_t *);
-int              tkvDropBatch(tkv_db_t *, tkv_key_t **, int);  // TODO: use array here
-int              tkvCommit(tkv_db_t *, void * /*TODO*/);
-
-typedef struct {
-} tkv_db_iter_t;
-
-tkv_db_iter_t *  tkvIterNew(tkv_db_t *);
-void             tkvIterFree(tkv_db_iter_t *);
-const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
+// Types exported
+typedef struct STkvDb        STkvDb;
+typedef struct STkvOpts      STkvOpts;
+typedef struct STkvCache     STkvCache;
+typedef struct STkvReadOpts  STkvReadOpts;
+typedef struct STkvWriteOpts STkvWriteOpts;
+
+// DB operations
+STkvDb *tkvOpen(const STkvOpts *options, const char *path);
+void    tkvClose(STkvDb *db);
+void    tkvPut(STkvDb *db, STkvWriteOpts *, char *key, size_t keylen, char *val, size_t vallen);
+char *  tkvGet(STkvDb *db, STkvReadOpts *, char *key, size_t keylen, size_t *vallen);
+
+// DB options
+STkvOpts *tkvOptionsCreate();
+void      tkvOptionsDestroy(STkvOpts *);
+void      tkvOptionsSetCache(STkvOpts *, STkvCache *);
+
+// DB cache
+typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
+STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
+void       tkvCacheDestroy(STkvCache *);
+
+// STkvReadOpts
+STkvReadOpts *tkvReadOptsCreate();
+void          tkvReadOptsDestroy(STkvReadOpts *);
+
+// STkvWriteOpts
+STkvWriteOpts *tkvWriteOptsCreate();
+void           tkvWriteOptsDestroy(STkvWriteOpts *);
 
 #ifdef __cplusplus
 }

include/os/osDir.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-void taosRemoveDir(char *dirname);
+void taosRemoveDir(const char *dirname);
 bool taosDirExist(char *dirname);
 bool taosMkDir(char *dirname);
 void taosRemoveOldFiles(char *dirname, int32_t keepDays);

include/server/vnode/meta/meta.h

@@ -18,16 +18,53 @@
 
 #include "taosmsg.h"
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef uint64_t tuid_t;
+
+// Types exported
 typedef struct SMeta            SMeta;
+typedef struct SMetaOpts        SMetaOpts;
+typedef struct SMetaQueryHandle SMetaQueryHandle;
+typedef struct SMetaQueryOpts   SMetaQueryOpts;
+typedef struct STableOpts       STableOpts;
+
+// SMeta operations
+int    metaCreate(const char *path);
+void   metaDestroy(const char *path);
+SMeta *metaOpen(SMetaOpts *);
+void   metaClose(SMeta *);
+int    metaCreateTable(SMeta *, STableOpts *);
+int    metaDropTable(SMeta *, uint64_t tuid_t);
+int    metaAlterTable(SMeta *, void *);
+int    metaCommit(SMeta *);
+
+// Options
+SMetaOpts *metaOptionsCreate();
+void       metaOptionsDestroy(SMetaOpts *);
+void       metaOptionsSetCache(SMetaOpts *, size_t capacity);
+
+// SMetaQueryHandle
+SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOpts *);
+void              metaQueryHandleDestroy(SMetaQueryHandle *);
+
+// SMetaQueryOpts
+SMetaQueryOpts *metaQueryOptionsCreate();
+void            metaQueryOptionsDestroy(SMetaQueryOpts *);
+
+// STableOpts
+void metaTableOptsInit(STableOpts *, int8_t type, const char *name, const STSchema *pSchema);
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq);
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq);
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq);
-int metaCommit(SMeta *pMeta);
+/* -------------------------------- Hided implementations -------------------------------- */
+struct STableOpts {
+  int8_t    type;
+  char *    name;
+  STSchema *pSchema;
+};
 
 #ifdef __cplusplus
 }

include/server/vnode/tsdb/tsdb.h

@@ -23,33 +23,60 @@
 extern "C" {
 #endif
 
+// Types exported
 typedef struct STsdb             STsdb;
-typedef struct {
-  int32_t id; // TODO: use a global definition
-  int32_t days;
-  int32_t keep;
-  int32_t keep1;
-  int32_t keep2;
-  int32_t minRows;
-  int32_t maxRows;
-  int8_t  precision;
-  int8_t  update;
-} STsdbCfg;
-
-// Module init and clear
-int tsdbInit();
-int tsdbClear();
-
-// Repository operations
-int    tsdbCreateRepo(int id);
-int    tsdbDropRepo(int id);
-STsdb *tsdbOpenRepo(STsdbCfg *pCfg);
-int    tsdbCloseRepo(STsdb *pTsdb);
-int    tsdbForceCloseRepo(STsdb *pTsdb);
-
-// Data commit
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg);
-int tsdbCommit(STsdb *pTsdb);
+typedef struct STsdbOptions      STsdbOptions;
+typedef struct STsdbSMAOptions   STsdbSMAOptions;  // SMA stands for Small Materialized Aggregation
+typedef struct STsdbReadOptions  STsdbReadOptions;
+typedef struct STsdbSnapshot     STsdbSnapshot;
+typedef struct STsdbQueryHandle  STsdbQueryHandle;
+
+// DB operations
+int    tsdbCreate(const char *path);
+int    tsdbDestroy(const char *path);
+STsdb *tsdbOpen(const STsdbOptions *options);
+void   tsdbClose(STsdb *);
+int    tsdbReset(STsdb *, const STsdbOptions *);
+int    tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *);
+int    tsdbCommit(STsdb *);
+int    tsdbCompact(STsdb *);
+
+// Options
+STsdbOptions *tsdbOptionsCreate();
+void          tsdbOptionsDestroy(STsdbOptions *);
+void          tsdbOptionsSetId(STsdbOptions *, int id);
+void          tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours);
+void          tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2);
+void          tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows);
+void          tsdbOptionsSetPrecision(STsdbOptions *, int);
+void          tsdbOptionsSetCache(STsdbOptions *, int);
+typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType;
+void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType);
+void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *);
+
+// STsdbSMAOptions
+STsdbSMAOptions *tsdbSMAOptionsCreate();
+void             tsdbSMAOptionsDestroy(STsdbSMAOptions *);
+// void             tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/);
+// void             tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *);
+// void             tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *);
+
+// STsdbQueryHandle
+STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *);
+void              tsdbQueryHandleDestroy(STsdbQueryHandle *);
+void              tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *);
+bool              tsdbNextDataBlock(STsdbQueryHandle *);
+// void              tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *);
+// void              tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **);
+
+// STsdbReadOptions
+STsdbReadOptions *tsdbReadOptionsCreate();
+void              tsdbReadOptionsDestroy(STsdbReadOptions *);
+void              tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *);
+
+// STsdbSnapshot
+STsdbSnapshot *tsdbSnapshotCreate(STsdb *);
+void           tsdbSnapshotDestroy(STsdbSnapshot *);
 
 #ifdef __cplusplus
 }

include/util/tcompare.h → include/util/compare.h

@@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte
 
 int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
-
-__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr);
 
 int32_t taosArrayCompareString(const void* a, const void* b);
 
@@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
 
 int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
 int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -110,33 +110,6 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
-#define GET_INT8_VAL(x)    (*(int8_t *)(x))
-#define GET_INT16_VAL(x)   (*(int16_t *)(x))
-#define GET_INT32_VAL(x)   (*(int32_t *)(x))
-#define GET_INT64_VAL(x)   (*(int64_t *)(x))
-#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
-#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
-#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
-#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
-
-#ifdef _TD_ARM_32
-  float  taos_align_get_float(const char* pBuf);
-  double taos_align_get_double(const char* pBuf);
-
-  #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
-  #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
-  #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
-  #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
-  #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
-  #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
-#else
-  #define GET_FLOAT_VAL(x)       (*(float *)(x))
-  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
-  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
-  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
-  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
-  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
-#endif
 
 // TODO: check if below is necessary
 #define TSDB_RELATION_INVALID     0

include/util/types.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TYPES_H_
+#define _TD_TYPES_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GET_INT8_VAL(x)    (*(int8_t *)(x))
+#define GET_INT16_VAL(x)   (*(int16_t *)(x))
+#define GET_INT32_VAL(x)   (*(int32_t *)(x))
+#define GET_INT64_VAL(x)   (*(int64_t *)(x))
+#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
+#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
+#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
+#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
+
+static FORCE_INLINE float taos_align_get_float(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
+#else
+  assert(sizeof(float) == sizeof(uint32_t));
+#endif
+  float fv = 0;
+  memcpy(&fv, pBuf, sizeof(fv));  // in ARM, return *((const float*)(pBuf)) may cause problem
+  return fv;
+}
+
+static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
+#else
+  assert(sizeof(double) == sizeof(uint64_t));
+#endif
+  double dv = 0;
+  memcpy(&dv, pBuf, sizeof(dv));  // in ARM, return *((const double*)(pBuf)) may cause problem
+  return dv;
+}
+
+// #ifdef _TD_ARM_32
+//   float  taos_align_get_float(const char* pBuf);
+//   double taos_align_get_double(const char* pBuf);
+
+//   #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
+//   #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
+//   #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
+//   #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
+//   #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
+//   #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
+// #else
+  #define GET_FLOAT_VAL(x)       (*(float *)(x))
+  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
+  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
+  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
+  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
+// #endif
+
+typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
+#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
+
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
+
+typedef int32_t VarDataOffsetT;
+typedef int16_t VarDataLenT;  // maxVarDataLen: 32767
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TYPES_H_*/
\ No newline at end of file

source/common/src/tcompare.c

@@ -12,334 +12,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _BSD_SOURCE
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _DEFAULT_SOURCE
 
-#include "os.h"
 #include "tcompare.h"
-#include "ulog.h"
-#include "thash.h"
-#include "regex.h"
-#include "ttypes.h"
-
-int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
-}
-
-int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
-}
-
-int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
-}
-
-int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
-}
-
-int32_t compareInt8Val(const void *pLeft, const void *pRight) {
-  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
-  return compareInt8Val(pRight, pLeft);
-}
-
-int32_t compareInt16Val(const void *pLeft, const void *pRight) {
-  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt16Val(pRight, pLeft);
-}
-
-int32_t compareInt32Val(const void *pLeft, const void *pRight) {
-  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt32Val(pRight, pLeft);
-}
-
-int32_t compareInt64Val(const void *pLeft, const void *pRight) {
-  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt64Val(pRight, pLeft);
-}
-
-int32_t compareUint32Val(const void *pLeft, const void *pRight) {
-  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint32Val(pRight, pLeft);
-}
-
-int32_t compareUint64Val(const void *pLeft, const void *pRight) {
-  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint64Val(pRight, pLeft);
-}
-
-int32_t compareUint16Val(const void *pLeft, const void *pRight) {
-  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint16Val(pRight, pLeft);
-}
-
-int32_t compareUint8Val(const void* pLeft, const void* pRight) {
-  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint8Val(pRight, pLeft);
-}
-
-int32_t compareFloatVal(const void *pLeft, const void *pRight) {
-  float p1 = GET_FLOAT_VAL(pLeft);
-  float p2 = GET_FLOAT_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
-  return compareFloatVal(pRight, pLeft);
-}
-
-int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
-  double p1 = GET_DOUBLE_VAL(pLeft);
-  double p2 = GET_DOUBLE_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
-  return compareDoubleVal(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1:-1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedStr(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1 : -1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedWStr(pRight, pLeft);
-}
-
-/*
- * Compare two strings
- *    TSDB_MATCH:            Match
- *    TSDB_NOMATCH:          No match
- *    TSDB_NOWILDCARDMATCH:  No match in spite of having * or % wildcards.
- * Like matching rules:
- *      '%': Matches zero or more characters
- *      '_': Matches one character
- *
- */
-int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
-  char c, c1;
-
-  int32_t i = 0;
-  int32_t j = 0;
-  int32_t o = 0;
-  int32_t m = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == pInfo->matchAll) { /* Match "*" */
-
-      while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
-        if (c == pInfo->matchOne) {
-          if (j > size || str[j++] == 0) {
-            // empty string, return not match
-            return TSDB_PATTERN_NOWILDCARDMATCH;
-          } else {
-            ++o;
-          }
-        }
-      }
-
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
-      }
-
-      char next[3] = {toupper(c), tolower(c), 0};
-      m = o;
-      while (1) {
-        size_t n = strcspn(str + m, next);
-        str += m + n;
-
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-        m = 0;
-      }
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-    ++o; 
-    
-    if (j <= size) {
-      if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
-      if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
-
-int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
-  wchar_t c, c1;
-  wchar_t matchOne = L'_';  // "_"
-  wchar_t matchAll = L'%';  // "%"
-
-  int32_t i = 0;
-  int32_t j = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == matchAll) { /* Match "%" */
-
-      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
-        if (c == matchOne && (j >= size || str[j++] == 0)) {
-          return TSDB_PATTERN_NOWILDCARDMATCH;
-        }
-      }
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH;
-      }
-
-      wchar_t accept[3] = {towupper(c), towlower(c), 0};
-      while (1) {
-        size_t n = wcscspn(str, accept);
-
-        str += n;
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-      }
-
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-
-    if (j <= size) {
-      if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-  
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
 
 int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
@@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
-int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight);
-}
-
-int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
-}
-
-int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
-  size_t sz = varDataLen(pRight);
-  char *pattern = malloc(sz + 1);
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  pattern[sz] = 0;
-
-  sz = varDataLen(pLeft);
-  char *str = malloc(sz + 1);
-  memcpy(str, varDataVal(pLeft), sz);
-  str[sz] = 0;
-
-  int errCode = 0;
-  regex_t regex;
-  char    msgbuf[256] = {0};
-
-  int cflags = REG_EXTENDED;
-  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
-    regfree(&regex);
-    free(str);
-    free(pattern);
-    return 1;
-  }
-
-  errCode = regexec(&regex, str, 0, NULL, 0);
-  if (errCode != 0 && errCode != REG_NOMATCH) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
-  }
-  int32_t result = (errCode == 0) ? 0 : 1;
-  regfree(&regex);
-  free(str);
-  free(pattern);
-  return result;
-}
-
-int32_t taosArrayCompareString(const void* a, const void* b) {
-  const char* x = *(const char**)a;
-  const char* y = *(const char**)b;
-
-  return compareLenPrefixedStr(x, y);
-}
-
-int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
-}
-
 int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 

source/common/src/tdataformat.c

@@ -48,7 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
 /**
  * Duplicate the schema and return a new object
  */
-STSchema *tdDupSchema(STSchema *pSchema) {
+STSchema *tdDupSchema(const STSchema *pSchema) {
 
   int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema);
   STSchema *tSchema = (STSchema *)malloc(tlen);

source/common/src/trow.c

@@ -12,3 +12,78 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "trow.h"
+
+/* ------------ Structures ---------- */
+struct SRowBatch {
+  int32_t compress : 1;  // if batch row is compressed
+  int32_t nrows : 31;    // number of rows
+  int32_t tlen;          // total length (including `nrows` and `tlen`)
+  char    rows[];
+};
+
+struct SRowBuilder {
+  // TODO
+};
+
+struct SRowBatchIter {
+  int32_t    counter;  // row counter
+  SRowBatch *rb;       // row batch to iter
+  SRow       nrow;     // next row
+};
+
+struct SRowBatchBuilder {
+  // TODO
+};
+
+/* ------------ Methods ---------- */
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate() {
+  SRowBuilder *pRowBuilder = NULL;
+  // TODO
+
+  return pRowBuilder;
+}
+
+void rowBuilderDestroy(SRowBuilder *pRowBuilder) {
+  if (pRowBuilder) {
+    free(pRowBuilder);
+  }
+}
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) {
+  SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter));
+  if (pRowBatchIter == NULL) {
+    return NULL;
+  }
+
+  pRowBatchIter->counter = 0;
+  pRowBatchIter->rb = pRowBatch;
+  pRowBatchIter->nrow = pRowBatch->rows;
+
+  return pRowBatchIter;
+};
+
+void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) {
+  if (pRowBatchIter) {
+    free(pRowBatchIter);
+  }
+}
+
+const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) {
+  SRow r = NULL;
+  if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) {
+    r = pRowBatchIter->nrow;
+    pRowBatchIter->counter += 1;
+    pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r));
+  }
+
+  return r;
+}
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);
\ No newline at end of file

source/common/src/ttime.c

@@ -94,8 +94,6 @@ static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t time
   parseLocaltimeWithDst
 };
 
-int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
-
 int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {

source/libs/tkv/CMakeLists.txt

@@ -5,3 +5,7 @@ target_include_directories(
     PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
+target_link_libraries(
+    tkv
+    PUBLIC os
+)
\ No newline at end of file

source/libs/tkv/src/tkv.c

@@ -12,3 +12,70 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "tkv.h"
+
+struct STkvDb {
+  // TODO
+};
+struct STkvOpts {
+  // TODO
+};
+struct STkvCache {
+  // TODO
+};
+struct STkvReadOpts {
+  // TODO
+};
+struct STkvWriteOpts {
+  // TODO
+};
+
+STkvDb *tkvOpen(const STkvOpts *options, const char *path) {
+  // TODO
+  return NULL;
+}
+
+void tkvClose(STkvDb *db) {
+  // TODO
+}
+
+void tkvPut(STkvDb *db, STkvWriteOpts *pwopts, char *key, size_t keylen, char *val, size_t vallen) {
+  // TODO
+}
+
+char *tkvGet(STkvDb *db, STkvReadOpts *propts, char *key, size_t keylen, size_t *vallen) {
+  // TODO
+  return NULL;
+}
+
+STkvOpts *tkvOptionsCreate() {
+  // TODO
+  return NULL;
+}
+
+void tkvOptionsDestroy(STkvOpts *popts) {
+  // TODO
+}
+
+void tkvOptionsSetCache(STkvOpts *popts, STkvCache *pCache) {
+  // TODO
+}
+
+STkvReadOpts *tkvReadOptsCreate() {
+    // TODO
+    return NULL;
+}
+
+void tkvReadOptsDestroy(STkvReadOpts *propts) {
+  // TODO
+}
+
+STkvWriteOpts *tkvWriteOptsCreate() {
+    // TODO
+    return NULL;
+}
+
+void tkvWriteOptsDestroy(STkvWriteOpts *pwopts) {
+  // TODO
+}
\ No newline at end of file

source/os/src/osDir.c

@@ -34,7 +34,7 @@
 #include <unistd.h>
 #include <wordexp.h>
 
-void taosRemoveDir(char *dirname) {
+void taosRemoveDir(const char *dirname) {
   DIR *dir = opendir(dirname);
   if (dir == NULL) return;
 
@@ -48,7 +48,7 @@ void taosRemoveDir(char *dirname) {
       taosRemoveDir(filename);
     } else {
       (void)remove(filename);
-      printf("file:%s is removed", filename);
+      printf("file:%s is removed\n", filename);
     }
   }
 

source/os/src/osTime.c

@@ -63,4 +63,6 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) {
   return gettimeofday(tv, NULL);
 }
 
+int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
+
 #endif

source/server/CMakeLists.txt

@@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC})
 target_link_libraries(
     taosd
     PUBLIC dnode
+    PUBLIC util
 )
\ No newline at end of file

source/server/vnode/CMakeLists.txt

@@ -3,7 +3,7 @@ add_subdirectory(tq)
 add_subdirectory(tsdb)
 
 aux_source_directory(src VNODE_SRC)
-add_library(vnode ${VNODE_SRC})
+add_library(vnode STATIC ${VNODE_SRC})
 target_include_directories(
     vnode
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode"
@@ -18,3 +18,8 @@ target_link_libraries(
     PUBLIC wal
     PUBLIC cjson
 )
+
+# test
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})
\ No newline at end of file

source/server/vnode/inc/vnodeInt.h

@@ -79,7 +79,7 @@ typedef struct {
   void *   cq;  // continuous query
   int32_t  dbCfgVersion;
   int32_t  vgCfgVersion;
-  STsdbCfg tsdbCfg;
+  // STsdbCfg tsdbCfg;
 #if 0  
   SSyncCfg syncCfg;
 #endif  

source/server/vnode/inc/vnodeMemAllocator.h

@@ -16,12 +16,38 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "amalloc.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SVnodeMemAllocator SVnodeMemAllocator;
+
+SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
+                              size_t threshold /* threshold size when full*/);
+void                VMADestroy(SVnodeMemAllocator *pvma);
+void                VMAReset(SVnodeMemAllocator *pvma);
+void *              VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
+void                VMAFree(SVnodeMemAllocator *pvma, void *ptr);
+bool                VMAIsFull(SVnodeMemAllocator *pvma);
+
+// ------------------ FOR TEST ONLY ------------------
+typedef struct SVMANode {
+  struct SVMANode *prev;
+  size_t           tsize;
+  size_t           used;
+  char             data[];
+} SVMANode;
+
+struct SVnodeMemAllocator {
+  bool      full;       // if allocator is full
+  size_t    threshold;  // threshold;
+  size_t    ssize;      // step size to allocate
+  SVMANode *inuse;      // inuse node to allocate
+  SVMANode  node;       // basic node to use
+};
+
 #ifdef __cplusplus
 }
 #endif

source/server/vnode/meta/CMakeLists.txt

 aux_source_directory(src META_SRC)
-add_library(meta ${META_SRC})
+add_library(meta STATIC ${META_SRC})
 target_include_directories(
     meta
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
@@ -8,4 +8,9 @@ target_include_directories(
 target_link_libraries(
     meta
     PUBLIC common
+    PUBLIC tkv
 )
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/server/vnode/meta/inc/metaUid.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_UID_H_
+#define _TD_META_UID_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef uint64_t tb_uid_t;
+tb_uid_t         metaGenerateUid();
+
+#define IVLD_TB_UID 0
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_UID_H_*/
\ No newline at end of file

source/server/vnode/meta/src/meta.c

@@ -13,24 +13,207 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tkv.h"
+#include "thash.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "ttypes.h"
+
 #include "meta.h"
+#include "metaUid.h"
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) {
-  // TODO
-  return 0;
+/* -------------------- Structures -------------------- */
+
+typedef struct STable {
+  tb_uid_t uid;
+  char *   name;
+  tb_uid_t suid;
+  SArray * schema;
+} STable;
+
+typedef struct STableObj {
+  bool     pin;
+  uint64_t ref;
+  SRWLatch latch;
+  uint64_t offset;
+  SList *  ctbList;  //  child table list
+  STable * pTable;
+} STableObj;
+
+struct SMeta {
+  pthread_rwlock_t rwLock;
+
+  SHashObj *pTableObjHash;  // uid --> STableObj
+  SList *   stbList;        // super table list
+  STkvDb *  tbnameDb;       // tbname --> uid
+  STkvDb *  tagDb;          // uid --> tag
+  STkvDb *  schemaDb;
+  STkvDb *  tagIdx;
+  size_t    totalUsed;
+};
+
+static STable *   metaTableNew(tb_uid_t uid, const char *name, int32_t sver);
+static STableObj *metaTableObjNew();
+
+/* -------------------- Methods -------------------- */
+
+SMeta *metaOpen(SMetaOpts *options) {
+  SMeta *pMeta = NULL;
+  char * err = NULL;
+
+  pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
+  if (pMeta == NULL) {
+    return NULL;
+  }
+
+  pthread_rwlock_init(&(pMeta->rwLock), NULL);
+
+  pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+
+  pMeta->stbList = tdListNew(sizeof(STableObj *));
+
+  // Options
+  STkvOpts *dbOptions = tkvOptionsCreate();
+
+  taosMkDir("meta");
+
+  // Open tbname DB
+  pMeta->tbnameDb = tkvOpen(dbOptions, "meta/tbname_uid_db");
+
+  // Open tag DB
+  pMeta->tagDb = tkvOpen(dbOptions, "meta/uid_tag_db");
+
+  // Open schema DB
+  pMeta->schemaDb = tkvOpen(dbOptions, "meta/schema_db");
+
+  // Open tag index
+  pMeta->tagIdx = tkvOpen(dbOptions, "meta/tag_idx_db");
+
+  tkvOptionsDestroy(dbOptions);
+
+  return pMeta;
 }
 
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) {
-  // TODO
-  return 0;
+void metaClose(SMeta *pMeta) {
+  if (pMeta) {
+    tkvClose(pMeta->tagIdx);
+    tkvClose(pMeta->schemaDb);
+    tkvClose(pMeta->tagDb);
+    tkvClose(pMeta->tbnameDb);
+
+    tdListFree(pMeta->stbList);
+    taosHashCleanup(pMeta->pTableObjHash);
+    pthread_rwlock_destroy(&(pMeta->rwLock));
+  }
 }
 
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) {
+int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) {
+  size_t        vallen;
+  STkvReadOpts *ropt;
+  STableObj *   pTableObj = NULL;
+  STkvWriteOpts *wopt;
+
+  // Check if table already exists
+  ropt = tkvReadOptsCreate();
+
+  char *uidStr = tkvGet(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen);
+  if (uidStr != NULL) {
+    // Has duplicate named table
+    return -1;
+  }
+
+  tkvReadOptsDestroy(ropt);
+
+  // Create table obj
+  pTableObj = metaTableObjNew();
+  if (pTableObj == NULL) {
+    // TODO
+    return -1;
+  }
+
+  // Create table object
+  pTableObj->pTable = metaTableNew(metaGenerateUid(), pTableOpts->name, schemaVersion(pTableOpts->pSchema));
+  if (pTableObj->pTable == NULL) {
     // TODO
+  }
+
+  pthread_rwlock_rdlock(&pMeta->rwLock);
+
+  taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj));
+
+  wopt = tkvWriteOptsCreate();
+  // rocksdb_writeoptions_disable_WAL(wopt, 1);
+
+  // Add to tbname db
+  tkvPut(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), (char *)&pTableObj->pTable->uid,
+              sizeof(tb_uid_t));
+
+  // Add to schema db
+  char  id[12];
+  char  buf[256];
+  void *pBuf = buf;
+  *(tb_uid_t *)id = pTableObj->pTable->uid;
+  *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema);
+  int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema);
+
+  tkvPut(pMeta->schemaDb, wopt, id, 12, buf, size);
+
+  tkvWriteOptsDestroy(wopt);
+
+  pthread_rwlock_unlock(&pMeta->rwLock);
+
   return 0;
 }
 
-int metaCommit(SMeta *pMeta) {
+void metaDestroy(const char *path) { taosRemoveDir(path); }
+
+int metaCommit(SMeta *meta) { return 0; }
+
+void metaTableOptsInit(STableOpts *pTableOpts, int8_t type, const char *name, const STSchema *pSchema) {
+  pTableOpts->type = type;
+  pTableOpts->name = strdup(name);
+  pTableOpts->pSchema = tdDupSchema(pSchema);
+}
+
+/* -------------------- Static Methods -------------------- */
+
+static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) {
+  STable *pTable = NULL;
+
+  pTable = (STable *)malloc(sizeof(*pTable));
+  if (pTable == NULL) {
     // TODO
-  return 0;
+    return NULL;
+  }
+
+  pTable->schema = taosArrayInit(0, sizeof(int32_t));
+  if (pTable->schema == NULL) {
+    // TODO
+    return NULL;
+  }
+
+  pTable->uid = uid;
+  pTable->name = strdup(name);
+  pTable->suid = IVLD_TB_UID;
+  taosArrayPush(pTable->schema, &sver);
+
+  return pTable;
+}
+
+static STableObj *metaTableObjNew() {
+  STableObj *pTableObj = NULL;
+
+  pTableObj = (STableObj *)malloc(sizeof(*pTableObj));
+  if (pTableObj == NULL) {
+    return NULL;
+  }
+
+  pTableObj->pin = true;
+  pTableObj->ref = 1;
+  taosInitRWLatch(&(pTableObj->latch));
+  pTableObj->offset = UINT64_MAX;
+  pTableObj->ctbList = NULL;
+  pTableObj->pTable = NULL;
+
+  return pTableObj;
 }
\ No newline at end of file

source/server/vnode/meta/src/metaUid.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaUid.h"
+
+static tb_uid_t nuid = IVLD_TB_UID;
+
+tb_uid_t metaGenerateUid() {
+  // TODO: need a more complex UID generator
+  return ++nuid;
+}
\ No newline at end of file

source/server/vnode/meta/test/CMakeLists.txt

+add_executable(metaTest "")
+target_sources(metaTest
+    PRIVATE
+    "../src/meta.c"
+    "../src/metaUid.c"
+    "metaTests.cpp"
+)
+target_include_directories(metaTest 
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_link_libraries(metaTest
+    os
+    util
+    common
+    gtest_main
+    tkv
+)
+enable_testing()
+add_test(
+    NAME meta_test
+    COMMAND metaTest
+)
\ No newline at end of file

source/server/vnode/meta/test/metaTests.cpp

+#include <gtest/gtest.h>
+#include <string.h>
+#include <iostream>
+
+#include "meta.h"
+
+TEST(MetaTest, meta_open_test) {
+  // Open Meta
+  SMeta *meta = metaOpen(NULL);
+  std::cout << "Meta is opened!" << std::endl;
+
+  // Create tables
+  STableOpts      tbOpts;
+  char            tbname[128];
+  STSchema *      pSchema;
+  STSchemaBuilder sb;
+  tdInitTSchemaBuilder(&sb, 0);
+  for (size_t i = 0; i < 10; i++) {
+    tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, i, 8);
+  }
+  pSchema = tdGetSchemaFromBuilder(&sb);
+  tdDestroyTSchemaBuilder(&sb);
+  for (size_t i = 0; i < 1000000; i++) {
+    sprintf(tbname, "tb%ld", i);
+    metaTableOptsInit(&tbOpts, 0, tbname, pSchema);
+
+    metaCreateTable(meta, &tbOpts);
+  }
+
+  // Close Meta
+  metaClose(meta);
+  std::cout << "Meta is closed!" << std::endl;
+
+  // Destroy Meta
+  metaDestroy("meta");
+  std::cout << "Meta is destroyed!" << std::endl;
+}
\ No newline at end of file

source/server/vnode/src/vnodeMemAllocator.c

@@ -12,3 +12,113 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "vnodeMemAllocator.h"
+
+#define VMA_IS_FULL(pvma) \
+  (((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
+
+static SVMANode *VMANodeNew(size_t size);
+static void      VMANodeFree(SVMANode *node);
+
+SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
+  SVnodeMemAllocator *pvma = NULL;
+
+  if (size < threshold) {
+    return NULL;
+  }
+
+  pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
+  if (pvma) {
+    pvma->full = false;
+    pvma->threshold = threshold;
+    pvma->ssize = ssize;
+    pvma->inuse = &(pvma->node);
+
+    pvma->inuse->prev = NULL;
+    pvma->inuse->tsize = size;
+    pvma->inuse->used = 0;
+  }
+
+  return pvma;
+}
+
+void VMADestroy(SVnodeMemAllocator *pvma) {
+  if (pvma) {
+    VMAReset(pvma);
+    free(pvma);
+  }
+}
+
+void VMAReset(SVnodeMemAllocator *pvma) {
+  while (pvma->inuse != &(pvma->node)) {
+    SVMANode *node = pvma->inuse;
+    pvma->inuse = node->prev;
+    VMANodeFree(node);
+  }
+
+  pvma->inuse->used = 0;
+  pvma->full = false;
+}
+
+void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
+  void * ptr = NULL;
+  size_t tsize = size + sizeof(size_t);
+
+  if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
+    SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
+    if (pNode == NULL) {
+      return NULL;
+    }
+
+    pNode->prev = pvma->inuse;
+    pvma->inuse = pNode;
+  }
+
+  ptr = pvma->inuse->data + pvma->inuse->used;
+  pvma->inuse->used += tsize;
+  *(size_t *)ptr = size;
+  ptr = POINTER_SHIFT(ptr, sizeof(size_t));
+
+  pvma->full = VMA_IS_FULL(pvma);
+
+  return ptr;
+}
+
+void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
+  if (ptr) {
+    size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
+    if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
+      pvma->inuse->used -= (size + sizeof(size_t));
+
+      if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
+        SVMANode *node = pvma->inuse;
+        pvma->inuse = node->prev;
+        VMANodeFree(node);
+      }
+
+      pvma->full = VMA_IS_FULL(pvma);
+    }
+  }
+}
+
+bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
+
+static SVMANode *VMANodeNew(size_t size) {
+  SVMANode *node = NULL;
+
+  node = (SVMANode *)malloc(sizeof(*node) + size);
+  if (node) {
+    node->prev = NULL;
+    node->tsize = size;
+    node->used = 0;
+  }
+
+  return node;
+}
+
+static void VMANodeFree(SVMANode *node) {
+  if (node) {
+    free(node);
+  }
+}
\ No newline at end of file

source/server/vnode/test/CMakeLists.txt

+# vnodeMemAllocatorTest
+add_executable(VMATest "")
+target_sources(VMATest
+    PRIVATE
+    "../src/vnodeMemAllocator.c"
+    "vnodeMemAllocatorTest.cpp"
+)
+target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
+target_link_libraries(VMATest os gtest_main)
\ No newline at end of file

source/server/vnode/test/vnodeMemAllocatorTest.cpp

+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "vnodeMemAllocator.h"
+
+TEST(VMATest, basic_create_and_destroy_test) {
+  SVnodeMemAllocator *vma = VMACreate(1024, 512, 64);
+  EXPECT_TRUE(vma != nullptr);
+  EXPECT_EQ(vma->full, false);
+  EXPECT_EQ(vma->ssize, 512);
+  EXPECT_EQ(vma->threshold, 64);
+  EXPECT_EQ(vma->inuse->tsize, 1024);
+  VMADestroy(vma);
+
+  vma = VMACreate(1024, 512, 1024);
+  EXPECT_TRUE(vma != nullptr);
+  VMADestroy(vma);
+
+  vma = VMACreate(1024, 512, 1025);
+  EXPECT_TRUE(vma == nullptr);
+  VMADestroy(vma);
+}
\ No newline at end of file

source/server/vnode/tsdb/src/tsdb.c

@@ -15,5 +15,5 @@
 
 #include "tsdb.h"
 
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; }
+int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; }
 int tsdbCommit(STsdb *pTsdb) { return 0; }
\ No newline at end of file

source/util/CMakeLists.txt

 aux_source_directory(src UTIL_SRC)
-add_library(util ${UTIL_SRC})
+add_library(util STATIC ${UTIL_SRC})
 target_include_directories(
     util
     PUBLIC "${CMAKE_SOURCE_DIR}/include/util"

source/util/src/compare.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#define _BSD_SOURCE
+#define _GNU_SOURCE
+#define _XOPEN_SOURCE
+#define _DEFAULT_SOURCE
+
+#include "os.h"
+#include "types.h"
+#include "compare.h"
+#include "ulog.h"
+#include "thash.h"
+#include "regex.h"
+
+int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
+}
+
+int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
+}
+
+int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
+}
+
+int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
+}
+
+int32_t compareInt8Val(const void *pLeft, const void *pRight) {
+  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
+  return compareInt8Val(pRight, pLeft);
+}
+
+int32_t compareInt16Val(const void *pLeft, const void *pRight) {
+  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt16Val(pRight, pLeft);
+}
+
+int32_t compareInt32Val(const void *pLeft, const void *pRight) {
+  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt32Val(pRight, pLeft);
+}
+
+int32_t compareInt64Val(const void *pLeft, const void *pRight) {
+  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt64Val(pRight, pLeft);
+}
+
+int32_t compareUint32Val(const void *pLeft, const void *pRight) {
+  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint32Val(pRight, pLeft);
+}
+
+int32_t compareUint64Val(const void *pLeft, const void *pRight) {
+  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint64Val(pRight, pLeft);
+}
+
+int32_t compareUint16Val(const void *pLeft, const void *pRight) {
+  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint16Val(pRight, pLeft);
+}
+
+int32_t compareUint8Val(const void* pLeft, const void* pRight) {
+  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint8Val(pRight, pLeft);
+}
+
+int32_t compareFloatVal(const void *pLeft, const void *pRight) {
+  float p1 = GET_FLOAT_VAL(pLeft);
+  float p2 = GET_FLOAT_VAL(pRight);
+
+  if (isnan(p1) && isnan(p2)) {
+    return 0;
+  }
+
+  if (isnan(p1)) {
+    return -1;
+  }
+
+  if (isnan(p2)) {
+    return 1;
+  }
+  if (FLT_EQUAL(p1, p2)) {
+    return 0;
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
+  return compareFloatVal(pRight, pLeft);
+}
+
+int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
+  double p1 = GET_DOUBLE_VAL(pLeft);
+  double p2 = GET_DOUBLE_VAL(pRight);
+
+  if (isnan(p1) && isnan(p2)) {
+    return 0;
+  }
+
+  if (isnan(p1)) {
+    return -1;
+  }
+
+  if (isnan(p2)) {
+    return 1;
+  }
+  if (FLT_EQUAL(p1, p2)) {
+    return 0;
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
+  return compareDoubleVal(pRight, pLeft);
+}
+
+int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1:-1;
+    }
+  }
+}
+
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedStr(pRight, pLeft);
+}
+
+int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1 : -1;
+    }
+  }
+}
+
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedWStr(pRight, pLeft);
+}
+
+/*
+ * Compare two strings
+ *    TSDB_MATCH:            Match
+ *    TSDB_NOMATCH:          No match
+ *    TSDB_NOWILDCARDMATCH:  No match in spite of having * or % wildcards.
+ * Like matching rules:
+ *      '%': Matches zero or more characters
+ *      '_': Matches one character
+ *
+ */
+int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
+  char c, c1;
+
+  int32_t i = 0;
+  int32_t j = 0;
+  int32_t o = 0;
+  int32_t m = 0;
+
+  while ((c = patterStr[i++]) != 0) {
+    if (c == pInfo->matchAll) { /* Match "*" */
+
+      while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
+        if (c == pInfo->matchOne) {
+          if (j > size || str[j++] == 0) {
+            // empty string, return not match
+            return TSDB_PATTERN_NOWILDCARDMATCH;
+          } else {
+            ++o;
+          }
+        }
+      }
+
+      if (c == 0) {
+        return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
+      }
+
+      char next[3] = {toupper(c), tolower(c), 0};
+      m = o;
+      while (1) {
+        size_t n = strcspn(str + m, next);
+        str += m + n;
+
+        if (str[0] == 0 || (n >= size)) {
+          break;
+        }
+
+        int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
+        if (ret != TSDB_PATTERN_NOMATCH) {
+          return ret;
+        }
+        m = 0;
+      }
+      return TSDB_PATTERN_NOWILDCARDMATCH;
+    }
+
+    c1 = str[j++];
+    ++o; 
+    
+    if (j <= size) {
+      if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
+      if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
+        continue;
+      }
+    }
+
+    return TSDB_PATTERN_NOMATCH;
+  }
+
+  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
+}
+
+int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
+  wchar_t c, c1;
+  wchar_t matchOne = L'_';  // "_"
+  wchar_t matchAll = L'%';  // "%"
+
+  int32_t i = 0;
+  int32_t j = 0;
+
+  while ((c = patterStr[i++]) != 0) {
+    if (c == matchAll) { /* Match "%" */
+
+      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
+        if (c == matchOne && (j >= size || str[j++] == 0)) {
+          return TSDB_PATTERN_NOWILDCARDMATCH;
+        }
+      }
+      if (c == 0) {
+        return TSDB_PATTERN_MATCH;
+      }
+
+      wchar_t accept[3] = {towupper(c), towlower(c), 0};
+      while (1) {
+        size_t n = wcscspn(str, accept);
+
+        str += n;
+        if (str[0] == 0 || (n >= size)) {
+          break;
+        }
+
+        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
+        if (ret != TSDB_PATTERN_NOMATCH) {
+          return ret;
+        }
+      }
+
+      return TSDB_PATTERN_NOWILDCARDMATCH;
+    }
+
+    c1 = str[j++];
+
+    if (j <= size) {
+      if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
+        continue;
+      }
+    }
+
+    return TSDB_PATTERN_NOMATCH;
+  }
+  
+  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
+}
+
+int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
+  return compareStrRegexComp(pLeft, pRight);
+}
+
+int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
+  return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
+}
+
+int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
+  size_t sz = varDataLen(pRight);
+  char *pattern = malloc(sz + 1);
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+  pattern[sz] = 0;
+
+  sz = varDataLen(pLeft);
+  char *str = malloc(sz + 1);
+  memcpy(str, varDataVal(pLeft), sz);
+  str[sz] = 0;
+
+  int errCode = 0;
+  regex_t regex;
+  char    msgbuf[256] = {0};
+
+  int cflags = REG_EXTENDED;
+  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
+    regfree(&regex);
+    free(str);
+    free(pattern);
+    return 1;
+  }
+
+  errCode = regexec(&regex, str, 0, NULL, 0);
+  if (errCode != 0 && errCode != REG_NOMATCH) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
+  }
+  int32_t result = (errCode == 0) ? 0 : 1;
+  regfree(&regex);
+  free(str);
+  free(pattern);
+  return result;
+}
+
+int32_t taosArrayCompareString(const void* a, const void* b) {
+  const char* x = *(const char**)a;
+  const char* y = *(const char**)b;
+
+  return compareLenPrefixedStr(x, y);
+}
+
+int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
+  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+}

source/util/src/thashutil.c

@@ -15,8 +15,9 @@
 
 #include "os.h"
 #include "thash.h"
-#include "tcompare.h"
+#include "compare.h"
 #include "tdef.h"
+#include "types.h"
 
 #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
 

source/util/src/tskiplist.c

@@ -15,7 +15,7 @@
  */
 #include "tskiplist.h"
 #include "os.h"
-#include "tcompare.h"
+#include "compare.h"
 #include "ulog.h"
 #include "tutil.h"
 

source/util/src/tutil.c

@@ -450,24 +450,3 @@ char *taosIpStr(uint32_t ipInt) {
   return ipStr;
 }
 
-FORCE_INLINE float taos_align_get_float(const char* pBuf) {
-#if __STDC_VERSION__ >= 201112L
-  static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
-#else
-  assert(sizeof(float) == sizeof(uint32_t));
-#endif
-  float fv = 0;
-  memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
-  return fv; 
-}
-
-FORCE_INLINE double taos_align_get_double(const char* pBuf) {
-#if __STDC_VERSION__ >= 201112L
-  static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
-#else
-  assert(sizeof(double) == sizeof(uint64_t));
-#endif
-  double dv = 0;
-  memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
-  return dv; 
-}