Merge branch '3.0' of github.com:taosdata/tdengine into 3.0

CMakeLists.txt

@@ -50,6 +50,11 @@ if(${BUILD_WITH_LUCENE})
     cat("${CMAKE_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 endif(${BUILD_WITH_LUCENE})
 
+## NuRaft
+if(${BUILD_WITH_NURAFT})
+    cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_NURAFT})
+
 ## download dependencies
 configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -69,4 +74,7 @@ target_include_directories(api INTERFACE "include/client")
 # src
 add_subdirectory(source)
 
+# docs 
+add_subdirectory(docs)
+
 # tests (TODO)

Jenkinsfile

@@ -142,7 +142,7 @@ pipeline {
           }
       }
   //     stage('Parallel test stage') {
-  //       //only build pr
+  //     skip defaultCheckout 
   //       options { skipDefaultCheckout() } 
   //       when {
   //         allOf{

cmake/cmake.options

@@ -16,7 +16,7 @@ option(
 option(
     BUILD_WITH_ROCKSDB 
     "If build with rocksdb" 
-    OFF
+    ON
 )
 
 option(
@@ -25,8 +25,20 @@ option(
     OFF
 )
 
+option(
+    BUILD_WITH_NURAFT
+    "If build with NuRaft" 
+    OFF
+)
+
 option(
     BUILD_DEPENDENCY_TESTS 
     "If build dependency tests" 
     OFF
+)
+
+option(
+    BUILD_DOCS
+    "If use doxygen build documents" 
+    ON
 )
\ No newline at end of file

cmake/nuraft_CMakeLists.txt.in

+
+# NuRaft
+ExternalProject_Add(NuRaft
+        GIT_REPOSITORY https://github.com/eBay/NuRaft.git
+        GIT_TAG v1.3.0
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/nuraft"
+        BINARY_DIR "${CMAKE_SOURCE_DIR}/deps/nuraft"
+        CONFIGURE_COMMAND "./prepare.sh"
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )
\ No newline at end of file

deps/CMakeLists.txt

@@ -67,6 +67,12 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
+# NuRaft
+if(${BUILD_WITH_NURAFT})
+    add_subdirectory(nuraft)
+endif(${BUILD_WITH_NURAFT})
+
+
 # ================================================================================================
 # DEPENDENCY TEST
 # ================================================================================================

docs/CMakeLists.txt

+# Generate API documentation
+## https://vicrucann.github.io/tutorials/quick-cmake-doxygen/
+if(${BUILD_DOCS})
+    find_package(Doxygen)
+    if (DOXYGEN_FOUND)
+        # Build the doc
+        set(DOXYGEN_IN ${CMAKE_SOURCE_DIR}/docs/Doxyfile.in)
+        set(DOXYGEN_OUT ${CMAKE_BINARY_DIR}/Doxyfile)
+
+        configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
+        message("Doxygen build start")
+
+        add_custom_target(
+            tdengine_doxygen ALL
+            COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            COMMENT "Generating API doxumentation with Doxygen"
+            VERBATIM
+        )
+    else(DOXYGEN_FOUND)
+        message("Doxygen need to be installed to generate the doxygen documentation")
+    endif(DOXYGEN_FOUND)
+endif(${BUILD_DOCS})
\ No newline at end of file

docs/Doxyfile.in

+# Doxyfile 1.8.17
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = @CMAKE_PROJECT_NAME@
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = 3.0
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          =
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO           =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       =
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS         = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES    = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE        = English
+
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION  = None
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       = "The $name class" \
+                         "The $name widget" \
+                         "The $name file" \
+                         is \
+                         provides \
+                         specifies \
+                         contains \
+                         represents \
+                         a \
+                         an \
+                         the
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES        = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH        =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH    =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF      = NO
+
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER         = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF           = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE               = 4
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
+
+ALIASES                =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 5.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS   = 5
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING            = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE      = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL   = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC         = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# (including Cygwin) ands Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES       = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES        = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS               = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC       = NO
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR          = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+INPUT                  = @CMAKE_SOURCE_DIR@/include @CMAKE_SOURCE_DIR@/source
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
+# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
+
+FILE_PATTERNS          = *.c \
+                         *.cc \
+                         *.cxx \
+                         *.cpp \
+                         *.c++ \
+                         *.java \
+                         *.ii \
+                         *.ixx \
+                         *.ipp \
+                         *.i++ \
+                         *.inl \
+                         *.idl \
+                         *.ddl \
+                         *.odl \
+                         *.h \
+                         *.hh \
+                         *.hxx \
+                         *.hpp \
+                         *.h++ \
+                         *.cs \
+                         *.d \
+                         *.php \
+                         *.php4 \
+                         *.php5 \
+                         *.phtml \
+                         *.inc \
+                         *.m \
+                         *.markdown \
+                         *.md \
+                         *.mm \
+                         *.dox \
+                         *.doc \
+                         *.txt \
+                         *.py \
+                         *.pyw \
+                         *.f90 \
+                         *.f95 \
+                         *.f03 \
+                         *.f08 \
+                         *.f \
+                         *.for \
+                         *.tcl \
+                         *.vhd \
+                         *.vhdl \
+                         *.ucf \
+                         *.qsf \
+                         *.ice
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH           =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS       = *
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH             =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES    = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# entity all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = NO
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION    = NO
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see https://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS       = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS          =
+
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
+# were built. This is equivalent to specifying the "-p" option to a clang tool,
+# such as clang-check. These options will then be passed to the parser.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH    =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX     = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX          =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET        =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = NO
+
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via JavaScript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have JavaScript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET        = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE               =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION           =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI           = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING     =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX          = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW      = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH         = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE       = 10
+
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE      =
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using JavaScript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         =
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD    = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER           =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER           =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX           = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE        = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES     = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP        = NO
+
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE    =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE        = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION          = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR             =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT             = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING     = YES
+
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS        = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS         = YES
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT               = YES
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME           = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK               = YES
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH          = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT       = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH               =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS           =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH      =
+
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE      =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH  =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT        = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS      = NO
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP            = YES

include/common/trequest.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TREQUEST_H_
+#define _TD_TREQUEST_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ------------------------ TYPES EXPOSED ------------------------ */
+typedef struct SRequest      SRequest;
+typedef struct SReqBatch     SReqBatch;
+typedef struct SReqBatchIter SReqBatchIter;
+
+// SRequest
+
+// SReqBatch
+
+// SReqBatchIter
+void            tdInitRBIter(SReqBatchIter *pIter, SReqBatch *pReqBatch);
+const SRequest *tdRBIterNext(SReqBatchIter *pIter);
+void            tdClearRBIter(SReqBatchIter *pIter);
+
+/* ------------------------ TYPES DEFINITION ------------------------ */
+struct SReqBatchIter {
+  int        iReq;
+  SReqBatch *pReqBatch;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TREQUEST_H_*/
\ No newline at end of file

include/libs/sync/sync.h

@@ -26,13 +26,13 @@ extern "C" {
 typedef int32_t  SyncNodeId;
 typedef int32_t  SyncGroupId;
 typedef int64_t  SyncIndex;
-typedef uint64_t SSyncTerm;
+typedef uint64_t SyncTerm;
 
 typedef enum {
-  TAOS_SYNC_ROLE_FOLLOWER = 0,
-  TAOS_SYNC_ROLE_CANDIDATE = 1,
-  TAOS_SYNC_ROLE_LEADER = 2,
-} ESyncRole;
+  TAOS_SYNC_STATE_FOLLOWER = 0,
+  TAOS_SYNC_STATE_CANDIDATE = 1,
+  TAOS_SYNC_STATE_LEADER = 2,
+} ESyncState;
 
 typedef struct {
   void*  data;
@@ -55,19 +55,19 @@ typedef struct {
   int32_t   selfIndex;
   int32_t   replica;
   SNodeInfo node[TSDB_MAX_REPLICA];
-  ESyncRole role[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
 } SNodesRole;
 
 typedef struct SSyncFSM {
   void* pData;
 
-  // apply committed log, bufs will be free by raft module
+  // apply committed log, bufs will be free by sync module
   int32_t (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData);
 
   // cluster commit callback
   int32_t (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData);
 
-  // fsm return snapshot in ppBuf, bufs will be free by raft module
+  // fsm return snapshot in ppBuf, bufs will be free by sync module
   // TODO: getSnapshot SHOULD be async?
   int32_t (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int32_t* objId, bool* isLast);
 
@@ -89,19 +89,30 @@ typedef struct SSyncLogStore {
   // write log with given index
   int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf);
 
+  /** 
+   * read log from given index(included) with limit, return the actual num in nBuf,
+   * pBuf will be free in sync module
+   **/
+  int32_t (*logRead)(struct SSyncLogStore* logStore, SyncIndex index, int limit,
+                      SSyncBuffer* pBuf, int* nBuf);
+
   // mark log with given index has been commtted
   int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index);
 
-  // prune log before given index
+  // prune log before given index(not included)
   int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index);
 
-  // rollback log after given index
+  // rollback log after given index(included)
   int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index);
+
+  // return last index of log
+  SyncIndex (*logLastIndex)(struct SSyncLogStore* logStore);
 } SSyncLogStore;
 
 typedef struct SSyncServerState {
   SyncNodeId voteFor;
-  SSyncTerm  term;
+  SyncTerm  term;
+  SyncIndex  commitIndex;
 } SSyncServerState;
 
 typedef struct SSyncClusterConfig {
@@ -122,9 +133,9 @@ typedef struct SStateManager {
 
   int32_t (*readServerState)(struct SStateManager* stateMng, SSyncServerState* state);
 
-  // void (*saveCluster)(struct SStateManager* stateMng, const SSyncClusterConfig* cluster);
+  void (*saveCluster)(struct SStateManager* stateMng, const SSyncClusterConfig* cluster);
 
-  // const SSyncClusterConfig* (*readCluster)(struct SStateManager* stateMng);
+  const SSyncClusterConfig* (*readCluster)(struct SStateManager* stateMng);
 } SStateManager;
 
 typedef struct {
@@ -146,13 +157,13 @@ SSyncNode* syncStart(const SSyncInfo*);
 void       syncReconfig(const SSyncNode*, const SSyncCluster*);
 void       syncStop(const SSyncNode*);
 
-int32_t syncPropose(SSyncNode* syncNode, SSyncBuffer buffer, void* pData, bool isWeak);
+int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak);
 
-// int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode);
+int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode);
 
-// int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode);
+int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode);
 
-extern int32_t syncDebugFlag;
+extern int32_t sDebugFlag;
 
 #ifdef __cplusplus
 }

include/os/osDir.h

@@ -22,7 +22,7 @@ extern "C" {
 
 void taosRemoveDir(const char *dirname);
 bool taosDirExist(char *dirname);
-bool taosMkDir(char *dirname);
+bool taosMkDir(const char *dirname);
 void taosRemoveOldFiles(char *dirname, int32_t keepDays);
 bool taosExpandDir(char *dirname, char *outname, int32_t maxlen);
 bool taosRealPath(char *dirname, int32_t maxlen);

include/util/amalloc.h → include/server/vnode/meta/impl/metaImpl.h

@@ -13,40 +13,63 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_AMALLOC_H_
-#define _TD_AMALLOC_H_
+#ifndef _TD_META_IMPL_H_
+#define _TD_META_IMPL_H_
 
 #include "os.h"
 
+#include "taosmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
+typedef uint64_t tb_uid_t;
+
+/* ------------------------ SMetaOptions ------------------------ */
+struct SMetaOptions {
+  size_t lruCacheSize;  // LRU cache size
+};
+
+/* ------------------------ STbOptions ------------------------ */
+#define META_NORMAL_TABLE ((uint8_t)1)
+#define META_SUPER_TABLE ((uint8_t)2)
+#define META_CHILD_TABLE ((uint8_t)3)
 
-#define AMALLOC_APIS                                  \
-  void *(*malloc)(void *, size_t size);               \
-  void *(*calloc)(void *, size_t nmemb, size_t size); \
-  void *(*realloc)(void *, size_t size);              \
-  void (*free)(void *ptr);
+typedef struct {
+} SSMAOptions;
+
+// super table options
+typedef struct {
+  tb_uid_t  uid;
+  STSchema* pSchema;
+  STSchema* pTagSchema;
+} SSTbOptions;
 
-// Interfaces to implement
+// child table options
 typedef struct {
-  AMALLOC_APIS
-} SMemAllocatorIf;
+  tb_uid_t suid;
+  SKVRow   tags;
+} SCTbOptions;
 
+// normal table options
 typedef struct {
-  void *impl;
-  AMALLOC_APIS
-} SMemAllocator;
+  STSchema* pSchame;
+} SNTbOptions;
 
-#define amalloc(allocator, size) ((allocator) ? (*((allocator)->malloc))((allocator)->impl, (size)) : malloc(size))
-#define acalloc(allocator, nmemb, size) \
-  ((allocator) ? (*((allocator)->calloc))((allocator)->impl, (nmemb), (size)) : calloc((nmemb), (size)))
-#define arealloc(allocator, ptr, size) \
-  ((allocator) ? (*((allocator)->realloc))((allocator)->impl, (ptr), (size)) : realloc((ptr), (size)))
-#define afree(allocator, ptr, size) ((allocator) ? (*((allocator)->free))((allocator)->impl, (ptr), (size)) : free(ptr))
+struct STbOptions {
+  uint8_t     type;
+  char*       name;
+  uint32_t    ttl;   // time to live in (SECONDS)
+  SSMAOptions bsma;  // Block-wise sma
+  union {
+    SSTbOptions stbOptions;
+    SNTbOptions ntbOptions;
+    SCTbOptions ctbOptions;
+  };
+};
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_AMALLOC_H_*/
\ No newline at end of file
+#endif /*_TD_META_IMPL_H_*/
\ No newline at end of file

include/server/vnode/meta/meta.h

@@ -16,81 +16,36 @@
 #ifndef _TD_META_H_
 #define _TD_META_H_
 
-#include "taosmsg.h"
-
-#include "os.h"
+#include "impl/metaImpl.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* ------------------------ APIs Exposed ------------------------ */
-
 // Types exported
-typedef uint64_t                tb_uid_t;
-typedef struct SMeta            SMeta;
-typedef struct SMetaOpts        SMetaOpts;
-typedef struct SMetaQueryHandle SMetaQueryHandle;
-typedef struct SMetaQueryOpts   SMetaQueryOpts;
-typedef struct STableOpts       STableOpts;
+typedef struct SMeta        SMeta;
+typedef struct SMetaOptions SMetaOptions;
+typedef struct STbOptions   STbOptions;
 
 // SMeta operations
-int    metaCreate(const char *path);
-void   metaDestroy(const char *path);
-SMeta *metaOpen(SMetaOpts *);
+SMeta *metaOpen(const char *path, const SMetaOptions *);
 void   metaClose(SMeta *);
-int    metaCreateTable(SMeta *, const STableOpts *);
-int    metaDropTable(SMeta *, uint64_t tuid_t);
-int    metaAlterTable(SMeta *, void *);
+void   metaRemove(const char *path);
+int    metaCreateTable(SMeta *pMeta, const STbOptions *);
+int    metaDropTable(SMeta *pMeta, tb_uid_t uid);
 int    metaCommit(SMeta *);
 
 // Options
-SMetaOpts *metaOptionsCreate();
-void       metaOptionsDestroy(SMetaOpts *);
-void       metaOptionsSetCache(SMetaOpts *, size_t capacity);
-
-// SMetaQueryHandle
-SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOpts *);
-void              metaQueryHandleDestroy(SMetaQueryHandle *);
-
-// SMetaQueryOpts
-SMetaQueryOpts *metaQueryOptionsCreate();
-void            metaQueryOptionsDestroy(SMetaQueryOpts *);
+void metaOptionsInit(SMetaOptions *);
+void metaOptionsClear(SMetaOptions *);
 
 // STableOpts
 #define META_TABLE_OPTS_DECLARE(name) STableOpts name = {0}
-void metaNormalTableOptsInit(STableOpts *, const char *name, const STSchema *pSchema);
-void metaSuperTableOptsInit(STableOpts *, const char *name, tb_uid_t uid, const STSchema *pSchema,
+void metaNormalTableOptsInit(STbOptions *, const char *name, const STSchema *pSchema);
+void metaSuperTableOptsInit(STbOptions *, const char *name, tb_uid_t uid, const STSchema *pSchema,
                             const STSchema *pTagSchema);
-void metaChildTableOptsInit(STableOpts *, const char *name, tb_uid_t suid, const SKVRow tags);
-void metaTableOptsClear(STableOpts *);
-
-/* ------------------------ Impl should hidden ------------------------ */
-typedef enum { META_INIT_TABLE = 0, META_SUPER_TABLE = 1, META_CHILD_TABLE = 2, META_NORMAL_TABLE = 3 } EMetaTableT;
-typedef struct SSuperTableOpts {
-  tb_uid_t  uid;
-  STSchema *pSchema;     // (ts timestamp, a int)
-  STSchema *pTagSchema;  // (tag1 binary(10), tag2 int)
-} SSuperTableOpts;
-
-typedef struct SChildTableOpts {
-  tb_uid_t suid;  // super table uid
-  SKVRow   tags;  // tag value of the child table
-} SChildTableOpts;
-
-typedef struct SNormalTableOpts {
-  STSchema *pSchema;
-} SNormalTableOpts;
-
-struct STableOpts {
-  int8_t type;
-  char * name;
-  union {
-    SSuperTableOpts  superOpts;
-    SChildTableOpts  childOpts;
-    SNormalTableOpts normalOpts;
-  };
-};
+void metaChildTableOptsInit(STbOptions *, const char *name, tb_uid_t suid, const SKVRow tags);
+void metaTableOptsClear(STbOptions *);
 
 #ifdef __cplusplus
 }

include/server/vnode/tsdb/impl/tsdbImpl.h

@@ -23,6 +23,7 @@ extern "C" {
 #endif
 
 struct STsdbOptions {
+  size_t lruCacheSize;
   /* TODO */
 };
 

include/server/vnode/vnode.h

@@ -17,14 +17,80 @@
 #define _TD_VNODE_H_
 
 #include "os.h"
-#include "taosmsg.h"
-#include "trpc.h"
+#include "trequest.h"
+
+#include "meta.h"
+#include "tq.h"
+#include "tsdb.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SVnode SVnode;
+/* ------------------------ TYPES EXPOSED ------------------------ */
+typedef struct SVnode        SVnode;
+typedef struct SVnodeOptions SVnodeOptions;
+
+/* ------------------------ SVnode ------------------------ */
+SVnode *vnodeOpen(const char *path, const SVnodeOptions *pVnodeOptions);
+void    vnodeClose(SVnode *pVnode);
+void    vnodeDestroy(const char *path);
+int     vnodeProcessWriteReqs(SVnode *pVnode, SReqBatch *pReqBatch);
+int     vnodeApplyWriteRequest(SVnode *pVnode, const SRequest *pRequest);
+int     vnodeProcessReadReq(SVnode *pVnode, SRequest *pReq);
+int     vnodeProcessSyncReq(SVnode *pVnode, SRequest *pReq);
+
+/* ------------------------ SVnodeOptions ------------------------ */
+void vnodeOptionsInit(SVnodeOptions *);
+void vnodeOptionsClear(SVnodeOptions *);
+
+/* ------------------------ STRUCT DEFINITIONS ------------------------ */
+struct SVnodeOptions {
+  /**
+   * @brief write buffer size in BYTES
+   *
+   */
+  uint64_t wsize;
+
+  /**
+   * @brief time to live of tables in this vnode
+   * in SECONDS
+   *
+   */
+  uint32_t ttl;
+
+  /**
+   * @brief if time-series requests eventual consistency
+   *
+   */
+  bool isWeak;
+
+  /**
+   * @brief if the allocator is heap allcator or arena allocator
+   *
+   */
+  bool isHeapAllocator;
+
+  /**
+   * @brief TSDB options
+   *
+   */
+  STsdbOptions tsdbOptions;
+
+  /**
+   * @brief META options
+   *
+   */
+  SMetaOptions metaOptions;
+  // STqOptions   tqOptions; // TODO
+};
+
+/* ------------------------ FOR COMPILE ------------------------ */
+
+#if 1
+
+#include "taosmsg.h"
+#include "trpc.h"
 
 typedef struct {
   char     db[TSDB_FULL_DB_NAME_LEN];
@@ -71,8 +137,6 @@ typedef struct {
 int32_t vnodeInit(SVnodePara);
 void    vnodeCleanup();
 
-SVnode *vnodeOpen(int32_t vgId, const char *path);
-void    vnodeClose(SVnode *pVnode);
 int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);
 SVnode *vnodeCreate(int32_t vgId, const char *path, const SVnodeCfg *pCfg);
 void    vnodeDrop(SVnode *pVnode);
@@ -86,6 +150,8 @@ int32_t    vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg);
 void       vnodeCleanupMsg(SVnodeMsg *pMsg);
 void       vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType);
 
+#endif
+
 #ifdef __cplusplus
 }
 #endif

include/util/mallocator.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MALLOCATOR_H_
+#define _TD_MALLOCATOR_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SMemAllocator SMemAllocator;
+
+#define MALLOCATOR_APIS                                        \
+  void *(*malloc)(SMemAllocator *, size_t size);               \
+  void *(*calloc)(SMemAllocator *, size_t nmemb, size_t size); \
+  void *(*realloc)(SMemAllocator *, void *ptr, size_t size);   \
+  void (*free)(SMemAllocator *, void *ptr);                    \
+  size_t (*usage)(SMemAllocator *);
+
+// Interfaces to implement
+typedef struct {
+  MALLOCATOR_APIS
+} SMemAllocatorIf;
+
+struct SMemAllocator {
+  void * impl;
+  size_t usize;
+  MALLOCATOR_APIS
+};
+
+// heap allocator
+SMemAllocator *tdCreateHeapAllocator();
+void           tdDestroyHeapAllocator(SMemAllocator *pMemAllocator);
+
+// arena allocator
+SMemAllocator *tdCreateArenaAllocator(size_t size);
+void           tdDestroyArenaAllocator(SMemAllocator *);
+
+#define mMalloc(pMemAllocator, size) (*(pMemAllocator->malloc))(pMemAllocator, size)
+#define mCalloc(pMemAllocator, nmemb, size) (*(pMemAllocator->calloc))(pMemAllocator, nmemb, size)
+#define mRealloc(pMemAllocator, ptr, size) (*(pMemAllocator->realloc))(pMemAllocator, ptr, size)
+#define mFree(pMemAllocator, ptr) (*(pMemAllocator->free))(pMemAllocator, ptr)
+#define mUsage(pMemAllocator) (*(pMemAllocator->usage))(pMemAllocator)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MALLOCATOR_H_*/
\ No newline at end of file

source/dnode/mgmt/src/dnodeVnodes.c

@@ -376,7 +376,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
 
     char path[PATH_MAX + 20] = {0};
     snprintf(path, sizeof(path),"%s/vnode%d", tsVnodeDir, pVnode->vgId);
-    SVnode *pImpl = vnodeOpen(pVnode->vgId, path);
+    SVnode *pImpl = vnodeOpen(path, NULL);
     if (pImpl == NULL) {
       dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
       pThread->failed++;

source/dnode/vnode/impl/CMakeLists.txt

@@ -7,6 +7,7 @@ target_include_directories(
 )
 target_link_libraries(
     vnode
+    PUBLIC os
     PUBLIC transport
     PUBLIC meta
     PUBLIC tq

source/dnode/vnode/impl/inc/vnodeAllocatorPool.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_ALLOCATOR_POOL_H_
+#define _TD_VNODE_ALLOCATOR_POOL_H_
+
+#include "vnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int            nexta;
+  int            enda;
+  SMemAllocator *free[3];
+  SMemAllocator *used[3];
+} SVAllocatorPool;
+
+int  vnodeOpenAllocatorPool(SVnode *pVnode);
+void vnodeCloseAllocatorPool(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_ALLOCATOR_POOL_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeCommit.h

@@ -16,13 +16,14 @@
 #ifndef _TD_VNODE_COMMIT_H_
 #define _TD_VNODE_COMMIT_H_
 
-#include "vnodeInt.h"
+#include "vnode.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int vnodeAsyncCommit(SVnode *pVnode);
+bool vnodeShouldCommit(SVnode *pVnode);
+int  vnodeAsyncCommit(SVnode *pVnode);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/inc/vnodeDef.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_DEF_H_
+#define _TD_VNODE_DEF_H_
+
+#include "mallocator.h"
+#include "sync.h"
+#include "tlockfree.h"
+
+#include "vnode.h"
+#include "vnodeAllocatorPool.h"
+#include "vnodeCommit.h"
+#include "vnodeFileSystem.h"
+#include "vnodeOptions.h"
+#include "vnodeStateMgr.h"
+#include "vnodeSync.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct SVnode {
+  char*            path;
+  SVnodeOptions    options;
+  SVState          state;
+  SVAllocatorPool* pool;
+  SMemAllocator*   inuse;
+  SMeta*           pMeta;
+  STsdb*           pTsdb;
+  STQ*             pTq;
+  SVnodeSync*      pSync;
+  SVnodeFS*        pFs;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_DEF_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeFileSystem.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_FILE_SYSTEM_H_
+#define _TD_VNODE_FILE_SYSTEM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+} SVnodeFS;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_FILE_SYSTEM_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeInt.h

@@ -18,7 +18,6 @@
 
 #include "vnode.h"
 
-#include "amalloc.h"
 #include "meta.h"
 #include "sync.h"
 #include "tlog.h"
@@ -39,16 +38,6 @@ extern int32_t vDebugFlag;
 #define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
 #define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
 
-typedef struct SVnode {
-  int32_t    vgId;
-  SVnodeCfg  cfg;
-  SMeta     *pMeta;
-  STsdb     *pTsdb;
-  STQ       *pTQ;
-  SWal      *pWal;
-  SSyncNode *pSync;
-} SVnode;
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/impl/inc/vnodeMemAllocator.h

@@ -16,40 +16,12 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "vnodeInt.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SVnodeMemAllocator SVnodeMemAllocator;
-
-SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
-                              size_t threshold /* threshold size when full*/);
-void                VMADestroy(SVnodeMemAllocator *pvma);
-void                VMAReset(SVnodeMemAllocator *pvma);
-void *              VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
-void                VMAFree(SVnodeMemAllocator *pvma, void *ptr);
-bool                VMAIsFull(SVnodeMemAllocator *pvma);
-
-// ------------------ FOR TEST ONLY ------------------
-typedef struct SVMANode {
-  struct SVMANode *prev;
-  size_t           tsize;
-  size_t           used;
-  char             data[];
-} SVMANode;
-
-struct SVnodeMemAllocator {
-  bool      full;       // if allocator is full
-  size_t    threshold;  // threshold;
-  size_t    ssize;      // step size to allocate
-  SVMANode *inuse;      // inuse node to allocate
-  SVMANode  node;       // basic node to use
-};
-
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/
\ No newline at end of file
+#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/

source/dnode/vnode/impl/inc/vnodeOptions.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_OPTIONS_H_
+#define _TD_VNODE_OPTIONS_H_
+
+#include "vnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern const SVnodeOptions defaultVnodeOptions;
+
+int  vnodeValidateOptions(const SVnodeOptions *);
+void vnodeOptionsCopy(SVnodeOptions *pDest, const SVnodeOptions *pSrc);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_OPTIONS_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeRequest.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_REQUEST_H_
+#define _TD_VNODE_REQUEST_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_REQUEST_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeStateMgr.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_STATE_MGR_H_
+#define _TD_VNODE_STATE_MGR_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+} SVState;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_STATE_MGR_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeSync.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_SYNC_H_
+#define _TD_VNODE_SYNC_H_
+
+#include "sync.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  /* data */
+} SVnodeSync;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SYNC_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeWrite.h

@@ -16,10 +16,11 @@
 #ifndef _TD_VNODE_WRITE_H_
 #define _TD_VNODE_WRITE_H_
 
+#include "vnode.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "vnodeInt.h"
 
 void vnodeProcessWriteMsg(SVnode* pVnode, SVnodeMsg* pMsg);
 

source/dnode/vnode/impl/src/vnodeAllocatorPool.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
+
+int vnodeOpenAllocatorPool(SVnode *pVnode) {
+  // TODO
+  return 0;
+}
+
+void vnodeCloseAllocatorPool(SVnode *pVnode) {
+  if (pVnode->pool) {
+  }
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static SVAllocatorPool *vapCreate() {
+  SVAllocatorPool *pPool = NULL;
+  /* TODO */
+  return pPool;
+}
+
+static void vapDestroy() {
+  // TODO
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeCommit.c

@@ -13,13 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnodeDef.h"
 
 static int vnodeStartCommit(SVnode *pVnode);
 static int vnodeEndCommit(SVnode *pVnode);
 
+bool vnodeShouldCommit(SVnode *pVnode) { return false; }
+
 int vnodeAsyncCommit(SVnode *pVnode) {
-  #if 0
+#if 0
   if (vnodeStartCommit(pVnode) < 0) {
     // TODO
   }

source/dnode/vnode/impl/src/vnodeFileSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeInt.c

@@ -20,8 +20,6 @@
 int32_t vnodeInit(SVnodePara para) { return 0; }
 void    vnodeCleanup() {}
 
-SVnode *vnodeOpen(int32_t vgId, const char *path) { return NULL; }
-void    vnodeClose(SVnode *pVnode) {}
 int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; }
 SVnode *vnodeCreate(int32_t vgId, const char *path, const SVnodeCfg *pCfg) { return NULL; }
 void    vnodeDrop(SVnode *pVnode) {}
@@ -69,4 +67,4 @@ void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType) {
     case VN_MSG_TYPE_FETCH:
       break;
   }
-}
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeMain.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
+
+static SVnode *vnodeNew(const char *path, const SVnodeOptions *pVnodeOptions);
+static void    vnodeFree(SVnode *pVnode);
+static int     vnodeOpenImpl(SVnode *pVnode);
+static void    vnodeCloseImpl(SVnode *pVnode);
+
+SVnode *vnodeOpen(const char *path, const SVnodeOptions *pVnodeOptions) {
+  SVnode *pVnode = NULL;
+
+  // Set default options
+  if (pVnodeOptions == NULL) {
+    pVnodeOptions = &defaultVnodeOptions;
+  }
+
+  // Validate options
+  if (vnodeValidateOptions(pVnodeOptions) < 0) {
+    // TODO
+    return NULL;
+  }
+
+  // Create the handle
+  pVnode = vnodeNew(path, pVnodeOptions);
+  if (pVnode == NULL) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  taosMkDir(path);
+
+  // Open the vnode
+  if (vnodeOpenImpl(pVnode) < 0) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  return pVnode;
+}
+
+void vnodeClose(SVnode *pVnode) {
+  if (pVnode) {
+    vnodeCloseImpl(pVnode);
+    vnodeFree(pVnode);
+  }
+}
+
+void vnodeDestroy(const char *path) { taosRemoveDir(path); }
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static SVnode *vnodeNew(const char *path, const SVnodeOptions *pVnodeOptions) {
+  SVnode *pVnode = NULL;
+
+  pVnode = (SVnode *)calloc(1, sizeof(*pVnode));
+  if (pVnode == NULL) {
+    // TODO
+    return NULL;
+  }
+
+  pVnode->path = strdup(path);
+  vnodeOptionsCopy(&(pVnode->options), pVnodeOptions);
+
+  return pVnode;
+}
+
+static void vnodeFree(SVnode *pVnode) {
+  if (pVnode) {
+    tfree(pVnode->path);
+    free(pVnode);
+  }
+}
+
+static int vnodeOpenImpl(SVnode *pVnode) {
+  char dir[TSDB_FILENAME_LEN];
+
+  // Open allocator pool
+  if (vnodeOpenAllocatorPool(pVnode) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  // Open meta
+  sprintf(dir, "%s/meta", pVnode->path);
+  pVnode->pMeta = metaOpen(dir, &(pVnode->options.metaOptions));
+  if (pVnode->pMeta == NULL) {
+    // TODO: handle error
+    return -1;
+  }
+
+  // Open tsdb
+  sprintf(dir, "%s/tsdb", pVnode->path);
+  pVnode->pTsdb = tsdbOpen(dir, &(pVnode->options.tsdbOptions));
+  if (pVnode->pTsdb == NULL) {
+    // TODO: handle error
+    return -1;
+  }
+
+  // TODO: Open TQ
+
+  // TODO
+  return 0;
+}
+
+static void vnodeCloseImpl(SVnode *pVnode) {
+  if (pVnode) {
+    vnodeCloseAllocatorPool(pVnode);
+    // TODO: Close TQ
+    tsdbClose(pVnode->pTsdb);
+    metaClose(pVnode->pMeta);
+  }
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeMemAllocator.c

@@ -13,112 +13,88 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeMemAllocator.h"
-
-#define VMA_IS_FULL(pvma) \
-  (((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
-
-static SVMANode *VMANodeNew(size_t size);
-static void      VMANodeFree(SVMANode *node);
-
-SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
-  SVnodeMemAllocator *pvma = NULL;
+#include "vnodeDef.h"
+
+#define VNODE_HEAP_ALLOCATOR 0
+#define VNODE_ARENA_ALLOCATOR 1
+
+typedef struct {
+  uint64_t tsize;
+  uint64_t used;
+} SVHeapAllocator;
+
+typedef struct SVArenaNode {
+  struct SVArenaNode *prev;
+  void *              nptr;
+  char                data[];
+} SVArenaNode;
+
+typedef struct {
+  SVArenaNode *inuse;
+  SVArenaNode  node;
+} SVArenaAllocator;
+
+typedef struct {
+  int8_t   type;
+  uint64_t tsize;
+  T_REF_DECLARE()
+  union {
+    SVHeapAllocator  vha;
+    SVArenaAllocator vaa;
+  };
+} SVMemAllocator;
+
+SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) {
+  SMemAllocator * pma;
+  uint64_t        msize;
+  SVMemAllocator *pva;
+
+  msize = sizeof(*pma) + sizeof(SVMemAllocator);
+  if (type == VNODE_ARENA_ALLOCATOR) {
+    msize += tsize;
+  }
 
-  if (size < threshold) {
+  pma = (SMemAllocator *)calloc(1, msize);
+  if (pma == NULL) {
     return NULL;
   }
 
-  pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
-  if (pvma) {
-    pvma->full = false;
-    pvma->threshold = threshold;
-    pvma->ssize = ssize;
-    pvma->inuse = &(pvma->node);
-
-    pvma->inuse->prev = NULL;
-    pvma->inuse->tsize = size;
-    pvma->inuse->used = 0;
+  pma->impl = POINTER_SHIFT(pma, sizeof(*pma));
+  pva = (SVMemAllocator *)(pma->impl);
+  pva->type = type;
+  pva->tsize = tsize;
+
+  if (type == VNODE_HEAP_ALLOCATOR) {
+    pma->malloc = NULL;
+    pma->calloc = NULL;
+    pma->realloc = NULL;
+    pma->free = NULL;
+    pma->usage = NULL;
+  } else if (type == VNODE_ARENA_ALLOCATOR) {
+    pma->malloc = NULL;
+    pma->calloc = NULL;
+    pma->realloc = NULL;
+    pma->free = NULL;
+    pma->usage = NULL;
+  } else {
+    ASSERT(0);
   }
 
-  return pvma;
+  return pma;
 }
 
-void VMADestroy(SVnodeMemAllocator *pvma) {
-  if (pvma) {
-    VMAReset(pvma);
-    free(pvma);
-  }
+void vnodeDestroyMemAllocator(SMemAllocator *pma) {
+  // TODO
 }
 
-void VMAReset(SVnodeMemAllocator *pvma) {
-  while (pvma->inuse != &(pvma->node)) {
-    SVMANode *node = pvma->inuse;
-    pvma->inuse = node->prev;
-    VMANodeFree(node);
-  }
-
-  pvma->inuse->used = 0;
-  pvma->full = false;
+void vnodeRefMemAllocator(SMemAllocator *pma) {
+  // TODO
 }
 
-void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
-  void * ptr = NULL;
-  size_t tsize = size + sizeof(size_t);
-
-  if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
-    SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
-    if (pNode == NULL) {
-      return NULL;
-    }
-
-    pNode->prev = pvma->inuse;
-    pvma->inuse = pNode;
-  }
-
-  ptr = pvma->inuse->data + pvma->inuse->used;
-  pvma->inuse->used += tsize;
-  *(size_t *)ptr = size;
-  ptr = POINTER_SHIFT(ptr, sizeof(size_t));
-
-  pvma->full = VMA_IS_FULL(pvma);
-
-  return ptr;
-}
-
-void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
-  if (ptr) {
-    size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
-    if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
-      pvma->inuse->used -= (size + sizeof(size_t));
-
-      if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
-        SVMANode *node = pvma->inuse;
-        pvma->inuse = node->prev;
-        VMANodeFree(node);
-      }
-
-      pvma->full = VMA_IS_FULL(pvma);
-    }
-  }
+void vnodeUnrefMemAllocator(SMemAllocator *pma) {
+  // TODO
 }
 
-bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
+/* ------------------------ Heap Allocator IMPL ------------------------ */
 
-static SVMANode *VMANodeNew(size_t size) {
-  SVMANode *node = NULL;
-
-  node = (SVMANode *)malloc(sizeof(*node) + size);
-  if (node) {
-    node->prev = NULL;
-    node->tsize = size;
-    node->used = 0;
-  }
-
-  return node;
-}
-
-static void VMANodeFree(SVMANode *node) {
-  if (node) {
-    free(node);
-  }
-}
\ No newline at end of file
+/* ------------------------ Arena Allocator IMPL ------------------------ */
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeOptions.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
+
+const SVnodeOptions defaultVnodeOptions = {0}; /* TODO */
+
+void vnodeOptionsInit(SVnodeOptions *pVnodeOptions) { /* TODO */
+  vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions);
+}
+
+void vnodeOptionsClear(SVnodeOptions *pVnodeOptions) { /* TODO */
+}
+
+int vnodeValidateOptions(const SVnodeOptions *pVnodeOptions) {
+  // TODO
+  return 0;
+}
+
+void vnodeOptionsCopy(SVnodeOptions *pDest, const SVnodeOptions *pSrc) {
+  memcpy((void *)pDest, (void *)pSrc, sizeof(SVnodeOptions));
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeRead.c

@@ -13,5 +13,4 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
-#include "vnodeRead.h"
+#include "vnodeDef.h"
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeRequest.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeStateMgr.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeSync.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeWrite.c

@@ -13,5 +13,50 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
-#include "vnodeWrite.h"
+#include "vnodeDef.h"
+
+int vnodeProcessWriteReqs(SVnode *pVnode, SReqBatch *pReqBatch) {
+  /* TODO */
+  return 0;
+}
+
+int vnodeApplyWriteRequest(SVnode *pVnode, const SRequest *pRequest) {
+  int      reqType;        /* TODO */
+  size_t   reqSize;        /* TODO */
+  uint64_t reqVersion = 0; /* TODO */
+  int      code = 0;
+
+  // Copy the request to vnode buffer
+  SRequest *pReq = mMalloc(pVnode->inuse, reqSize);
+  if (pReq == NULL) {
+    // TODO: handle error
+  }
+
+  memcpy(pReq, pRequest, reqSize);
+
+  // Push the request to TQ so consumers can consume
+  tqPushMsg(pVnode->pTq, pReq, 0);
+
+  // Process the request
+  switch (reqType) {
+    case TSDB_MSG_TYPE_CREATE_TABLE:
+      code = metaCreateTable(pVnode->pMeta, NULL /* TODO */);
+      break;
+    case TSDB_MSG_TYPE_DROP_TABLE:
+      code = metaDropTable(pVnode->pMeta, 0 /* TODO */);
+      break;
+      /* TODO */
+    default:
+      break;
+  }
+
+  if (vnodeShouldCommit(pVnode)) {
+    if (vnodeAsyncCommit(pVnode) < 0) {
+      // TODO: handle error
+    }
+  }
+
+  return code;
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
\ No newline at end of file

source/dnode/vnode/impl/test/CMakeLists.txt

-# vnodeMemAllocatorTest
-add_executable(VMATest "")
-target_sources(VMATest
+# Vnode API test
+add_executable(vnodeApiTests "")
+target_sources(vnodeApiTests
     PRIVATE
-    "../src/vnodeMemAllocator.c"
-    "vnodeMemAllocatorTest.cpp"
+    "vnodeApiTests.cpp"
 )
-target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
-target_link_libraries(VMATest os gtest_main vnode)
\ No newline at end of file
+target_link_libraries(vnodeApiTests vnode gtest gtest_main)
\ No newline at end of file

source/dnode/vnode/impl/test/vnodeApiTests.cpp

+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "vnode.h"
+
+TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) {
+  // Create and open a vnode
+  SVnode *pVnode = vnodeOpen("vnode1", NULL);
+  ASSERT_NE(pVnode, nullptr);
+
+  // Close the vnode
+  vnodeClose(pVnode);
+}

source/dnode/vnode/impl/test/vnodeMemAllocatorTest.cpp

-#include <gtest/gtest.h>
-#include <iostream>
-
-#include "vnodeMemAllocator.h"
-
-TEST(VMATest, basic_create_and_destroy_test) {
-  SVnodeMemAllocator *vma = VMACreate(1024, 512, 64);
-  EXPECT_TRUE(vma != nullptr);
-  EXPECT_EQ(vma->full, false);
-  EXPECT_EQ(vma->ssize, 512);
-  EXPECT_EQ(vma->threshold, 64);
-  EXPECT_EQ(vma->inuse->tsize, 1024);
-  VMADestroy(vma);
-
-  vma = VMACreate(1024, 512, 1024);
-  EXPECT_TRUE(vma != nullptr);
-  VMADestroy(vma);
-
-  vma = VMACreate(1024, 512, 1025);
-  EXPECT_TRUE(vma == nullptr);
-  VMADestroy(vma);
-}
\ No newline at end of file

source/dnode/vnode/meta/inc/metaCache.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_CACHE_H_
+#define _TD_META_CACHE_H_
+
+#include "rocksdb/c.h"
+
+#include "meta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef rocksdb_cache_t meta_cache_t;
+
+int  metaOpenCache(SMeta *pMeta);
+void metaCloseCache(SMeta *pMeta);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_CACHE_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaDB.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_DB_H_
+#define _TD_META_DB_H_
+
+#include "rocksdb/c.h"
+
+#include "meta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  rocksdb_t *tbDb;      // uid -> tb obj
+  rocksdb_t *nameDb;    // name -> uid
+  rocksdb_t *tagDb;     // uid -> tag
+  rocksdb_t *schemaDb;  // uid+version -> schema
+  rocksdb_t *mapDb;     // suid -> uid_list
+} meta_db_t;
+
+int  metaOpenDB(SMeta *pMeta);
+void metaCloseDB(SMeta *pMeta);
+int  metaSaveTableToDB(SMeta *pMeta, const STbOptions *pTbOptions);
+int  metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_DB_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaDef.h

@@ -16,22 +16,26 @@
 #ifndef _TD_META_DEF_H_
 #define _TD_META_DEF_H_
 
-#include "metaUid.h"
-#include "tkv.h"
+#include "meta.h"
+#include "metaCache.h"
+#include "metaDB.h"
+#include "metaIdx.h"
+#include "metaOptions.h"
+#include "metaTbOptions.h"
+#include "metaTbTag.h"
+#include "metaTbUid.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct SMeta {
-  STableUidGenerator uidGenerator;
-
-  STkvDb* tableDb;   // uid->table obj
-  STkvDb* tbnameDb;  // tbname --> uid
-  STkvDb* schemaDb;  // uid+version --> schema
-  STkvDb* tagDb;     // uid --> tag
-  STkvDb* tagIdx;    // TODO: need to integrate lucene or our own
-  // STkvCache* metaCache; // TODO: add a global cache here
+  char*           path;     // path of current meta
+  SMetaOptions    options;  // meta option
+  meta_db_t*      pDB;      // raw data db
+  meta_index_t*   pIdx;     // tag index
+  meta_cache_t*   pCache;   // LRU cache
+  STbUidGenerator uidGnrt;  // meta table UID generator
 };
 
 #ifdef __cplusplus

source/dnode/vnode/meta/inc/metaIdx.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_IDX_H_
+#define _TD_META_IDX_H_
+
+#include "rocksdb/c.h"
+
+#include "meta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef rocksdb_t meta_index_t;
+
+int  metaOpenIdx(SMeta *pMeta);
+void metaCloseIdx(SMeta *pMeta);
+int  metaSaveTableToIdx(SMeta *pMeta, const STbOptions *pTbOptions);
+int  metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_IDX_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaOptions.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_OPTIONS_H_
+#define _TD_META_OPTIONS_H_
+
+#include "meta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern const SMetaOptions defaultMetaOptions;
+
+int  metaValidateOptions(const SMetaOptions *);
+void metaOptionsCopy(SMetaOptions *pDest, const SMetaOptions *pSrc);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_OPTIONS_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaQuery.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_QUERY_H_
+#define _TD_META_QUERY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_QUERY_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaTbOptions.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_TABLE_OPTIONS_H_
+#define _TD_META_TABLE_OPTIONS_H_
+
+#include "meta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int    metaValidateTbOptions(SMeta *pMeta, const STbOptions *);
+size_t metaEncodeTbObjFromTbOptions(const STbOptions *, void *pBuf, size_t bsize);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_TABLE_OPTIONS_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaTbTag.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_META_TB_TAG_H_
+#define _TD_META_TB_TAG_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_META_TB_TAG_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaUid.h → source/dnode/vnode/meta/inc/metaTbUid.h

@@ -23,20 +23,17 @@ extern "C" {
 #endif
 
 /* ------------------------ APIS EXPOSED ------------------------ */
-typedef struct STableUidGenerator STableUidGenerator;
-
-// tb_uid_t
-#define IVLD_TB_UID 0
-tb_uid_t generateUid(STableUidGenerator *);
+typedef struct STbUidGenerator {
+  tb_uid_t nextUid;
+} STbUidGenerator;
 
 // STableUidGenerator
-void tableUidGeneratorInit(STableUidGenerator *, tb_uid_t suid);
-#define tableUidGeneratorClear(ug)
+int  metaOpenUidGnrt(SMeta *pMeta);
+void metaCloseUidGnrt(SMeta *pMeta);
 
-/* ------------------------ FOR TEST AND COMPILE ONLY ------------------------ */
-struct STableUidGenerator {
-  tb_uid_t nextUid;
-};
+// tb_uid_t
+#define IVLD_TB_UID 0
+tb_uid_t metaGenerateUid(SMeta *pMeta);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/meta/src/metaCache.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+#include "metaDef.h"
+
+int metaOpenCache(SMeta *pMeta) {
+  // TODO
+  if (pMeta->options.lruCacheSize) {
+    pMeta->pCache = rocksdb_cache_create_lru(pMeta->options.lruCacheSize);
+    if (pMeta->pCache == NULL) {
+      // TODO: handle error
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+void metaCloseCache(SMeta *pMeta) {
+  if (pMeta->pCache) {
+    rocksdb_cache_destroy(pMeta->pCache);
+    pMeta->pCache = NULL;
+  }
+}
\ No newline at end of file

source/dnode/vnode/meta/src/metaCommit.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+int metaCommit(SMeta *pMeta) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/meta/src/metaDB.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema);
+static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion);
+static int  metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid);
+
+#define SCHEMA_KEY_LEN (sizeof(tb_uid_t) + sizeof(int))
+
+#define META_OPEN_DB_IMPL(pDB, options, dir, err) \
+  do {                                            \
+    pDB = rocksdb_open(options, dir, &err);       \
+    if (pDB == NULL) {                            \
+      metaCloseDB(pMeta);                         \
+      rocksdb_options_destroy(options);           \
+      return -1;                                  \
+    }                                             \
+  } while (0)
+
+int metaOpenDB(SMeta *pMeta) {
+  char               dir[128];
+  char *             err = NULL;
+  rocksdb_options_t *options = rocksdb_options_create();
+
+  if (pMeta->pCache) {
+    rocksdb_options_set_row_cache(options, pMeta->pCache);
+  }
+  rocksdb_options_set_create_if_missing(options, 1);
+
+  pMeta->pDB = (meta_db_t *)calloc(1, sizeof(*(pMeta->pDB)));
+  if (pMeta->pDB == NULL) {
+    // TODO: handle error
+    rocksdb_options_destroy(options);
+    return -1;
+  }
+
+  // tbDb
+  sprintf(dir, "%s/tb_db", pMeta->path);
+  META_OPEN_DB_IMPL(pMeta->pDB->tbDb, options, dir, err);
+
+  // nameDb
+  sprintf(dir, "%s/name_db", pMeta->path);
+  META_OPEN_DB_IMPL(pMeta->pDB->nameDb, options, dir, err);
+
+  // tagDb
+  sprintf(dir, "%s/tag_db", pMeta->path);
+  META_OPEN_DB_IMPL(pMeta->pDB->tagDb, options, dir, err);
+
+  // schemaDb
+  sprintf(dir, "%s/schema_db", pMeta->path);
+  META_OPEN_DB_IMPL(pMeta->pDB->schemaDb, options, dir, err);
+
+  // mapDb
+  sprintf(dir, "%s/map_db", pMeta->path);
+  META_OPEN_DB_IMPL(pMeta->pDB->mapDb, options, dir, err);
+
+  rocksdb_options_destroy(options);
+  return 0;
+}
+
+#define META_CLOSE_DB_IMPL(pDB) \
+  do {                          \
+    if (pDB) {                  \
+      rocksdb_close(pDB);       \
+      pDB = NULL;               \
+    }                           \
+  } while (0)
+
+void metaCloseDB(SMeta *pMeta) {
+  if (pMeta->pDB) {
+    META_CLOSE_DB_IMPL(pMeta->pDB->mapDb);
+    META_CLOSE_DB_IMPL(pMeta->pDB->schemaDb);
+    META_CLOSE_DB_IMPL(pMeta->pDB->tagDb);
+    META_CLOSE_DB_IMPL(pMeta->pDB->nameDb);
+    META_CLOSE_DB_IMPL(pMeta->pDB->tbDb);
+    free(pMeta->pDB);
+    pMeta->pDB = NULL;
+  }
+}
+
+int metaSaveTableToDB(SMeta *pMeta, const STbOptions *pTbOptions) {
+  tb_uid_t uid;
+  char *   err = NULL;
+  size_t   size;
+  char     pBuf[1024];  // TODO
+
+  rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
+
+  // Generate a uid for child and normal table
+  if (pTbOptions->type == META_SUPER_TABLE) {
+    uid = pTbOptions->stbOptions.uid;
+  } else {
+    uid = metaGenerateUid(pMeta);
+  }
+
+  // Save tbname -> uid to tbnameDB
+  rocksdb_put(pMeta->pDB->nameDb, wopt, pTbOptions->name, strlen(pTbOptions->name), (char *)(&uid), sizeof(uid), &err);
+
+  // Save uid -> tb_obj to tbDB
+  size = metaEncodeTbObjFromTbOptions(pTbOptions, pBuf, 1024);
+  rocksdb_put(pMeta->pDB->tbDb, wopt, (char *)(&uid), sizeof(uid), pBuf, size, &err);
+
+  switch (pTbOptions->type) {
+    case META_NORMAL_TABLE:
+      // save schemaDB
+      metaSaveSchemaDB(pMeta, uid, pTbOptions->ntbOptions.pSchame);
+      break;
+    case META_SUPER_TABLE:
+      // save schemaDB
+      metaSaveSchemaDB(pMeta, uid, pTbOptions->stbOptions.pSchema);
+
+      // save mapDB (really need?)
+      rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err);
+      break;
+    case META_CHILD_TABLE:
+      // save tagDB
+      rocksdb_put(pMeta->pDB->tagDb, wopt, (char *)(&uid), sizeof(uid), pTbOptions->ctbOptions.tags,
+                  kvRowLen(pTbOptions->ctbOptions.tags), &err);
+
+      // save mapDB
+      metaSaveMapDB(pMeta, pTbOptions->ctbOptions.suid, uid);
+      break;
+    default:
+      ASSERT(0);
+  }
+
+  rocksdb_writeoptions_destroy(wopt);
+
+  return 0;
+}
+
+int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
+  /* TODO */
+  return 0;
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) {
+  char   key[64];
+  char   pBuf[1024];
+  char * ppBuf = pBuf;
+  size_t vsize;
+  char * err = NULL;
+
+  rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
+
+  metaGetSchemaDBKey(key, uid, schemaVersion(pSchema));
+  vsize = tdEncodeSchema((void **)(&ppBuf), pSchema);
+  rocksdb_put(pMeta->pDB->schemaDb, wopt, key, SCHEMA_KEY_LEN, pBuf, vsize, &err);
+
+  rocksdb_writeoptions_destroy(wopt);
+}
+
+static void metaGetSchemaDBKey(char *key, tb_uid_t uid, int sversion) {
+  *(tb_uid_t *)key = uid;
+  *(int *)POINTER_SHIFT(key, sizeof(tb_uid_t)) = sversion;
+}
+
+static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) {
+  size_t vlen;
+  char * val;
+  char * err = NULL;
+
+  rocksdb_readoptions_t *ropt = rocksdb_readoptions_create();
+  val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err);
+  rocksdb_readoptions_destroy(ropt);
+
+  void *nval = malloc(vlen + sizeof(uid));
+  if (nval == NULL) {
+    return -1;
+  }
+
+  if (vlen) {
+    memcpy(nval, val, vlen);
+  }
+  memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid));
+
+  rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
+
+  rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err);
+
+  rocksdb_writeoptions_destroy(wopt);
+
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/meta/src/metaIdx.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+int metaOpenIdx(SMeta *pMeta) {
+  char               idxDir[128];  // TODO
+  char *             err = NULL;
+  rocksdb_options_t *options = rocksdb_options_create();
+
+  // TODO
+  sprintf(idxDir, "%s/index", pMeta->path);
+
+  if (pMeta->pCache) {
+    rocksdb_options_set_row_cache(options, pMeta->pCache);
+  }
+  rocksdb_options_set_create_if_missing(options, 1);
+
+  pMeta->pIdx = rocksdb_open(options, idxDir, &err);
+  if (pMeta->pIdx == NULL) {
+    // TODO: handle error
+    rocksdb_options_destroy(options);
+    return -1;
+  }
+
+  rocksdb_options_destroy(options);
+
+  return 0;
+}
+
+void metaCloseIdx(SMeta *pMeta) { /* TODO */
+  if (pMeta->pIdx) {
+    rocksdb_close(pMeta->pIdx);
+    pMeta->pIdx = NULL;
+  }
+}
+
+int metaSaveTableToIdx(SMeta *pMeta, const STbOptions *pTbOptions) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/meta/src/metaMain.c

@@ -13,75 +13,134 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tcoding.h"
+
 #include "meta.h"
+#include "metaDB.h"
 #include "metaDef.h"
-#include "tcoding.h"
+#include "metaOptions.h"
 
-static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts);
-static int metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts);
-static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts);
+static SMeta *metaNew(const char *path, const SMetaOptions *pMetaOptions);
+static void   metaFree(SMeta *pMeta);
+static int    metaOpenImpl(SMeta *pMeta);
+static void   metaCloseImpl(SMeta *pMeta);
 
-SMeta *metaOpen(SMetaOpts *pMetaOpts) {
+SMeta *metaOpen(const char *path, const SMetaOptions *pMetaOptions) {
   SMeta *pMeta = NULL;
 
-  pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
+  // Set default options
+  if (pMetaOptions == NULL) {
+    pMetaOptions = &defaultMetaOptions;
+  }
+
+  // Validate the options
+  if (metaValidateOptions(pMetaOptions) < 0) {
+    // TODO: deal with error
+    return NULL;
+  }
+
+  // Allocate handle
+  pMeta = metaNew(path, pMetaOptions);
   if (pMeta == NULL) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  // Create META path (TODO)
+  taosMkDir(path);
+
+  // Open meta
+  if (metaOpenImpl(pMeta) < 0) {
+    metaFree(pMeta);
     return NULL;
   }
 
-  // TODO: check if file exists and handle the error
-  taosMkDir("meta");
-
-  // Open tableDb
-  STkvOpts *tableDbOpts = tkvOptsCreate();
-  tkvOptsSetCreateIfMissing(tableDbOpts, 1);
-  pMeta->tableDb = tkvOpen(tableDbOpts, "meta/table_db");
-  tkvOptsDestroy(tableDbOpts);
-
-  // Open tbnameDb
-  STkvOpts *tbnameDbOpts = tkvOptsCreate();
-  tkvOptsSetCreateIfMissing(tbnameDbOpts, 1);
-  pMeta->tbnameDb = tkvOpen(tbnameDbOpts, "meta/tbname_db");
-  tkvOptsDestroy(tbnameDbOpts);
-
-  // Open schemaDb
-  STkvOpts *schemaDbOpts = tkvOptsCreate();
-  tkvOptsSetCreateIfMissing(schemaDbOpts, 1);
-  pMeta->schemaDb = tkvOpen(schemaDbOpts, "meta/schema_db");
-  tkvOptsDestroy(schemaDbOpts);
-
-  // Open tagDb
-  STkvOpts *tagDbOpts = tkvOptsCreate();
-  tkvOptsSetCreateIfMissing(tagDbOpts, 1);
-  pMeta->tagDb = tkvOpen(tagDbOpts, "meta/tag_db");
-  tkvOptsDestroy(tagDbOpts);
-
-  // Open tagIdx
-  STkvOpts *tagIdxDbOpts = tkvOptsCreate();
-  tkvOptsSetCreateIfMissing(tagIdxDbOpts, 1);
-  pMeta->tagIdx = tkvOpen(tagIdxDbOpts, "meta/tag_idx_db");
-  tkvOptsDestroy(tagIdxDbOpts);
-
-  // TODO: need to figure out how to persist the START UID
-  tableUidGeneratorInit(&(pMeta->uidGenerator), IVLD_TB_UID);
   return pMeta;
 }
 
 void metaClose(SMeta *pMeta) {
   if (pMeta) {
-    tableUidGeneratorClear(&pMeta->uidGenerator);
+    metaCloseImpl(pMeta);
+    metaFree(pMeta);
+  }
+}
+
+void metaRemove(const char *path) { taosRemoveDir(path); }
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static SMeta *metaNew(const char *path, const SMetaOptions *pMetaOptions) {
+  SMeta *pMeta;
+  size_t psize = strlen(path);
+
+  pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
+  if (pMeta == NULL) {
+    return NULL;
+  }
 
-    tkvClose(pMeta->tagIdx);
-    tkvClose(pMeta->tagDb);
-    tkvClose(pMeta->schemaDb);
-    tkvClose(pMeta->tbnameDb);
-    tkvClose(pMeta->tableDb);
+  pMeta->path = strdup(path);
+  if (pMeta->path == NULL) {
+    metaFree(pMeta);
+    return NULL;
+  }
 
+  metaOptionsCopy(&(pMeta->options), pMetaOptions);
+
+  return pMeta;
+};
+
+static void metaFree(SMeta *pMeta) {
+  if (pMeta) {
+    tfree(pMeta->path);
     free(pMeta);
   }
 }
 
-int metaCreateTable(SMeta *pMeta, const STableOpts *pTableOpts) {
+static int metaOpenImpl(SMeta *pMeta) {
+  // Open meta cache
+  if (metaOpenCache(pMeta) < 0) {
+    // TODO: handle error
+    metaCloseImpl(pMeta);
+    return -1;
+  }
+
+  // Open meta db
+  if (metaOpenDB(pMeta) < 0) {
+    // TODO: handle error
+    metaCloseImpl(pMeta);
+    return -1;
+  }
+
+  // Open meta index
+  if (metaOpenIdx(pMeta) < 0) {
+    // TODO: handle error
+    metaCloseImpl(pMeta);
+    return -1;
+  }
+
+  // Open meta table uid generator
+  if (metaOpenUidGnrt(pMeta) < 0) {
+    // TODO: handle error
+    metaCloseImpl(pMeta);
+    return -1;
+  }
+
+  return 0;
+}
+
+static void metaCloseImpl(SMeta *pMeta) {
+  metaCloseUidGnrt(pMeta);
+  metaCloseIdx(pMeta);
+  metaCloseDB(pMeta);
+  metaCloseCache(pMeta);
+}
+
+// OLD -------------------------------------------------------------------
+#if 0
+static int    metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts);
+static int    metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts);
+static int    metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts);
+
+int metaCreateTable(SMeta *pMeta, const STableOptions *pTableOpts) {
   size_t vallen;
   char * pUid;
 
@@ -107,7 +166,6 @@ int metaCreateTable(SMeta *pMeta, const STableOpts *pTableOpts) {
   return 0;
 }
 
-/* ------------------------ STATIC METHODS ------------------------ */
 static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts) {
   size_t vallen;
   size_t keylen;
@@ -213,13 +271,13 @@ static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormal
   return 0;
 }
 
-void metaNormalTableOptsInit(STableOpts *pTableOpts, const char *name, const STSchema *pSchema) {
+void metaNormalTableOptsInit(STableOptions *pTableOpts, const char *name, const STSchema *pSchema) {
   pTableOpts->type = META_NORMAL_TABLE;
   pTableOpts->name = strdup(name);
   pTableOpts->normalOpts.pSchema = tdDupSchema(pSchema);
 }
 
-void metaSuperTableOptsInit(STableOpts *pTableOpts, const char *name, tb_uid_t uid, const STSchema *pSchema,
+void metaSuperTableOptsInit(STableOptions *pTableOpts, const char *name, tb_uid_t uid, const STSchema *pSchema,
                             const STSchema *pTagSchema) {
   pTableOpts->type = META_SUPER_TABLE;
   pTableOpts->name = strdup(name);
@@ -228,14 +286,14 @@ void metaSuperTableOptsInit(STableOpts *pTableOpts, const char *name, tb_uid_t u
   pTableOpts->superOpts.pTagSchema = tdDupSchema(pTagSchema);
 }
 
-void metaChildTableOptsInit(STableOpts *pTableOpts, const char *name, tb_uid_t suid, const SKVRow tags) {
+void metaChildTableOptsInit(STableOptions *pTableOpts, const char *name, tb_uid_t suid, const SKVRow tags) {
   pTableOpts->type = META_CHILD_TABLE;
   pTableOpts->name = strdup(name);
   pTableOpts->childOpts.suid = suid;
   pTableOpts->childOpts.tags = tdKVRowDup(tags);
 }
 
-void metaTableOptsClear(STableOpts *pTableOpts) {
+void metaTableOptsClear(STableOptions *pTableOpts) {
   switch (pTableOpts->type) {
     case META_NORMAL_TABLE:
       tfree(pTableOpts->name);
@@ -258,3 +316,5 @@ void metaTableOptsClear(STableOpts *pTableOpts) {
 }
 
 void metaDestroy(const char *path) { taosRemoveDir(path); }
+
+#endif
\ No newline at end of file

source/dnode/vnode/meta/src/metaOptions.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+const SMetaOptions defaultMetaOptions = {.lruCacheSize = 0};
+
+/* ------------------------ EXPOSED METHODS ------------------------ */
+void metaOptionsInit(SMetaOptions *pMetaOptions) { metaOptionsCopy(pMetaOptions, &defaultMetaOptions); }
+
+void metaOptionsClear(SMetaOptions *pMetaOptions) {
+  // TODO
+}
+
+int metaValidateOptions(const SMetaOptions *pMetaOptions) {
+  // TODO
+  return 0;
+}
+
+void metaOptionsCopy(SMetaOptions *pDest, const SMetaOptions *pSrc) { memcpy(pDest, pSrc, sizeof(*pSrc)); }
+
+/* ------------------------ STATIC METHODS ------------------------ */
\ No newline at end of file

source/dnode/vnode/meta/src/metaQuery.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/meta/src/metaTable.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+int metaCreateTable(SMeta *pMeta, const STbOptions *pTbOptions) {
+  // Validate the tbOptions
+  if (metaValidateTbOptions(pMeta, pTbOptions) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  // TODO: add atomicity
+
+  if (metaSaveTableToDB(pMeta, pTbOptions) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  if (metaSaveTableToIdx(pMeta, pTbOptions) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  return 0;
+}
+
+int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
+  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
+    // TODO
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/meta/src/metaTbOptions.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+#include "tcoding.h"
+
+int metaValidateTbOptions(SMeta *pMeta, const STbOptions *pTbOptions) {
+  // TODO
+  return 0;
+}
+
+size_t metaEncodeTbObjFromTbOptions(const STbOptions *pTbOptions, void *pBuf, size_t bsize) {
+  void **ppBuf = &pBuf;
+  int    tlen = 0;
+
+  tlen += taosEncodeFixedU8(ppBuf, pTbOptions->type);
+  tlen += taosEncodeString(ppBuf, pTbOptions->name);
+  tlen += taosEncodeFixedU32(ppBuf, pTbOptions->ttl);
+
+  switch (pTbOptions->type) {
+    case META_SUPER_TABLE:
+      tlen += taosEncodeFixedU64(ppBuf, pTbOptions->stbOptions.uid);
+      tlen += tdEncodeSchema(ppBuf, pTbOptions->stbOptions.pTagSchema);
+      // TODO: encode schema version array
+      break;
+    case META_CHILD_TABLE:
+      tlen += taosEncodeFixedU64(ppBuf, pTbOptions->ctbOptions.suid);
+      break;
+    case META_NORMAL_TABLE:
+      // TODO: encode schema version array
+      break;
+    default:
+      break;
+  }
+
+  return tlen;
+}
\ No newline at end of file

source/dnode/vnode/meta/src/metaTbTag.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/meta/src/metaUid.c → source/dnode/vnode/meta/src/metaTbUid.c

@@ -13,14 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "metaUid.h"
+#include "meta.h"
+#include "metaDef.h"
 
-tb_uid_t generateUid(STableUidGenerator *pGen) {
-  // Generate a new table UID
-  return ++(pGen->nextUid);
+int metaOpenUidGnrt(SMeta *pMeta) {
+  // Init a generator
+  pMeta->uidGnrt.nextUid = IVLD_TB_UID;
+  return 0;
 }
 
-void tableUidGeneratorInit(STableUidGenerator *pGen, tb_uid_t suid) {
-  // Init a generator
-  pGen->nextUid = suid;
+void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */ }
+
+tb_uid_t metaGenerateUid(SMeta *pMeta) {
+  // Generate a new table UID
+  return ++(pMeta->uidGnrt.nextUid);
 }
\ No newline at end of file

source/dnode/vnode/meta/test/CMakeLists.txt

-add_executable(metaTest "")
-target_sources(metaTest
-    PRIVATE
-    "../src/metaMain.c"
-    "../src/metaUid.c"
-    "metaTests.cpp"
-)
-target_include_directories(metaTest 
-    PUBLIC
-    "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_link_libraries(metaTest
-    os
-    util
-    common
-    gtest_main
-    tkv
-)
-enable_testing()
-add_test(
-    NAME meta_test
-    COMMAND metaTest
-)
\ No newline at end of file
+# add_executable(metaTest "")
+# target_sources(metaTest
+#     PRIVATE
+#     "../src/metaMain.c"
+#     "../src/metaUid.c"
+#     "metaTests.cpp"
+# )
+# target_include_directories(metaTest 
+#     PUBLIC
+#     "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
+#     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+# )
+# target_link_libraries(metaTest
+#     os
+#     util
+#     common
+#     gtest_main
+#     tkv
+# )
+# enable_testing()
+# add_test(
+#     NAME meta_test
+#     COMMAND metaTest
+# )
\ No newline at end of file

source/dnode/vnode/meta/test/metaTests.cpp → source/dnode/vnode/meta/test/metaAPITest.cpp

+#if 0
 #include <gtest/gtest.h>
 #include <string.h>
 #include <iostream>
@@ -38,7 +39,7 @@ static SKVRow metaGetSimpleTags() {
 
 TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) {
   // Open Meta
-  SMeta *meta = metaOpen(NULL);
+  SMeta *meta = metaOpen(NULL, NULL);
   std::cout << "Meta is opened!" << std::endl;
 
   // Create 1000000 normal tables
@@ -100,4 +101,5 @@ TEST(MetaTest, meta_create_1m_child_tables_test) {
   // Destroy Meta
   metaDestroy("meta");
   std::cout << "Meta is destroyed!" << std::endl;
-}
\ No newline at end of file
+}
+#endif
\ No newline at end of file

source/dnode/vnode/tq/inc/tqMetaStore.h

@@ -24,18 +24,18 @@
 extern "C" {
 #endif
 
-#define TQ_BUCKET_SIZE 0xFF
+#define TQ_BUCKET_MASK 0xFF
+#define TQ_BUCKET_SIZE 256
+
 #define TQ_PAGE_SIZE 4096
 //key + offset + size
-#define TQ_IDX_ENTRY_SIZE 24
-
-inline static int TqMaxEntryOnePage() { //170
-  return TQ_PAGE_SIZE / TQ_IDX_ENTRY_SIZE;
-}
-
-inline static int TqEmptyTail() { //16
-  return TQ_PAGE_SIZE - TqMaxEntryOnePage();
-}
+#define TQ_IDX_SIZE 24
+//4096 / 24
+#define TQ_MAX_IDX_ONE_PAGE 170
+//24 * 170
+#define TQ_IDX_PAGE_BODY_SIZE 4080
+//4096 - 4080
+#define TQ_IDX_PAGE_HEAD_SIZE 16
 
 #define TQ_ACTION_CONST      0
 #define TQ_ACTION_INUSE      1
@@ -92,19 +92,17 @@ int32_t       tqStoreClose(TqMetaStore*);
 //int32_t       tqStoreDelete(TqMetaStore*);
 //int32_t       TqStoreCommitAll(TqMetaStore*);
 int32_t       tqStorePersist(TqMetaStore*);
+//clean deleted idx and data from persistent file
+int32_t       tqStoreCompact(TqMetaStore*);
 
 void*   tqHandleGet(TqMetaStore*, int64_t key);
 int32_t tqHandleMovePut(TqMetaStore*, int64_t key, void* value);
 int32_t tqHandleCopyPut(TqMetaStore*, int64_t key, void* value, size_t vsize);
-//do commit
-int32_t tqHandleCommit(TqMetaStore*, int64_t key);
-//delete uncommitted
-int32_t tqHandleAbort(TqMetaStore*, int64_t key);
 //delete committed kv pair
 //notice that a delete action still needs to be committed
 int32_t tqHandleDel(TqMetaStore*, int64_t key);
-//delete both committed and uncommitted
-int32_t tqHandleClear(TqMetaStore*, int64_t key);
+int32_t tqHandleCommit(TqMetaStore*, int64_t key);
+int32_t tqHandleAbort(TqMetaStore*, int64_t key);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/tq/src/tq.c

@@ -22,6 +22,9 @@
 //
 //handle management message
 //
+
+int tqGetgHandleSSize(const TqGroupHandle *gHandle);
+
 static int tqProtoCheck(TmqMsgHead *pMsg) {
   return pMsg->protoVer == 0;
 }

source/dnode/vnode/tq/src/tqMetaStore.c

@@ -27,19 +27,46 @@ static int32_t tqHandlePutCommitted(TqMetaStore*, int64_t key, void* value);
 static void*   tqHandleGetUncommitted(TqMetaStore*, int64_t key);
 
 static inline void tqLinkUnpersist(TqMetaStore *pMeta, TqMetaList* pNode) {
-    if(pNode->unpersistNext == NULL) {
-      pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
-      pNode->unpersistPrev = pMeta->unpersistHead;
-      pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
-      pMeta->unpersistHead->unpersistNext = pNode;
-    }
+  if(pNode->unpersistNext == NULL) {
+    pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
+    pNode->unpersistPrev = pMeta->unpersistHead;
+    pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
+    pMeta->unpersistHead->unpersistNext = pNode;
+  }
 }
 
+static inline int tqSeekLastPage(int fd) {
+  int offset = lseek(fd, 0, SEEK_END);
+  int pageNo = offset / TQ_PAGE_SIZE;
+  int curPageOffset = pageNo * TQ_PAGE_SIZE;
+  return lseek(fd, curPageOffset, SEEK_SET);
+}
 
-typedef struct TqMetaPageBuf {
-  int16_t offset;
-  char buffer[TQ_PAGE_SIZE];
-} TqMetaPageBuf;
+//TODO: the struct is tightly coupled with index entry
+typedef struct TqIdxPageHead {
+  int16_t writeOffset;
+  int8_t  unused[14];
+} TqIdxPageHead;
+
+typedef struct TqIdxPageBuf {
+  TqIdxPageHead head;
+  char buffer[TQ_IDX_PAGE_BODY_SIZE];
+} TqIdxPageBuf;
+
+static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
+  int offset = tqSeekLastPage(fd);
+  int nBytes;
+  if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
+    return -1;
+  }
+  if(nBytes == 0) {
+    memset(pBuf, 0, TQ_PAGE_SIZE);
+    pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
+  }
+  ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
+
+  return lseek(fd, offset, SEEK_SET);
+}
 
 TqMetaStore* tqStoreOpen(const char* path,
     int serializer(const void* pObj, TqSerializedHead** ppHead),
@@ -103,27 +130,30 @@ TqMetaStore* tqStoreOpen(const char* path,
   pMeta->deleter = deleter;
 
   //read idx file and load into memory
-  char idxBuf[TQ_PAGE_SIZE];
+  TqIdxPageBuf idxBuf;
   TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
   if(serializedObj == NULL) {
     //TODO:memory insufficient
   }
   int idxRead;
   int allocated = TQ_PAGE_SIZE;
-  while((idxRead = read(idxFd, idxBuf, TQ_PAGE_SIZE))) {
+  bool readEnd = false;
+  while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
     if(idxRead == -1) {
       //TODO: handle error
       ASSERT(false);
     }
+    ASSERT(idxBuf.head.writeOffset == idxRead);
     //loop read every entry
-    for(int i = 0; i < idxRead; i += TQ_IDX_ENTRY_SIZE) {
+    for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
       TqMetaList *pNode = malloc(sizeof(TqMetaList));
       if(pNode == NULL) {
         //TODO: free memory and return error
       }
       memset(pNode, 0, sizeof(TqMetaList));
-      memcpy(&pNode->handle, &idxBuf[i], TQ_IDX_ENTRY_SIZE);
-      lseek(fileFd, pNode->handle.offset, SEEK_CUR);
+      memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
+
+      lseek(fileFd, pNode->handle.offset, SEEK_SET);
       if(allocated < pNode->handle.serializedSize) {
         void *ptr = realloc(serializedObj, pNode->handle.serializedSize);
         if(ptr == NULL) {
@@ -154,9 +184,9 @@ TqMetaStore* tqStoreOpen(const char* path,
         } else {
           pNode->handle.valueInUse = TQ_DELETE_TOKEN;
         }
-        serializedObj = POINTER_SHIFT(serializedObj, serializedObj->ssize);
-        if(serializedObj->ssize != sizeof(TqSerializedHead)) {
-          pMeta->deserializer(serializedObj, &pNode->handle.valueInTxn);
+        TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
+        if(ptr->ssize != sizeof(TqSerializedHead)) {
+          pMeta->deserializer(ptr, &pNode->handle.valueInTxn);
         } else {
           pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
         }
@@ -165,7 +195,7 @@ TqMetaStore* tqStoreOpen(const char* path,
       }
 
       //put into list
-      int bucketKey = pNode->handle.key & TQ_BUCKET_SIZE;
+      int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
       TqMetaList* pBucketNode = pMeta->bucket[bucketKey];
       if(pBucketNode == NULL) {
         pMeta->bucket[bucketKey] = pNode;
@@ -174,15 +204,18 @@ TqMetaStore* tqStoreOpen(const char* path,
         pMeta->bucket[bucketKey] = pNode;
       } else {
         while(pBucketNode->next &&
-            pBucketNode->next->handle.key == pNode->handle.key) {
+            pBucketNode->next->handle.key != pNode->handle.key) {
           pBucketNode = pBucketNode->next; 
         }
         if(pBucketNode->next) {
           ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
-          TqMetaList *pNodeTmp = pBucketNode->next;
-          pBucketNode->next = pNodeTmp->next;
-          pBucketNode = pNodeTmp;
+          TqMetaList *pNodeFound = pBucketNode->next;
+          pNode->next = pNodeFound->next;
+          pBucketNode->next = pNode;
+          pBucketNode = pNodeFound;
         } else {
+          pNode->next = pMeta->bucket[bucketKey];
+          pMeta->bucket[bucketKey] = pNode;
           pBucketNode = NULL;
         }
       }
@@ -264,8 +297,8 @@ int32_t tqStoreDelete(TqMetaStore* pMeta) {
 
 //TODO: wrap in tfile
 int32_t tqStorePersist(TqMetaStore* pMeta) {
-  char writeBuf[TQ_PAGE_SIZE];
-  int64_t* bufPtr = (int64_t*)writeBuf;
+  TqIdxPageBuf idxBuf;
+  int64_t* bufPtr = (int64_t*)idxBuf.buffer;
   TqMetaList *pHead = pMeta->unpersistHead;
   TqMetaList *pNode = pHead->unpersistNext;
   TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead));
@@ -278,6 +311,17 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
   pSHead->ssize = sizeof(TqSerializedHead);
   int allocatedSize = sizeof(TqSerializedHead);
   int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
+
+  tqReadLastPage(pMeta->idxFd, &idxBuf);
+
+  if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
+    lseek(pMeta->idxFd, 0, SEEK_END);
+    memset(&idxBuf, 0, TQ_PAGE_SIZE);
+    idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
+  } else {
+    bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
+  }
+
   while(pHead != pNode) {
     int nBytes = 0;
 
@@ -308,18 +352,23 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
       ASSERT(nBytesTxn == pSHead->ssize);
       nBytes += nBytesTxn;
     }
+    pNode->handle.offset = offset;
+    offset += nBytes;
 
     //write idx file
     //TODO: endian check and convert
     *(bufPtr++) = pNode->handle.key;
     *(bufPtr++) = pNode->handle.offset;
     *(bufPtr++) = (int64_t)nBytes;
-    if((char*)(bufPtr + 3) > writeBuf + TQ_PAGE_SIZE) {
-      nBytes = write(pMeta->idxFd, writeBuf, sizeof(writeBuf));
+    idxBuf.head.writeOffset += TQ_IDX_SIZE;
+
+    if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
+      nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
       //TODO: handle error with tfile
-      ASSERT(nBytes == sizeof(writeBuf));
-      memset(writeBuf, 0, TQ_PAGE_SIZE);
-      bufPtr = (int64_t*)writeBuf;
+      ASSERT(nBytes == TQ_PAGE_SIZE);
+      memset(&idxBuf, 0, TQ_PAGE_SIZE);
+      idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
+      bufPtr = (int64_t*)&idxBuf.buffer;
     }
     //remove from unpersist list
     pHead->unpersistNext = pNode->unpersistNext;
@@ -331,7 +380,7 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
     if(pNode->handle.valueInUse == TQ_DELETE_TOKEN &&
         pNode->handle.valueInTxn == NULL
         ) {
-      int bucketKey = pNode->handle.key & TQ_BUCKET_SIZE;
+      int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
       TqMetaList* pBucketHead = pMeta->bucket[bucketKey];
       if(pBucketHead == pNode) {
         pMeta->bucket[bucketKey] = pNode->next;
@@ -351,11 +400,11 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
 
   //write left bytes
   free(pSHead);
-  if((char*)bufPtr != writeBuf) {
-    int used = (char*)bufPtr - writeBuf;
-    int nBytes = write(pMeta->idxFd, writeBuf, used);
+  //TODO: write new version in tfile
+  if((char*)bufPtr != idxBuf.buffer) {
+    int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
     //TODO: handle error in tfile
-    ASSERT(nBytes == used);
+    ASSERT(nBytes == idxBuf.head.writeOffset);
   }
   //TODO: using fsync in tfile
   fsync(pMeta->idxFd);
@@ -364,7 +413,7 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
 }
 
 static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
@@ -397,11 +446,12 @@ static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value
 }
 
 void* tqHandleGet(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
-      if(pNode->handle.valueInUse != NULL) {
+      if(pNode->handle.valueInUse != NULL
+          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         return pNode->handle.valueInUse;
       } else {
         return NULL;
@@ -414,7 +464,7 @@ void* tqHandleGet(TqMetaStore* pMeta, int64_t key) {
 }
 
 int32_t tqHandleMovePut(TqMetaStore* pMeta, int64_t key, void* value) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
@@ -452,7 +502,7 @@ int32_t tqHandleCopyPut(TqMetaStore* pMeta, int64_t key, void* value, size_t vsi
     return -1;
   }
   memcpy(vmem, value, vsize);
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
@@ -484,7 +534,7 @@ int32_t tqHandleCopyPut(TqMetaStore* pMeta, int64_t key, void* value, size_t vsi
 }
 
 static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
@@ -502,10 +552,13 @@ static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) {
 }
 
 int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
+      if(pNode->handle.valueInTxn == NULL) {
+        return -1;
+      }
       if(pNode->handle.valueInUse
           && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         pMeta->deleter(pNode->handle.valueInUse);
@@ -518,11 +571,11 @@ int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) {
       pNode = pNode->next;
     }
   }
-  return -1;
+  return -2;
 }
 
 int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
     if(pNode->handle.key == key) {
@@ -543,12 +596,13 @@ int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) {
 }
 
 int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
+  int64_t bucketKey = key & TQ_BUCKET_MASK;
   TqMetaList* pNode = pMeta->bucket[bucketKey];
   while(pNode) {
-      if(pNode->handle.valueInTxn
-          && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-      pMeta->deleter(pNode->handle.valueInTxn);
+    if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+      if(pNode->handle.valueInTxn) {
+        pMeta->deleter(pNode->handle.valueInTxn);
+      }
       pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
       tqLinkUnpersist(pMeta, pNode);
       return 0;
@@ -560,34 +614,7 @@ int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) {
   return -1;
 }
 
-int32_t tqHandleClear(TqMetaStore* pMeta, int64_t key) {
-  int64_t bucketKey = key & TQ_BUCKET_SIZE;
-  TqMetaList* pNode = pMeta->bucket[bucketKey];
-  bool exist = false;
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInUse != NULL) {
-        exist = true;
-        if(pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-          pMeta->deleter(pNode->handle.valueInUse);
-        }
-        pNode->handle.valueInUse = TQ_DELETE_TOKEN;
-      }
-      if(pNode->handle.valueInTxn != NULL) {
-        exist = true;
-        if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-          pMeta->deleter(pNode->handle.valueInTxn);
-        }
-        pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
-      }
-      if(exist) {
-        tqLinkUnpersist(pMeta, pNode);
-        return 0;
-      }
-      return -1;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  return -2;
+//TODO: clean deleted idx and data from persistent file
+int32_t tqStoreCompact(TqMetaStore *pMeta) {
+  return 0;
 }

source/dnode/vnode/tq/test/tqMetaTest.cpp

@@ -57,6 +57,10 @@ TEST_F(TqMetaTest, copyPutTest) {
 
   Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1);
   EXPECT_EQ(pFoo == NULL, true);
+
+  tqHandleCommit(pMeta, 1);
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo->a, 3);
 }
 
 TEST_F(TqMetaTest, persistTest) {
@@ -82,8 +86,6 @@ TEST_F(TqMetaTest, persistTest) {
 
   pBar = (Foo*)tqHandleGet(pMeta, 2);
   EXPECT_EQ(pBar == NULL, true);
-
-  //taosRemoveDir(pathName);
 }
 
 TEST_F(TqMetaTest, uncommittedTest) {
@@ -130,4 +132,163 @@ TEST_F(TqMetaTest, deleteTest) {
   tqHandleCommit(pMeta, 1);
   pFoo = (Foo*) tqHandleGet(pMeta, 1);
   EXPECT_EQ(pFoo == NULL, true);
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+}
+
+TEST_F(TqMetaTest, intxnPersist) {
+  Foo* pFoo = (Foo*)malloc(sizeof(Foo));
+  pFoo->a = 3;
+  tqHandleMovePut(pMeta, 1, pFoo);
+  tqHandleCommit(pMeta, 1);
+
+  Foo* pBar = (Foo*)malloc(sizeof(Foo));
+  pBar->a = 4;
+  tqHandleMovePut(pMeta, 1, pBar);
+
+  Foo* pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo1->a, 3);
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+
+  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo1->a, 3);
+
+  tqHandleCommit(pMeta, 1);
+
+  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo1->a, 4);
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+
+  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo1->a, 4);
+}
+
+TEST_F(TqMetaTest, multiplePage) {
+  srand(0);
+  std::vector<int> v;
+  for(int i = 0; i < 1000; i++) {
+    v.push_back(rand());
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+  }
+  for(int i = 0; i < 500; i++) {
+    tqHandleCommit(pMeta, i);
+    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
+    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
+    EXPECT_EQ(pFoo->a, v[i]);
+  }
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+  
+  for(int i = 500; i < 1000; i++) {
+    tqHandleCommit(pMeta, i);
+    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
+    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
+    EXPECT_EQ(pFoo->a, v[i]);
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
+    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
+    EXPECT_EQ(pFoo->a, v[i]);
+  }
+
+}
+
+TEST_F(TqMetaTest, multipleRewrite) {
+  srand(0);
+  std::vector<int> v;
+  for(int i = 0; i < 1000; i++) {
+    v.push_back(rand());
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+  }
+
+  for(int i = 0; i < 500; i++) {
+    tqHandleCommit(pMeta, i);
+    v[i] = rand();
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+  }
+
+  for(int i = 500; i < 1000; i++) {
+    v[i] = rand();
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    tqHandleCommit(pMeta, i);
+  }
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+  
+  for(int i = 500; i < 1000; i++) {
+    v[i] = rand();
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+    tqHandleCommit(pMeta, i);
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
+    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
+    EXPECT_EQ(pFoo->a, v[i]);
+  }
+
+}
+
+TEST_F(TqMetaTest, dupCommit) {
+  srand(0);
+  std::vector<int> v;
+  for(int i = 0; i < 1000; i++) {
+    v.push_back(rand());
+    Foo foo;
+    foo.a = v[i];
+    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    int ret = tqHandleCommit(pMeta, i);
+    EXPECT_EQ(ret, 0);
+    ret = tqHandleCommit(pMeta, i);
+    EXPECT_EQ(ret, -1);
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    int ret = tqHandleCommit(pMeta, i);
+    EXPECT_EQ(ret, -1);
+  }
+
+  for(int i = 0; i < 1000; i++) {
+    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
+    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
+    EXPECT_EQ(pFoo->a, v[i]);
+  }
+
 }

source/libs/sync/inc/raftInt.h → source/dnode/vnode/tsdb/inc/tsdbIdx.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_RAFT_INT_H_
-#define _TD_RAFT_INT_H_
+#ifndef _TD_TSDB_IDX_H_
+#define _TD_TSDB_IDX_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,4 +24,4 @@ extern "C" {
 }
 #endif
 
-#endif /*_TD_RAFT_INT_H_*/
\ No newline at end of file
+#endif /*_TD_TSDB_IDX_H_*/
\ No newline at end of file

source/dnode/vnode/tsdb/inc/tsdbOptions.h

@@ -20,6 +20,11 @@
 extern "C" {
 #endif
 
+extern const STsdbOptions defautlTsdbOptions;
+
+int  tsdbValidateOptions(const STsdbOptions *);
+void tsdbOptionsCopy(STsdbOptions *pDest, const STsdbOptions *pSrc);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/tsdb/inc/tsdbSMA.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_SMA_H_
+#define _TD_TSDB_SMA_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TSDB_SMA_H_*/
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbIdx.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbMain.c

@@ -13,4 +13,82 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
\ No newline at end of file
+#include "tsdbDef.h"
+
+static STsdb *tsdbNew(const char *path, const STsdbOptions *pTsdbOptions);
+static void   tsdbFree(STsdb *pTsdb);
+static int    tsdbOpenImpl(STsdb *pTsdb);
+static void   tsdbCloseImpl(STsdb *pTsdb);
+
+STsdb *tsdbOpen(const char *path, const STsdbOptions *pTsdbOptions) {
+  STsdb *pTsdb = NULL;
+
+  // Set default TSDB Options
+  if (pTsdbOptions == NULL) {
+    pTsdbOptions = &defautlTsdbOptions;
+  }
+
+  // Validate the options
+  if (tsdbValidateOptions(pTsdbOptions) < 0) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  // Create the handle
+  pTsdb = tsdbNew(path, pTsdbOptions);
+  if (pTsdb == NULL) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  taosMkDir(path);
+
+  // Open the TSDB
+  if (tsdbOpenImpl(pTsdb) < 0) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  return pTsdb;
+}
+
+void tsdbClose(STsdb *pTsdb) {
+  if (pTsdb) {
+    tsdbCloseImpl(pTsdb);
+    tsdbFree(pTsdb);
+  }
+}
+
+void tsdbRemove(const char *path) { taosRemoveDir(path); }
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static STsdb *tsdbNew(const char *path, const STsdbOptions *pTsdbOptions) {
+  STsdb *pTsdb = NULL;
+
+  pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
+  if (pTsdb == NULL) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  pTsdb->path = strdup(path);
+  tsdbOptionsCopy(&(pTsdb->options), pTsdbOptions);
+
+  return pTsdb;
+}
+
+static void tsdbFree(STsdb *pTsdb) {
+  if (pTsdb) {
+    tfree(pTsdb->path);
+    free(pTsdb);
+  }
+}
+
+static int tsdbOpenImpl(STsdb *pTsdb) {
+  // TODO
+  return 0;
+}
+
+static void tsdbCloseImpl(STsdb *pTsdb) {
+  // TODO
+}
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbOptions.c

@@ -11,4 +11,24 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
+ */
+
+#include "tsdbDef.h"
+
+const STsdbOptions defautlTsdbOptions = {.lruCacheSize = 0};
+
+int tsdbOptionsInit(STsdbOptions *pTsdbOptions) {
+  // TODO
+  return 0;
+}
+
+void tsdbOptionsClear(STsdbOptions *pTsdbOptions) {
+  // TODO
+}
+
+int tsdbValidateOptions(const STsdbOptions *pTsdbOptions) {
+  // TODO
+  return 0;
+}
+
+void tsdbOptionsCopy(STsdbOptions *pDest, const STsdbOptions *pSrc) { memcpy(pDest, pSrc, sizeof(STsdbOptions)); }
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbSMA.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
\ No newline at end of file

source/libs/sync/CMakeLists.txt

@@ -4,6 +4,7 @@ add_library(sync ${SYNC_SRC})
 target_link_libraries(
     sync
     PUBLIC common
+    PUBLIC transport
     PUBLIC util
     PUBLIC wal
 )

source/libs/sync/inc/raft.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_H
+#define _TD_LIBS_SYNC_RAFT_H
+
+#include "sync.h"
+#include "sync_type.h"
+#include "raft_message.h"
+#include "sync_raft_impl.h"
+#include "sync_raft_quorum.h"
+
+typedef struct RaftLeaderState {
+
+} RaftLeaderState;
+
+typedef struct RaftCandidateState {
+  /* true if in pre-vote phase */
+  bool inPreVote;
+} RaftCandidateState;
+
+typedef struct SSyncRaftIOMethods {
+  // send SSyncMessage to node
+  int (*send)(const SSyncMessage* pMsg, const SNodeInfo* pNode);
+} SSyncRaftIOMethods;
+
+typedef int   (*SyncRaftStepFp)(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+typedef void  (*SyncRaftTickFp)(SSyncRaft* pRaft);
+
+struct SSyncRaft {
+  // owner sync node
+  SSyncNode* pNode;
+
+  SSyncCluster cluster; 
+
+  int selfIndex;
+  SyncNodeId selfId;
+  SyncGroupId selfGroupId;
+
+  SSyncRaftIOMethods io;
+
+  SSyncFSM      fsm;
+  SSyncLogStore logStore;
+  SStateManager stateManager;
+
+  union {
+    RaftLeaderState leaderState;
+    RaftCandidateState candidateState;
+  };
+
+  SyncTerm term;
+  SyncNodeId voteFor;
+
+  SSyncRaftLog *log;
+
+  int maxMsgSize;
+  SSyncRaftProgressTracker *tracker;
+
+  ESyncState state;
+
+  // isLearner is true if the local raft node is a learner.
+  bool isLearner;
+
+  /**
+   * the leader id
+  **/
+  SyncNodeId leaderId;
+
+	/**
+   * leadTransferee is id of the leader transfer target when its value is not zero.
+	 * Follow the procedure defined in raft thesis 3.10.
+   **/
+  SyncNodeId leadTransferee;
+
+	/**
+   * Only one conf change may be pending (in the log, but not yet
+	 * applied) at a time. This is enforced via pendingConfIndex, which
+	 * is set to a value >= the log index of the latest pending
+	 * configuration change (if any). Config changes are only allowed to
+	 * be proposed if the leader's applied index is greater than this
+	 * value.
+   **/
+  SyncIndex pendingConfigIndex;
+
+	/** 
+   * an estimate of the size of the uncommitted tail of the Raft log. Used to
+	 * prevent unbounded log growth. Only maintained by the leader. Reset on
+	 * term changes.
+   **/
+  uint32_t uncommittedSize;
+ 
+	/** 
+   * number of ticks since it reached last electionTimeout when it is leader
+	 * or candidate.
+	 * number of ticks since it reached last electionTimeout or received a
+	 * valid message from current leader when it is a follower.
+  **/
+  uint16_t electionElapsed;
+
+	/**
+   *  number of ticks since it reached last heartbeatTimeout.
+	 * only leader keeps heartbeatElapsed.
+   **/
+  uint16_t heartbeatElapsed;
+
+  bool preVote;
+  bool checkQuorum;
+
+  int heartbeatTimeout;
+  int electionTimeout;
+
+	/**
+   * randomizedElectionTimeout is a random number between
+	 * [electiontimeout, 2 * electiontimeout - 1]. It gets reset
+	 * when raft changes its state to follower or candidate.
+   **/
+	int randomizedElectionTimeout;
+	bool disableProposalForwarding;
+
+  // current tick count since start up
+  uint32_t currentTick;
+
+  SyncRaftStepFp stepFp;
+
+  SyncRaftTickFp tickFp;
+};
+
+int32_t syncRaftStart(SSyncRaft* pRaft, const SSyncInfo* pInfo);
+int32_t syncRaftStep(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+int32_t syncRaftTick(SSyncRaft* pRaft);
+
+#endif /* _TD_LIBS_SYNC_RAFT_H */
\ No newline at end of file

source/libs/sync/inc/raft_configuration.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_CONFIGURATION_H
+#define _TD_LIBS_SYNC_RAFT_CONFIGURATION_H
+
+#include "sync.h"
+#include "sync_type.h"
+
+// return -1 if cannot find this id
+int syncRaftConfigurationIndexOfNode(SSyncRaft *pRaft, SyncNodeId id);
+
+int syncRaftConfigurationVoterCount(SSyncRaft *pRaft);
+
+#endif  /* _TD_LIBS_SYNC_RAFT_CONFIGURATION_H */
\ No newline at end of file

source/libs/sync/inc/raft_log.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_LOG_H
+#define _TD_LIBS_SYNC_RAFT_LOG_H
+
+#include "sync.h"
+#include "sync_type.h"
+
+typedef enum SyncEntryType {
+  SYNC_ENTRY_TYPE_LOG = 1,
+}SyncEntryType;
+
+struct SSyncRaftEntry {
+  SyncTerm term;
+
+  SyncIndex index;
+
+  SyncEntryType type;
+
+  SSyncBuffer buffer;
+};
+
+struct SSyncRaftLog {
+  SyncIndex uncommittedConfigIndex;
+
+  SyncIndex commitIndex;
+
+  SyncIndex appliedIndex;
+
+
+};
+
+SSyncRaftLog* syncRaftLogOpen();
+
+SyncIndex syncRaftLogLastIndex(SSyncRaftLog* pLog);
+
+SyncIndex syncRaftLogSnapshotIndex(SSyncRaftLog* pLog);
+
+SyncTerm syncRaftLogLastTerm(SSyncRaftLog* pLog);
+
+bool syncRaftLogIsUptodate(SSyncRaftLog* pLog, SyncIndex index, SyncTerm term);
+
+int syncRaftLogNumOfPendingConf(SSyncRaftLog* pLog);
+
+bool syncRaftHasUnappliedLog(SSyncRaftLog* pLog);
+
+SyncTerm syncRaftLogTermOf(SSyncRaftLog* pLog, SyncIndex index);
+
+int syncRaftLogAppend(SSyncRaftLog* pLog, SSyncRaftEntry *pEntries, int n);
+
+int syncRaftLogAcquire(SSyncRaftLog* pLog, SyncIndex index, int maxMsgSize,
+                      SSyncRaftEntry **ppEntries, int *n);
+
+void syncRaftLogRelease(SSyncRaftLog* pLog, SyncIndex index,
+                      SSyncRaftEntry *pEntries, int n);
+
+bool syncRaftLogMatchTerm();
+
+static FORCE_INLINE bool syncRaftLogIsCommitted(SSyncRaftLog* pLog, SyncIndex index) {
+  return pLog->commitIndex > index;
+}
+
+#endif  /* _TD_LIBS_SYNC_RAFT_LOG_H */

source/libs/sync/inc/raft_message.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_MESSAGE_H
+#define _TD_LIBS_SYNC_RAFT_MESSAGE_H
+
+#include "sync.h"
+#include "sync_type.h"
+
+/** 
+ * below define message type which handled by Raft.
+ * 
+ * internal message, which communicate between threads, start with RAFT_MSG_INTERNAL_*.
+ * internal message use pointer only and stack memory, need not to be decode/encode and free.
+ * 
+ * outter message start with RAFT_MSG_*, which communicate between cluster peers,
+ * need to implement its decode/encode functions.
+ **/
+typedef enum RaftMessageType {
+  // client propose a cmd
+  RAFT_MSG_INTERNAL_PROP = 1,
+
+  // node election timeout
+  RAFT_MSG_INTERNAL_ELECTION = 2,
+
+  RAFT_MSG_VOTE = 3,
+  RAFT_MSG_VOTE_RESP = 4,
+
+  RAFT_MSG_APPEND = 5,
+  RAFT_MSG_APPEND_RESP = 6,
+} RaftMessageType;
+
+typedef struct RaftMsgInternal_Prop {
+  const SSyncBuffer *pBuf;
+  bool isWeak;
+  void* pData;
+} RaftMsgInternal_Prop;
+
+typedef struct RaftMsgInternal_Election {
+
+} RaftMsgInternal_Election;
+
+typedef struct RaftMsg_Vote {
+  SyncRaftElectionType cType;
+  SyncIndex lastIndex;
+  SyncTerm lastTerm;
+} RaftMsg_Vote;
+
+typedef struct RaftMsg_VoteResp {
+  bool rejected;
+  SyncRaftElectionType cType;
+} RaftMsg_VoteResp;
+
+typedef struct RaftMsg_Append_Entries {
+  // index of log entry preceeding new ones
+  SyncIndex index;
+
+  // term of entry at prevIndex
+  SyncTerm term;
+
+  // leader's commit index.
+  SyncIndex commitIndex;
+
+  // size of the log entries array
+  int nEntries;
+
+  // log entries array
+  SSyncRaftEntry* entries;
+} RaftMsg_Append_Entries;
+
+typedef struct RaftMsg_Append_Resp {
+  SyncIndex index;
+} RaftMsg_Append_Resp;
+
+typedef struct SSyncMessage {
+  RaftMessageType msgType;
+  SyncTerm term;
+  SyncGroupId groupId;
+  SyncNodeId from;
+
+  union {
+    RaftMsgInternal_Prop propose;
+
+    RaftMsgInternal_Election election;
+
+    RaftMsg_Vote vote;
+    RaftMsg_VoteResp voteResp;
+
+    RaftMsg_Append_Entries appendEntries;
+    RaftMsg_Append_Resp appendResp;
+  };
+} SSyncMessage;
+
+static FORCE_INLINE SSyncMessage* syncInitPropMsg(SSyncMessage* pMsg, const SSyncBuffer* pBuf, void* pData, bool isWeak) {
+  *pMsg = (SSyncMessage) {
+    .msgType = RAFT_MSG_INTERNAL_PROP,
+    .term = 0,
+    .propose = (RaftMsgInternal_Prop) {
+      .isWeak = isWeak,
+      .pBuf = pBuf,
+      .pData = pData,
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE SSyncMessage* syncInitElectionMsg(SSyncMessage* pMsg, SyncNodeId from) {
+  *pMsg = (SSyncMessage) {
+    .msgType = RAFT_MSG_INTERNAL_ELECTION,
+    .term = 0,
+    .from = from,
+    .election = (RaftMsgInternal_Election) {
+
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE SSyncMessage* syncNewVoteMsg(SyncGroupId groupId, SyncNodeId from,
+                                                SyncTerm term, SyncRaftElectionType cType, 
+                                                SyncIndex lastIndex, SyncTerm lastTerm) {
+  SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage));
+  if (pMsg == NULL) {
+    return NULL;
+  }
+  *pMsg = (SSyncMessage) {
+    .groupId = groupId,
+    .from = from,
+    .term = term,
+    .msgType = RAFT_MSG_VOTE,
+    .vote = (RaftMsg_Vote) {
+      .cType = cType,
+      .lastIndex = lastIndex,
+      .lastTerm = lastTerm,
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE SSyncMessage* syncNewVoteRespMsg(SyncGroupId groupId, SyncNodeId from,
+                                                  SyncRaftElectionType cType, bool rejected) {
+  SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage));
+  if (pMsg == NULL) {
+    return NULL;
+  }
+  *pMsg = (SSyncMessage) {
+    .groupId = groupId,
+    .from = from,
+    .msgType = RAFT_MSG_VOTE_RESP,
+    .voteResp = (RaftMsg_VoteResp) {
+      .cType = cType,
+      .rejected = rejected,
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE SSyncMessage* syncNewAppendMsg(SyncGroupId groupId, SyncNodeId from,
+                                                  SyncTerm term, SyncIndex logIndex, SyncTerm logTerm,
+                                                  SyncIndex commitIndex, int nEntries, SSyncRaftEntry* entries) {
+  SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage));
+  if (pMsg == NULL) {
+    return NULL;
+  }
+  *pMsg = (SSyncMessage) {
+    .groupId = groupId,
+    .from = from,
+    .term = term,
+    .msgType = RAFT_MSG_APPEND,
+    .appendEntries = (RaftMsg_Append_Entries) {
+      .index = logIndex,
+      .term = logTerm,
+      .commitIndex = commitIndex,
+      .nEntries = nEntries,
+      .entries = entries,
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE SSyncMessage* syncNewEmptyAppendRespMsg(SyncGroupId groupId, SyncNodeId from, SyncTerm term) {
+  SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage));
+  if (pMsg == NULL) {
+    return NULL;
+  }
+  *pMsg = (SSyncMessage) {
+    .groupId = groupId,
+    .from = from,
+    .term = term,
+    .msgType = RAFT_MSG_APPEND_RESP,
+    .appendResp = (RaftMsg_Append_Resp) {
+
+    },
+  };
+
+  return pMsg;
+}
+
+static FORCE_INLINE bool syncIsInternalMsg(RaftMessageType msgType) {
+  return msgType == RAFT_MSG_INTERNAL_PROP ||
+         msgType == RAFT_MSG_INTERNAL_ELECTION;
+}
+
+static FORCE_INLINE bool syncIsPreVoteRespMsg(const SSyncMessage* pMsg) {
+  return pMsg->msgType == RAFT_MSG_VOTE_RESP && pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION;
+}
+
+static FORCE_INLINE bool syncIsPreVoteMsg(const SSyncMessage* pMsg) {
+  return pMsg->msgType == RAFT_MSG_VOTE && pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION;
+}
+
+void syncFreeMessage(const SSyncMessage* pMsg);
+
+// message handlers
+int syncRaftHandleElectionMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+int syncRaftHandleVoteRespMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+int syncRaftHandleAppendEntriesMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+
+#endif  /* _TD_LIBS_SYNC_RAFT_MESSAGE_H */
\ No newline at end of file

source/libs/sync/inc/raft_replication.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_SYNC_RAFT_REPLICATION_H
+#define TD_SYNC_RAFT_REPLICATION_H
+
+#include "sync.h"
+#include "syncInt.h"
+#include "sync_type.h"
+
+int syncRaftReplicate(SSyncRaft* pRaft, int i);
+
+#endif  /* TD_SYNC_RAFT_REPLICATION_H */

source/libs/sync/inc/raft_unstable_log.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_SYNC_RAFT_UNSTABLE_LOG_H
+#define TD_SYNC_RAFT_UNSTABLE_LOG_H
+
+#include "sync_type.h"
+
+/* in-memory unstable raft log storage */
+struct SSyncRaftUnstableLog {
+#if 0
+  /* Circular buffer of log entries */
+  RaftEntry *entries;
+
+  /* size of Circular buffer */
+  int size;
+
+  /* Indexes of used slots [front, back) */
+  int front, back;
+
+  /* Index of first entry is offset + 1 */
+  SyncIndex offset;
+
+  /* meta data of snapshot */
+  SSyncRaftUnstableLog snapshot;
+#endif
+};
+
+/**
+ * return index of last in memory log, return 0 if log is empty
+ **/
+//SyncIndex syncRaftLogLastIndex(SSyncRaftUnstableLog* pLog);
+
+#if 0
+void raftLogInit(RaftLog* pLog);
+
+void raftLogClose(RaftLog* pLog);
+
+/**
+ * When startup populating log entrues loaded from disk,
+ * init raft memory log with snapshot index,term and log start idnex.
+ **/ 
+/*
+void raftLogStart(RaftLog* pLog,
+                  RaftSnapshotMeta snapshot,
+                  SyncIndex startIndex);
+*/
+/** 
+ * Get the number of entries the log. 
+ **/
+int raftLogNumEntries(const RaftLog* pLog);
+
+
+
+/**
+ * return last term of in memory log, return 0 if log is empty
+ **/
+SyncTerm raftLogLastTerm(RaftLog* pLog);
+
+/**
+ * return term of log with the given index, return 0 if the term of index cannot be found
+ * , errCode will save the error code.
+ **/
+SyncTerm raftLogTermOf(RaftLog* pLog, SyncIndex index, RaftCode* errCode);
+
+/** 
+ * Get the last index of the most recent snapshot. Return 0 if there are no *
+ * snapshots. 
+ **/
+SyncIndex raftLogSnapshotIndex(RaftLog* pLog);
+
+/* Append a new entry to the log. */
+int raftLogAppend(RaftLog* pLog,
+                  SyncTerm term,
+                  const SSyncBuffer *buf);
+
+/**
+ * acquire log from given index onwards.
+ **/ 
+/*
+int raftLogAcquire(RaftLog* pLog,
+                  SyncIndex index,
+                  RaftEntry **ppEntries,
+                  int *n);
+
+void raftLogRelease(RaftLog* pLog,
+                    SyncIndex index,
+                    RaftEntry *pEntries,
+                    int n);
+*/
+/* Delete all entries from the given index (included) onwards. */
+void raftLogTruncate(RaftLog* pLog, SyncIndex index);
+
+/** 
+ * when taking a new snapshot, the function will update the last snapshot information and delete
+ * all entries up last_index - trailing (included). If the log contains no entry
+ * a last_index - trailing, then no entry will be deleted. 
+ **/
+void raftLogSnapshot(RaftLog* pLog, SyncIndex index, SyncIndex trailing);
+
+#endif
+
+#endif /* TD_SYNC_RAFT_UNSTABLE_LOG_H */
\ No newline at end of file

source/libs/sync/inc/syncInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_INT_H
+#define _TD_LIBS_SYNC_INT_H
+
+#include "thash.h"
+#include "os.h"
+#include "sync.h"
+#include "sync_type.h"
+#include "raft.h"
+#include "tlog.h"
+
+#define TAOS_SYNC_MAX_WORKER 3
+
+typedef struct SSyncWorker {
+  pthread_t thread;
+} SSyncWorker;
+
+struct SSyncNode {
+  pthread_mutex_t   mutex;
+  int32_t      refCount;
+  SyncGroupId   vgId;
+  SSyncRaft raft;
+  void* syncTimer;
+};
+
+typedef struct SSyncManager {
+  pthread_mutex_t   mutex;
+
+  // sync server rpc
+  void* serverRpc;
+  // rpc server hash table base on FQDN:port key
+  SHashObj* rpcServerTable;
+
+  // sync client rpc
+  void* clientRpc;
+
+  // worker threads
+  SSyncWorker worker[TAOS_SYNC_MAX_WORKER];
+
+  // vgroup hash table
+  SHashObj* vgroupTable;
+
+  // timer manager
+  void* syncTimerManager;
+
+} SSyncManager;
+
+extern SSyncManager* gSyncManager;
+
+#define syncFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYNC FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define syncError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYNC ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define syncWarn(...)  do { if (sDebugFlag & DEBUG_WARN)  { taosPrintLog("SYNC WARN ", 255, __VA_ARGS__); }}      while(0)
+#define syncInfo(...)  do { if (sDebugFlag & DEBUG_INFO)  { taosPrintLog("SYNC ", 255, __VA_ARGS__); }}           while(0)
+#define syncDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYNC ", sDebugFlag, __VA_ARGS__); }} while(0)
+#define syncTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYNC ", sDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif  /* _TD_LIBS_SYNC_INT_H */
\ No newline at end of file

source/libs/sync/inc/sync_raft_impl.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_IMPL_H
+#define _TD_LIBS_SYNC_RAFT_IMPL_H
+
+#include "sync.h"
+#include "sync_type.h"
+#include "raft_message.h"
+#include "sync_raft_quorum.h"
+
+void syncRaftBecomeFollower(SSyncRaft* pRaft, SyncTerm term, SyncNodeId leaderId);
+void syncRaftBecomePreCandidate(SSyncRaft* pRaft);
+void syncRaftBecomeCandidate(SSyncRaft* pRaft);
+void syncRaftBecomeLeader(SSyncRaft* pRaft);
+
+void syncRaftStartElection(SSyncRaft* pRaft, SyncRaftElectionType cType);
+
+void syncRaftTriggerHeartbeat(SSyncRaft* pRaft);
+
+void syncRaftRandomizedElectionTimeout(SSyncRaft* pRaft);
+bool syncRaftIsPromotable(SSyncRaft* pRaft);
+bool syncRaftIsPastElectionTimeout(SSyncRaft* pRaft);
+int  syncRaftQuorum(SSyncRaft* pRaft);
+
+SSyncRaftVoteResult  syncRaftPollVote(SSyncRaft* pRaft, SyncNodeId id, 
+                                    bool preVote, bool accept, 
+                                    int* rejectNum, int *granted);
+
+#endif /* _TD_LIBS_SYNC_RAFT_IMPL_H */

source/libs/sync/inc/sync_raft_inflights.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http: *www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_SYNC_RAFT_INFLIGHTS_H
+#define TD_SYNC_RAFT_INFLIGHTS_H
+
+#include "sync.h"
+
+/**
+ * SSyncRaftInflights limits the number of MsgApp (represented by the largest index
+ * contained within) sent to followers but not yet acknowledged by them. Callers
+ * use syncRaftInflightFull() to check whether more messages can be sent, 
+ * call syncRaftInflightAdd() whenever they are sending a new append, 
+ * and release "quota" via FreeLE() whenever an ack is received.
+**/
+typedef struct SSyncRaftInflights {
+  /* the starting index in the buffer */
+  int start;
+
+  /* number of inflights in the buffer */
+  int count;
+
+  /* the size of the buffer */
+  int size;
+
+	/** 
+   * buffer contains the index of the last entry
+	 * inside one message.
+   **/
+  SyncIndex* buffer;
+} SSyncRaftInflights;
+
+SSyncRaftInflights* syncRaftOpenInflights(int size);
+void syncRaftCloseInflights(SSyncRaftInflights*);
+
+static FORCE_INLINE void syncRaftInflightReset(SSyncRaftInflights* inflights) {  
+  inflights->count = 0;
+  inflights->start = 0;
+}
+
+static FORCE_INLINE bool syncRaftInflightFull(SSyncRaftInflights* inflights) {
+  return inflights->count == inflights->size;
+}
+
+/**
+ * syncRaftInflightAdd notifies the Inflights that a new message with the given index is being
+ * dispatched. syncRaftInflightFull() must be called prior to syncRaftInflightAdd() 
+ * to verify that there is room for one more message, 
+ * and consecutive calls to add syncRaftInflightAdd() must provide a
+ * monotonic sequence of indexes.
+ **/
+void syncRaftInflightAdd(SSyncRaftInflights* inflights, SyncIndex inflightIndex);
+
+/**
+ * syncRaftInflightFreeLE frees the inflights smaller or equal to the given `to` flight.
+ **/
+void syncRaftInflightFreeLE(SSyncRaftInflights* inflights, SyncIndex toIndex);
+
+/** 
+ * syncRaftInflightFreeFirstOne releases the first inflight. 
+ * This is a no-op if nothing is inflight.
+ **/
+void syncRaftInflightFreeFirstOne(SSyncRaftInflights* inflights);
+
+#endif /* TD_SYNC_RAFT_INFLIGHTS_H */
\ No newline at end of file

source/libs/sync/inc/sync_raft_progress.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http: *www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_SYNC_RAFT_PROGRESS_H
+#define TD_SYNC_RAFT_PROGRESS_H
+
+#include "sync_type.h"
+#include "sync_raft_inflights.h"
+
+/** 
+ * State defines how the leader should interact with the follower.
+ *
+ * When in PROGRESS_STATE_PROBE, leader sends at most one replication message
+ * per heartbeat interval. It also probes actual progress of the follower.
+ * 
+ * When in PROGRESS_STATE_REPLICATE, leader optimistically increases next
+ * to the latest entry sent after sending replication message. This is
+ * an optimized state for fast replicating log entries to the follower.
+ *
+ * When in PROGRESS_STATE_SNAPSHOT, leader should have sent out snapshot
+ * before and stops sending any replication message.
+ * 
+ * PROGRESS_STATE_PROBE is the initial state.
+ **/
+typedef enum RaftProgressState {
+  /**
+	* StateProbe indicates a follower whose last index isn't known. Such a
+	* follower is "probed" (i.e. an append sent periodically) to narrow down
+	* its last index. In the ideal (and common) case, only one round of probing
+	* is necessary as the follower will react with a hint. Followers that are
+	* probed over extended periods of time are often offline.
+  **/
+  PROGRESS_STATE_PROBE = 0,
+
+  /**
+	* StateReplicate is the state steady in which a follower eagerly receives
+	* log entries to append to its log.
+  **/
+  PROGRESS_STATE_REPLICATE,
+
+  /**
+	* StateSnapshot indicates a follower that needs log entries not available
+	* from the leader's Raft log. Such a follower needs a full snapshot to
+	* return to StateReplicate.
+  **/
+  PROGRESS_STATE_SNAPSHOT,
+} RaftProgressState;
+
+/**
+ * Progress represents a follower’s progress in the view of the leader. Leader maintains
+ * progresses of all followers, and sends entries to the follower based on its progress.
+ **/
+struct SSyncRaftProgress {
+	SyncNodeId id;
+
+  SyncIndex nextIndex;
+
+  SyncIndex matchIndex;
+
+  /**
+	* State defines how the leader should interact with the follower.
+	*
+	* When in StateProbe, leader sends at most one replication message
+	* per heartbeat interval. It also probes actual progress of the follower.
+	*
+	* When in StateReplicate, leader optimistically increases next
+	* to the latest entry sent after sending replication message. This is
+	* an optimized state for fast replicating log entries to the follower.
+	*
+	* When in StateSnapshot, leader should have sent out snapshot
+	* before and stops sending any replication message.
+  **/
+  RaftProgressState state;
+
+	/** 
+   * pendingSnapshotIndex is used in PROGRESS_STATE_SNAPSHOT.
+	 * If there is a pending snapshot, the pendingSnapshotIndex will be set to the
+	 * index of the snapshot. If pendingSnapshotIndex is set, the replication process of
+	 * this Progress will be paused. raft will not resend snapshot until the pending one
+	 * is reported to be failed.
+   **/
+  SyncIndex pendingSnapshotIndex;
+
+	/** 
+   * recentActive is true if the progress is recently active. Receiving any messages
+   * from the corresponding follower indicates the progress is active.
+	 * RecentActive can be reset to false after an election timeout.
+   **/
+  bool recentActive;
+
+  /**
+	* probeSent is used while this follower is in StateProbe. When probeSent is
+	* true, raft should pause sending replication message to this peer until
+	* probeSent is reset. See ProbeAcked() and IsPaused().
+  **/
+	bool probeSent;
+
+  /**
+	 * inflights is a sliding window for the inflight messages.
+	 * Each inflight message contains one or more log entries.
+	 * The max number of entries per message is defined in raft config as MaxSizePerMsg.
+	 * Thus inflight effectively limits both the number of inflight messages
+	 * and the bandwidth each Progress can use.
+	 * When inflights is Full, no more message should be sent.
+	 * When a leader sends out a message, the index of the last
+	 * entry should be added to inflights. The index MUST be added
+	 * into inflights in order.
+	 * When a leader receives a reply, the previous inflights should
+	 * be freed by calling inflights.FreeLE with the index of the last
+	 * received entry.
+  **/
+  SSyncRaftInflights* inflights;
+
+  /**
+   *  IsLearner is true if this progress is tracked for a learner.
+   **/
+  bool isLearner;
+};
+
+void syncRaftInitProgress(int i, SSyncRaft* pRaft, SSyncRaftProgress* progress);
+
+/**
+ * syncRaftProgressBecomeProbe transitions into StateProbe. Next is reset to Match+1 or,
+ * optionally and if larger, the index of the pending snapshot.
+ **/
+void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress);
+
+/**
+ * syncRaftProgressBecomeReplicate transitions into StateReplicate, resetting Next to Match+1.
+ **/ 
+void syncRaftProgressBecomeReplicate(SSyncRaftProgress* progress);
+
+/**
+ * syncRaftProgressMaybeUpdate is called when an MsgAppResp arrives from the follower, with the
+ * index acked by it. The method returns false if the given n index comes from
+ * an outdated message. Otherwise it updates the progress and returns true.
+ **/
+bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastIndex);
+
+/**
+ * syncRaftProgressOptimisticNextIndex signals that appends all the way up to and including index n
+ * are in-flight. As a result, Next is increased to n+1.
+ **/
+static FORCE_INLINE void syncRaftProgressOptimisticNextIndex(SSyncRaftProgress* progress, SyncIndex nextIndex) {
+  progress->nextIndex = nextIndex + 1;
+}
+
+/**
+ * syncRaftProgressMaybeDecrTo adjusts the Progress to the receipt of a MsgApp rejection. The
+ * arguments are the index of the append message rejected by the follower, and
+ * the hint that we want to decrease to.
+ *
+ * Rejections can happen spuriously as messages are sent out of order or
+ * duplicated. In such cases, the rejection pertains to an index that the
+ * Progress already knows were previously acknowledged, and false is returned
+ * without changing the Progress.
+ *
+ * If the rejection is genuine, Next is lowered sensibly, and the Progress is
+ * cleared for sending log entries.
+**/
+bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress,
+                                SyncIndex rejected, SyncIndex matchHint);
+
+/**
+ * syncRaftProgressIsPaused returns whether sending log entries to this node has been throttled.
+ * This is done when a node has rejected recent MsgApps, is currently waiting
+ * for a snapshot, or has reached the MaxInflightMsgs limit. In normal
+ * operation, this is false. A throttled node will be contacted less frequently
+ * until it has reached a state in which it's able to accept a steady stream of
+ * log entries again.
+ **/
+bool syncRaftProgressIsPaused(SSyncRaftProgress* progress);
+
+static FORCE_INLINE SyncIndex syncRaftProgressNextIndex(SSyncRaftProgress* progress) {
+  return progress->nextIndex;
+}
+
+static FORCE_INLINE RaftProgressState syncRaftProgressInReplicate(SSyncRaftProgress* progress) {
+  return progress->state == PROGRESS_STATE_REPLICATE;
+}
+
+static FORCE_INLINE RaftProgressState syncRaftProgressInSnapshot(SSyncRaftProgress* progress) {
+  return progress->state == PROGRESS_STATE_SNAPSHOT;
+}
+
+static FORCE_INLINE RaftProgressState syncRaftProgressInProbe(SSyncRaftProgress* progress) {
+  return progress->state == PROGRESS_STATE_PROBE;
+}
+
+static FORCE_INLINE bool syncRaftProgressRecentActive(SSyncRaftProgress* progress) {
+  return progress->recentActive;
+}
+
+/** 
+ * return true if progress's log is up-todate
+ **/
+bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress);
+
+void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex);
+
+
+
+#if 0
+
+void syncRaftProgressAbortSnapshot(SSyncRaft* pRaft, int i);
+
+
+
+SyncIndex syncRaftProgressMatchIndex(SSyncRaft* pRaft, int i);
+
+void syncRaftProgressUpdateLastSend(SSyncRaft* pRaft, int i);
+
+void syncRaftProgressUpdateSnapshotLastSend(SSyncRaft* pRaft, int i);
+
+bool syncRaftProgressResetRecentRecv(SSyncRaft* pRaft, int i);
+
+void syncRaftProgressMarkRecentRecv(SSyncRaft* pRaft, int i);
+
+
+
+void syncRaftProgressAbortSnapshot(SSyncRaft* pRaft, int i);
+
+#endif
+
+#endif /* TD_SYNC_RAFT_PROGRESS_H */
\ No newline at end of file

source/libs/sync/inc/sync_raft_progress_tracker.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H
+#define _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H
+
+#include "sync_type.h"
+#include "sync_raft_quorum_joint.h"
+#include "sync_raft_progress.h"
+
+struct SSyncRaftProgressTrackerConfig {
+  SSyncRaftQuorumJointConfig voters;
+
+	/** 
+	 * autoLeave is true if the configuration is joint and a transition to the
+	 * incoming configuration should be carried out automatically by Raft when
+	 * this is possible. If false, the configuration will be joint until the
+	 * application initiates the transition manually.
+   **/
+  bool autoLeave;
+
+	/** 
+   * Learners is a set of IDs corresponding to the learners active in the
+	 * current configuration.
+   * 
+	 * Invariant: Learners and Voters does not intersect, i.e. if a peer is in
+	 * either half of the joint config, it can't be a learner; if it is a
+	 * learner it can't be in either half of the joint config. This invariant
+	 * simplifies the implementation since it allows peers to have clarity about
+	 * its current role without taking into account joint consensus.
+   **/
+  SyncNodeId learners[TSDB_MAX_REPLICA];
+
+  /**
+   *  When we turn a voter into a learner during a joint consensus transition,
+	 * we cannot add the learner directly when entering the joint state. This is
+	 * because this would violate the invariant that the intersection of
+	 * voters and learners is empty. For example, assume a Voter is removed and
+	 * immediately re-added as a learner (or in other words, it is demoted):
+   * 
+	 * Initially, the configuration will be
+   * 
+	 *   voters:   {1 2 3}
+	 *   learners: {}
+   * 
+	 * and we want to demote 3. Entering the joint configuration, we naively get
+   * 
+	 *   voters:   {1 2} & {1 2 3}
+	 *   learners: {3}
+   * 
+	 * but this violates the invariant (3 is both voter and learner). Instead,
+	 * we get
+   * 
+	 *   voters:   {1 2} & {1 2 3}
+	 *   learners: {}
+	 *   next_learners: {3}
+   * 
+	 * Where 3 is now still purely a voter, but we are remembering the intention
+	 * to make it a learner upon transitioning into the final configuration:
+   * 
+	 *   voters:   {1 2}
+	 *   learners: {3}
+	 *   next_learners: {}
+   * 
+	 * Note that next_learners is not used while adding a learner that is not
+	 * also a voter in the joint config. In this case, the learner is added
+	 * right away when entering the joint configuration, so that it is caught up
+	 * as soon as possible.
+   **/
+  SyncNodeId learnersNext[TSDB_MAX_REPLICA];
+};
+
+struct SSyncRaftProgressTracker {
+  SSyncRaftProgressTrackerConfig config;
+
+  SSyncRaftProgress progressMap[TSDB_MAX_REPLICA];
+
+	SyncRaftVoteResult votes[TSDB_MAX_REPLICA];
+  int maxInflight;
+};
+
+SSyncRaftProgressTracker* syncRaftOpenProgressTracker();
+
+void syncRaftResetVotes(SSyncRaftProgressTracker*);
+
+typedef void (*visitProgressFp)(int i, SSyncRaftProgress* progress, void* arg);
+void syncRaftProgressVisit(SSyncRaftProgressTracker*, visitProgressFp visit, void* arg);
+
+/**
+ * syncRaftRecordVote records that the node with the given id voted for this Raft
+ * instance if v == true (and declined it otherwise).
+ **/
+void syncRaftRecordVote(SSyncRaftProgressTracker* tracker, int i, bool grant);
+
+/** 
+ * syncRaftTallyVotes returns the number of granted and rejected Votes, and whether the
+ * election outcome is known.
+ **/
+SyncRaftVoteResult syncRaftTallyVotes(SSyncRaftProgressTracker* tracker, int* rejected, int *granted);
+
+#endif  /* _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H */

source/libs/sync/inc/sync_raft_quorum.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_SYNC_RAFT_QUORUM_H
+#define TD_SYNC_RAFT_QUORUM_H
+
+/**
+ * SSyncRaftVoteResult indicates the outcome of a vote.
+ **/
+typedef enum {
+	/**
+	 *  SYNC_RAFT_VOTE_PENDING indicates that the decision of the vote depends on future
+	 * votes, i.e. neither "yes" or "no" has reached quorum yet.
+	 **/
+	SYNC_RAFT_VOTE_PENDING = 1,
+
+	/** 
+	 * SYNC_RAFT_VOTE_LOST indicates that the quorum has voted "no".
+	 **/
+	SYNC_RAFT_VOTE_LOST = 2,
+
+	/** 
+	 * SYNC_RAFT_VOTE_WON indicates that the quorum has voted "yes".
+	 **/
+	SYNC_RAFT_VOTE_WON = 3,
+} SSyncRaftVoteResult;
+
+#endif /* TD_SYNC_RAFT_QUORUM_H */
\ No newline at end of file

source/libs/sync/inc/sync_raft_quorum_joint.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H
+#define _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H
+
+#include "taosdef.h"
+#include "sync.h"
+#include "sync_type.h"
+
+/**
+ * SSyncRaftQuorumJointConfig is a configuration of two groups of (possibly overlapping)
+ * majority configurations. Decisions require the support of both majorities.
+ **/
+typedef struct SSyncRaftQuorumJointConfig {
+  SSyncCluster majorityConfig[2];
+}SSyncRaftQuorumJointConfig;
+
+/**
+ * syncRaftVoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+ * a result indicating whether the vote is pending, lost, or won. A joint quorum
+ * requires both majority quorums to vote in favor.
+ **/
+SyncRaftVoteResult syncRaftVoteResult(SSyncRaftQuorumJointConfig* config, const SyncRaftVoteResult* votes);
+
+#endif /* _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H */

source/libs/sync/inc/sync_raft_quorum_majority.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H
+#define _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H
+
+#include "sync.h"
+#include "sync_type.h"
+
+/**
+ * syncRaftMajorityVoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+ * a result indicating whether the vote is pending (i.e. neither a quorum of
+ * yes/no has been reached), won (a quorum of yes has been reached), or lost (a
+ * quorum of no has been reached).
+ **/
+SyncRaftVoteResult syncRaftMajorityVoteResult(SSyncCluster* config, const SyncRaftVoteResult* votes);
+
+#endif /* _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H */

source/libs/sync/inc/sync_type.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TYPE_H
+#define _TD_LIBS_SYNC_TYPE_H
+
+#include <stdint.h>
+#include "osMath.h"
+
+#define SYNC_NON_NODE_ID -1
+#define SYNC_NON_TERM     0
+
+typedef int32_t SyncTime;
+typedef uint32_t SyncTick;
+
+typedef struct SSyncRaft SSyncRaft;
+
+typedef struct SSyncRaftProgress SSyncRaftProgress;
+typedef struct SSyncRaftProgressTrackerConfig SSyncRaftProgressTrackerConfig;
+
+typedef struct SSyncRaftProgressTracker SSyncRaftProgressTracker;
+
+typedef struct SSyncRaftLog SSyncRaftLog;
+
+typedef struct SSyncRaftEntry SSyncRaftEntry;
+
+#if 0
+#ifndef MIN
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+#endif
+
+#ifndef MAX
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#endif
+#endif
+
+typedef enum {
+  SYNC_RAFT_CAMPAIGN_PRE_ELECTION = 0,
+  SYNC_RAFT_CAMPAIGN_ELECTION     = 1,
+  SYNC_RAFT_CAMPAIGN_TRANSFER     = 2,
+} SyncRaftElectionType;
+
+typedef enum {
+  // the init vote resp status
+  SYNC_RAFT_VOTE_RESP_UNKNOWN = 0,
+
+  // grant the vote request
+  SYNC_RAFT_VOTE_RESP_GRANT   = 1,
+
+  //reject the vote request
+  SYNC_RAFT_VOTE_RESP_REJECT  = 2,
+} SyncRaftVoteResult;
+
+#endif  /* _TD_LIBS_SYNC_TYPE_H */

source/libs/sync/src/raft.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft.h"
+#include "raft_configuration.h"
+#include "raft_log.h"
+#include "raft_replication.h"
+#include "sync_raft_progress_tracker.h"
+#include "syncInt.h"
+
+#define RAFT_READ_LOG_MAX_NUM 100
+
+static bool preHandleMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+static bool preHandleNewTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+static bool preHandleOldTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+
+int32_t syncRaftStart(SSyncRaft* pRaft, const SSyncInfo* pInfo) {
+  SSyncNode* pNode = pRaft->pNode;
+  SSyncServerState serverState;
+  SStateManager* stateManager;
+  SSyncLogStore* logStore;
+  SSyncFSM* fsm;
+  SyncIndex initIndex = pInfo->snapshotIndex;
+  SSyncBuffer buffer[RAFT_READ_LOG_MAX_NUM];
+  int nBuf, limit, i;
+  
+  memset(pRaft, 0, sizeof(SSyncRaft));
+
+  memcpy(&pRaft->fsm, &pInfo->fsm, sizeof(SSyncFSM));
+  memcpy(&pRaft->logStore, &pInfo->logStore, sizeof(SSyncLogStore));
+  memcpy(&pRaft->stateManager, &pInfo->stateManager, sizeof(SStateManager));
+
+  stateManager = &(pRaft->stateManager);
+  logStore = &(pRaft->logStore);
+  fsm = &(pRaft->fsm);
+
+  // init progress tracker
+  pRaft->tracker = syncRaftOpenProgressTracker();
+  if (pRaft->tracker == NULL) {
+    return -1;
+  }
+
+  // open raft log
+  if ((pRaft->log = syncRaftLogOpen()) == NULL) {
+    return -1;
+  }
+  // read server state
+  if (stateManager->readServerState(stateManager, &serverState) != 0) {
+    syncError("readServerState for vgid %d fail", pInfo->vgId);
+    return -1;
+  }
+  assert(initIndex <= serverState.commitIndex);
+
+  // restore fsm state from snapshot index + 1 until commitIndex
+  ++initIndex;
+  while (initIndex <= serverState.commitIndex) {
+    limit = MIN(RAFT_READ_LOG_MAX_NUM, serverState.commitIndex - initIndex + 1);
+
+    if (logStore->logRead(logStore, initIndex, limit, buffer, &nBuf) != 0) {
+      return -1;
+    }
+    assert(limit == nBuf);
+  
+    for (i = 0; i < limit; ++i) {
+      fsm->applyLog(fsm, initIndex + i, &(buffer[i]), NULL);
+      free(buffer[i].data);
+    }
+    initIndex += nBuf;
+  }
+  assert(initIndex == serverState.commitIndex);
+
+  //pRaft->heartbeatTimeoutTick = 1;
+
+  syncRaftBecomeFollower(pRaft, pRaft->term, SYNC_NON_NODE_ID);
+
+  pRaft->selfIndex = pRaft->cluster.selfIndex;
+
+  syncInfo("[%d:%d] restore vgid %d state: snapshot index success", 
+    pRaft->selfGroupId, pRaft->selfId, pInfo->vgId);
+  return 0;
+}
+
+int32_t syncRaftStep(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  syncDebug("from %d, type:%d, term:%" PRId64 ", state:%d",
+    pMsg->from, pMsg->msgType, pMsg->term, pRaft->state);
+
+  if (preHandleMessage(pRaft, pMsg)) {
+    syncFreeMessage(pMsg);
+    return 0;
+  }
+
+  RaftMessageType msgType = pMsg->msgType;
+  if (msgType == RAFT_MSG_INTERNAL_ELECTION) {
+    syncRaftHandleElectionMessage(pRaft, pMsg);
+  } else if (msgType == RAFT_MSG_VOTE) {
+    syncRaftHandleVoteMessage(pRaft, pMsg);
+  } else {
+    pRaft->stepFp(pRaft, pMsg);
+  }
+
+  syncFreeMessage(pMsg);
+  return 0;
+}
+
+int32_t syncRaftTick(SSyncRaft* pRaft) {
+  pRaft->currentTick += 1;
+  return 0;
+}
+
+/**
+ * pre-handle message, return true means no need to continue
+ * Handle the message term, which may result in our stepping down to a follower.
+ **/
+static bool preHandleMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  // local message?
+  if (pMsg->term == 0) {
+    return false;
+  }
+
+  if (pMsg->term > pRaft->term) {
+    return preHandleNewTermMessage(pRaft, pMsg);
+  } else if (pMsg->term < pRaft->term) {
+    return preHandleOldTermMessage(pRaft, pMsg);
+  }
+
+  return false;
+}
+
+static bool preHandleNewTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  SyncNodeId leaderId = pMsg->from;
+  RaftMessageType msgType = pMsg->msgType;
+
+  if (msgType == RAFT_MSG_VOTE) {
+    // TODO
+    leaderId = SYNC_NON_NODE_ID;
+  }
+
+  if (syncIsPreVoteMsg(pMsg)) {
+    // Never change our term in response to a PreVote
+  } else if (syncIsPreVoteRespMsg(pMsg) && !pMsg->voteResp.rejected) {
+		/**
+     * We send pre-vote requests with a term in our future. If the
+		 * pre-vote is granted, we will increment our term when we get a
+		 * quorum. If it is not, the term comes from the node that
+		 * rejected our vote so we should become a follower at the new
+		 * term.
+     **/
+  } else {
+    syncInfo("[%d:%d] [term:%" PRId64 "] received a %d message with higher term from %d [term:%" PRId64 "]",
+      pRaft->selfGroupId, pRaft->selfId, pRaft->term, msgType, pMsg->from, pMsg->term);
+    syncRaftBecomeFollower(pRaft, pMsg->term, leaderId);
+  }
+
+  return false;
+}
+
+static bool preHandleOldTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  if (pRaft->checkQuorum && pMsg->msgType == RAFT_MSG_APPEND) {
+		/**
+     * We have received messages from a leader at a lower term. It is possible
+		 * that these messages were simply delayed in the network, but this could
+		 * also mean that this node has advanced its term number during a network
+		 * partition, and it is now unable to either win an election or to rejoin
+	   * the majority on the old term. If checkQuorum is false, this will be
+		 * handled by incrementing term numbers in response to MsgVote with a
+		 * higher term, but if checkQuorum is true we may not advance the term on
+		 * MsgVote and must generate other messages to advance the term. The net
+		 * result of these two features is to minimize the disruption caused by
+		 * nodes that have been removed from the cluster's configuration: a
+		 * removed node will send MsgVotes (or MsgPreVotes) which will be ignored,
+		 * but it will not receive MsgApp or MsgHeartbeat, so it will not create
+		 * disruptive term increases
+    **/
+    int peerIndex = syncRaftConfigurationIndexOfNode(pRaft, pMsg->from);
+    if (peerIndex < 0) {
+      return true;
+    }
+    SSyncMessage* msg = syncNewEmptyAppendRespMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+    if (msg == NULL) {
+      return true;
+    }
+
+    pRaft->io.send(msg, &(pRaft->cluster.nodeInfo[peerIndex]));
+  } else {
+    // ignore other cases
+    syncInfo("[%d:%d] [term:%" PRId64 "] ignored a %d message with lower term from %d [term:%" PRId64 "]",
+      pRaft->selfGroupId, pRaft->selfId, pRaft->term, pMsg->msgType, pMsg->from, pMsg->term);    
+  }
+
+  return true;
+}
\ No newline at end of file

source/libs/sync/src/raft_configuration.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft_configuration.h"
+#include "raft.h"
+
+int syncRaftConfigurationIndexOfNode(SSyncRaft *pRaft, SyncNodeId id) {
+  return (int)(id);
+}
+
+int syncRaftConfigurationVoterCount(SSyncRaft *pRaft) {
+  return pRaft->cluster.replica;
+}
\ No newline at end of file

source/libs/sync/src/raft_election.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncInt.h"
+#include "raft.h"
+#include "raft_log.h"
+#include "raft_message.h"
+
+void syncRaftStartElection(SSyncRaft* pRaft, SyncRaftElectionType cType) {
+  SyncTerm term;
+  bool preVote;
+  RaftMessageType voteMsgType;
+
+  if (syncRaftIsPromotable(pRaft)) {
+    syncDebug("[%d:%d] is unpromotable; campaign() should have been called", pRaft->selfGroupId, pRaft->selfId);
+    return 0;
+  }
+
+  if (cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION) {
+    syncRaftBecomePreCandidate(pRaft);
+    preVote = true;
+    // PreVote RPCs are sent for the next term before we've incremented r.Term.
+    term = pRaft->term + 1;
+  } else {
+    syncRaftBecomeCandidate(pRaft);
+    voteMsgType = RAFT_MSG_VOTE;
+    term = pRaft->term;
+    preVote = false;
+  }
+
+  int quorum = syncRaftQuorum(pRaft);
+  SSyncRaftVoteResult result = syncRaftPollVote(pRaft, pRaft->selfId, preVote, true, NULL, NULL);
+  if (result == SYNC_RAFT_VOTE_WON) {
+		/**
+     * We won the election after voting for ourselves (which must mean that
+		 * this is a single-node cluster). Advance to the next state.
+     **/
+    if (cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION) {
+      syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION);
+    } else {
+      syncRaftBecomeLeader(pRaft);
+    }
+    return;
+  }
+
+  // broadcast vote message to other peers
+  int i;
+  SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log);
+  SyncTerm lastTerm = syncRaftLogLastTerm(pRaft->log);
+  for (i = 0; i < pRaft->cluster.replica; ++i) {
+    if (i == pRaft->cluster.selfIndex) {
+      continue;
+    }
+
+    SyncNodeId nodeId = pRaft->cluster.nodeInfo[i].nodeId;
+
+    SSyncMessage* pMsg = syncNewVoteMsg(pRaft->selfGroupId, pRaft->selfId, 
+                                        term, cType, lastIndex, lastTerm);
+    if (pMsg == NULL) {
+      continue;
+    }
+
+    syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 "] sent %d request to %d at term %" PRId64 "", 
+      pRaft->selfGroupId, pRaft->selfId, lastTerm, 
+      lastIndex, voteMsgType, nodeId, pRaft->term);
+
+    pRaft->io.send(pMsg, &(pRaft->cluster.nodeInfo[i]));
+  }
+}
\ No newline at end of file

source/libs/sync/src/raft_handle_append_entries_message.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncInt.h"
+#include "raft.h"
+#include "raft_log.h"
+#include "raft_configuration.h"
+#include "raft_message.h"
+
+int syncRaftHandleAppendEntriesMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  const RaftMsg_Append_Entries *appendEntries = &(pMsg->appendEntries);
+  
+  int peerIndex = syncRaftConfigurationIndexOfNode(pRaft, pMsg->from);
+  
+  if (peerIndex < 0) {
+    return 0;
+  }
+
+  SSyncMessage* pRespMsg = syncNewEmptyAppendRespMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+  if (pRespMsg == NULL) {
+    return 0;
+  }
+
+  RaftMsg_Append_Entries *appendResp = &(pRespMsg->appendResp);
+  // ignore committed logs
+  if (syncRaftLogIsCommitted(pRaft->log, appendEntries->index)) {
+    appendResp->index = pRaft->log->commitIndex;
+    goto out;
+  }
+
+  syncInfo("[%d:%d] recv append from %d index %" PRId64"",
+      pRaft->selfGroupId, pRaft->selfId, pMsg->from, appendEntries->index); 
+
+out:
+  pRaft->io.send(pRespMsg, &(pRaft->cluster.nodeInfo[peerIndex]));
+  return 0;
+}
\ No newline at end of file

source/libs/sync/src/raft_handle_election_message.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncInt.h"
+#include "raft.h"
+#include "raft_log.h"
+#include "raft_message.h"
+
+int syncRaftHandleElectionMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  if (pRaft->state == TAOS_SYNC_STATE_LEADER) {
+    syncDebug("[%d:%d] ignoring RAFT_MSG_INTERNAL_ELECTION because already leader", pRaft->selfGroupId, pRaft->selfId);
+    return 0;
+  }
+
+  if (!syncRaftIsPromotable(pRaft)) {
+    syncDebug("[%d:%d] is unpromotable and can not campaign", pRaft->selfGroupId, pRaft->selfId);
+    return 0;
+  }
+  // if there is pending uncommitted config,cannot start election
+  if (syncRaftLogNumOfPendingConf(pRaft->log) > 0 && syncRaftHasUnappliedLog(pRaft->log)) {
+    syncWarn("[%d:%d] cannot syncRaftStartElection at term %" PRId64 " since there are still pending configuration changes to apply",
+      pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+    return 0;
+  }
+
+  syncInfo("[%d:%d] is starting a new election at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+
+  if (pRaft->preVote) {
+    syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_PRE_ELECTION);
+  } else {
+    syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION);
+  }
+
+  return 0;
+}

source/libs/sync/src/raft_handle_vote_message.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncInt.h"
+#include "raft.h"
+#include "raft_configuration.h"
+#include "raft_log.h"
+#include "raft_message.h"
+
+static bool canGrantVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+
+int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  SSyncMessage* pRespMsg;
+  int voteIndex = syncRaftConfigurationIndexOfNode(pRaft, pMsg->from);
+  if (voteIndex == -1) {
+    return 0;
+  }
+  bool grant;
+  SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log);
+  SyncTerm lastTerm = syncRaftLogLastTerm(pRaft->log);
+
+  grant = canGrantVoteMessage(pRaft, pMsg);
+  pRespMsg = syncNewVoteRespMsg(pRaft->selfGroupId, pRaft->selfId, pMsg->vote.cType, !grant);
+  if (pRespMsg == NULL) {
+    return 0;
+  }
+  syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d"\    
+    "[logterm: %" PRId64 ", index: %" PRId64 "] at term %" PRId64 "",
+    pRaft->selfGroupId, pRaft->selfId, lastTerm, lastIndex, pRaft->voteFor,
+    grant ? "grant" : "reject",
+    pMsg->from, pMsg->vote.lastTerm, pMsg->vote.lastIndex, pRaft->term);
+
+  pRaft->io.send(pRespMsg, &(pRaft->cluster.nodeInfo[voteIndex]));
+  return 0;
+}
+
+static bool canGrantVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  if (!(pRaft->voteFor == SYNC_NON_NODE_ID || pMsg->term > pRaft->term || pRaft->voteFor == pMsg->from)) {
+    return false;
+  }
+  if (!syncRaftLogIsUptodate(pRaft->log, pMsg->vote.lastIndex, pMsg->vote.lastTerm)) {
+    return false;
+  }
+
+  return true;
+}
\ No newline at end of file

source/libs/sync/src/raft_handle_vote_resp_message.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncInt.h"
+#include "raft.h"
+#include "raft_configuration.h"
+#include "raft_message.h"
+
+int syncRaftHandleVoteRespMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  int granted, rejected;
+  int quorum;
+  int voterIndex;
+
+  assert(pRaft->state == TAOS_SYNC_STATE_CANDIDATE);
+
+  voterIndex = syncRaftConfigurationIndexOfNode(pRaft, pMsg->from);
+  if (voterIndex == -1) {
+    syncError("[%d:%d] recv vote resp from unknown server %d", pRaft->selfGroupId, pRaft->selfId, pMsg->from);
+    return 0;
+  }
+
+  if (pRaft->state != TAOS_SYNC_STATE_CANDIDATE) {
+    syncError("[%d:%d] is not candidate, ignore vote resp", pRaft->selfGroupId, pRaft->selfId);
+    return 0;
+  }
+
+  SSyncRaftVoteResult result = syncRaftPollVote(pRaft, pMsg->from, 
+                                pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION,
+                                !pMsg->voteResp.rejected, &rejected, &granted);
+
+  syncInfo("[%d:%d] [quorum:%d] has received %d votes and %d vote rejections",
+    pRaft->selfGroupId, pRaft->selfId, quorum, granted, rejected);
+
+  if (result == SYNC_RAFT_VOTE_WON) {
+    if (pRaft->candidateState.inPreVote) {
+      syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION);
+    } else {
+      syncRaftBecomeLeader(pRaft);
+
+    }
+  } else if (result == SYNC_RAFT_VOTE_LOST) {
+    syncRaftBecomeFollower(pRaft, pRaft->term, SYNC_NON_NODE_ID);
+  }
+
+  return 0;
+}
\ No newline at end of file

source/libs/sync/src/raft_log.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft_log.h"
+
+SSyncRaftLog* syncRaftLogOpen() {
+  return NULL;
+}
+
+SyncIndex syncRaftLogLastIndex(SSyncRaftLog* pLog) {
+  return 0;
+}
+
+SyncIndex syncRaftLogSnapshotIndex(SSyncRaftLog* pLog) {
+  return 0;
+}
+
+SyncTerm syncRaftLogLastTerm(SSyncRaftLog* pLog) {
+  return 0;
+}
+
+bool syncRaftLogIsUptodate(SSyncRaftLog* pLog, SyncIndex index, SyncTerm term) {
+  return true;
+}
+
+int syncRaftLogNumOfPendingConf(SSyncRaftLog* pLog) {
+  return 0;
+}
+
+bool syncRaftHasUnappliedLog(SSyncRaftLog* pLog) {
+  return pLog->commitIndex > pLog->appliedIndex;
+}
+
+SyncTerm syncRaftLogTermOf(SSyncRaftLog* pLog, SyncIndex index) {
+  return SYNC_NON_TERM;
+}
+
+int syncRaftLogAppend(SSyncRaftLog* pLog, SSyncRaftEntry *pEntries, int n) {
+
+}
+
+int syncRaftLogAcquire(SSyncRaftLog* pLog, SyncIndex index, int maxMsgSize,
+                      SSyncRaftEntry **ppEntries, int *n) {
+  return 0;
+}
+
+void syncRaftLogRelease(SSyncRaftLog* pLog, SyncIndex index,
+                      SSyncRaftEntry *pEntries, int n) {
+  return;
+}
\ No newline at end of file

source/libs/sync/src/raft_message.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft_message.h"
+
+void syncFreeMessage(const SSyncMessage* pMsg) {
+  if (!syncIsInternalMsg(pMsg->msgType)) {
+    free((SSyncMessage*)pMsg);
+  }
+}
\ No newline at end of file

source/libs/sync/src/raft_replication.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft.h"
+#include "raft_log.h"
+#include "sync_raft_progress.h"
+#include "raft_replication.h"
+
+static int sendSnapshot(SSyncRaft* pRaft, int i);
+static int sendAppendEntries(SSyncRaft* pRaft, int i, SyncIndex index, SyncTerm term);
+
+int syncRaftReplicate(SSyncRaft* pRaft, int i) {
+#if 0
+  assert(pRaft->state == TAOS_SYNC_STATE_LEADER);
+  assert(i >= 0 && i < pRaft->leaderState.nProgress);
+
+  SyncNodeId nodeId = pRaft->cluster.nodeInfo[i].nodeId;
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+  if (syncRaftProgressIsPaused(progress)) {
+    syncInfo("node %d paused", nodeId);
+    return 0;
+  }
+
+  SyncIndex nextIndex = syncRaftProgressNextIndex(progress);
+  SyncIndex snapshotIndex = syncRaftLogSnapshotIndex(pRaft->log);
+  bool inSnapshot = syncRaftProgressInSnapshot(progress);
+  SyncIndex prevIndex;
+  SyncTerm prevTerm;
+
+  /**
+   * From Section 3.5:
+   *
+   *   When sending an AppendEntries RPC, the leader includes the index and
+   *   term of the entry in its log that immediately precedes the new
+   *   entries. If the follower does not find an entry in its log with the
+   *   same index and term, then it refuses the new entries. The consistency
+   *   check acts as an induction step: the initial empty state of the logs
+   *   satisfies the Log Matching Property, and the consistency check
+   *   preserves the Log Matching Property whenever logs are extended. As a
+   *   result, whenever AppendEntries returns successfully, the leader knows
+   *   that the follower's log is identical to its own log up through the new
+   *   entries (Log Matching Property in Figure 3.2).
+   **/
+  if (nextIndex == 1) {
+    /**
+     * We're including the very first entry, so prevIndex and prevTerm are
+     * null. If the first entry is not available anymore, send the last
+     * snapshot if we're not already sending one. 
+     **/
+    if (snapshotIndex > 0 && !inSnapshot) {
+      goto send_snapshot;
+    }
+
+    // otherwise send append entries from start
+    prevIndex = 0;
+    prevTerm = 0;    
+  } else {
+    /**
+     * Set prevIndex and prevTerm to the index and term of the entry at
+     * nextIndex - 1. 
+     **/ 
+    prevIndex = nextIndex - 1;
+    prevTerm = syncRaftLogTermOf(pRaft->log, prevIndex);
+    /**
+     * If the entry is not anymore in our log, send the last snapshot if we're
+     * not doing so already.
+     **/
+    if (prevTerm == SYNC_NON_TERM && !inSnapshot) {
+      goto send_snapshot;
+    }
+  }
+
+  /* Send empty AppendEntries RPC when installing a snaphot */
+  if (inSnapshot) {
+    prevIndex = syncRaftLogLastIndex(pRaft->log);
+    prevTerm = syncRaftLogLastTerm(pRaft->log);
+  }
+
+  return sendAppendEntries(pRaft, i, prevIndex, prevTerm);
+
+send_snapshot:
+  if (syncRaftProgressRecentActive(progress)) {
+    /* Only send a snapshot when we have heard from the server */
+    return sendSnapshot(pRaft, i);
+  } else {
+    /* Send empty AppendEntries RPC when we haven't heard from the server */
+    prevIndex = syncRaftLogLastIndex(pRaft->log);
+    prevTerm  = syncRaftLogLastTerm(pRaft->log);
+    return sendAppendEntries(pRaft, i, prevIndex, prevTerm);
+  }
+#endif
+  return 0;
+}
+
+static int sendSnapshot(SSyncRaft* pRaft, int i) {
+  return 0;
+}
+
+static int sendAppendEntries(SSyncRaft* pRaft, int i, SyncIndex prevIndex, SyncTerm prevTerm) {
+#if 0
+  SyncIndex nextIndex = prevIndex + 1;
+  SSyncRaftEntry *entries;
+  int nEntry;
+  SNodeInfo* pNode = &(pRaft->cluster.nodeInfo[i]);
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+  syncRaftLogAcquire(pRaft->log, nextIndex, pRaft->maxMsgSize, &entries, &nEntry);
+
+  SSyncMessage* msg = syncNewAppendMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term,
+                                      prevIndex, prevTerm, pRaft->log->commitIndex,
+                                      nEntry, entries);
+
+  if (msg == NULL) {
+    goto err_release_log;
+  }
+
+  pRaft->io.send(msg, pNode);
+
+  if (syncRaftProgressInReplicate(progress)) {
+    SyncIndex lastIndex = nextIndex + nEntry;
+    syncRaftProgressOptimisticNextIndex(progress, lastIndex);
+    syncRaftInflightAdd(&progress->inflights, lastIndex);
+  } else if (syncRaftProgressInProbe(progress)) {
+    syncRaftProgressPause(progress);
+  } else {
+
+  }
+
+  syncRaftProgressUpdateSendTick(progress, pRaft->currentTick);
+
+  return 0;
+
+err_release_log:
+  syncRaftLogRelease(pRaft->log, nextIndex, entries, nEntry);
+#endif
+  return 0;
+}
\ No newline at end of file

source/libs/sync/src/raft_unstable_log.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync.h"
+#include "raft_unstable_log.h"
+
+/*
+SyncIndex syncRaftLogLastIndex(SSyncRaftUnstableLog* pLog) {
+  return 0;
+}
+*/
\ No newline at end of file

source/libs/sync/src/sync.c

@@ -13,14 +13,290 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sync.h"
+#include "syncInt.h"
+#include "trpc.h"
+#include "ttimer.h"
 
-int32_t syncInit() { return 0; }
+SSyncManager* gSyncManager = NULL;
 
-void syncCleanUp() {}
+#define SYNC_TICK_TIMER 50
+#define SYNC_ACTIVITY_TIMER 5
+#define SYNC_SERVER_WORKER 2
 
-SSyncNode* syncStart(const SSyncInfo* pInfo) { return NULL; }
+static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet);
+static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
 
-void syncStop(const SSyncNode* pNode) {}
+static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg);
+static int syncInitRpcClient(SSyncManager* syncManager);
+static int syncOpenWorkerPool(SSyncManager* syncManager);
+static int syncCloseWorkerPool(SSyncManager* syncManager);
+static void *syncWorkerMain(void *argv);
+static void syncNodeTick(void *param, void *tmrId);
 
-void syncReconfig(const SSyncNode* pNode, const SSyncCluster* pCfg) {}
\ No newline at end of file
+int32_t syncInit() { 
+  if (gSyncManager != NULL) {
+    return 0;
+  }
+
+  gSyncManager = (SSyncManager*)calloc(sizeof(SSyncManager), 0);
+  if (gSyncManager == NULL) {
+    syncError("malloc SSyncManager fail");
+    return -1;
+  }
+
+  pthread_mutex_init(&gSyncManager->mutex, NULL);
+
+  // init client rpc
+  if (syncInitRpcClient(gSyncManager) != 0) {
+    syncCleanUp();
+    return -1;
+  }
+
+  // init sync timer manager
+  gSyncManager->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC");
+  if (gSyncManager->syncTimerManager == NULL) {
+    syncCleanUp();
+    return -1;
+  }
+
+  // init worker pool
+  if (syncOpenWorkerPool(gSyncManager) != 0) {
+    syncCleanUp();
+    return -1;
+  }
+
+  // init vgroup hash table
+  gSyncManager->vgroupTable = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (gSyncManager->vgroupTable == NULL) {
+    syncCleanUp();
+    return -1;
+  }
+  return 0; 
+}
+
+void syncCleanUp() {
+  if (gSyncManager == NULL) {
+    return;
+  }
+  pthread_mutex_lock(&gSyncManager->mutex);
+  if (gSyncManager->vgroupTable) {
+    taosHashCleanup(gSyncManager->vgroupTable);
+  }
+  if (gSyncManager->clientRpc) {
+    rpcClose(gSyncManager->clientRpc);
+    syncInfo("sync inter-sync rpc client is closed");
+  }
+  if (gSyncManager->syncTimerManager) {
+    taosTmrCleanUp(gSyncManager->syncTimerManager);
+  }
+  syncCloseWorkerPool(gSyncManager);
+  pthread_mutex_unlock(&gSyncManager->mutex);
+  pthread_mutex_destroy(&gSyncManager->mutex);
+  free(gSyncManager);
+  gSyncManager = NULL;
+}
+
+SSyncNode* syncStart(const SSyncInfo* pInfo) {
+  pthread_mutex_lock(&gSyncManager->mutex);
+
+  SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &pInfo->vgId, sizeof(SyncGroupId));
+  if (ppNode != NULL) {
+    syncInfo("vgroup %d already exist", pInfo->vgId);
+    pthread_mutex_unlock(&gSyncManager->mutex);
+    return *ppNode;
+  }
+
+  // init rpc server
+  if (syncInitRpcServer(gSyncManager, &pInfo->syncCfg) != 0) {
+    pthread_mutex_unlock(&gSyncManager->mutex);
+    return NULL;
+  }
+
+  SSyncNode *pNode = (SSyncNode*)malloc(sizeof(SSyncNode));
+  if (pNode == NULL) {
+    syncError("malloc vgroup %d node fail", pInfo->vgId);
+    pthread_mutex_unlock(&gSyncManager->mutex);
+    return NULL;
+  }
+
+  pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pInfo->vgId, gSyncManager->syncTimerManager);
+
+  // start raft
+  pNode->raft.pNode = pNode;
+  if (syncRaftStart(&pNode->raft, pInfo) != 0) {
+    syncError("raft start at %d node fail", pInfo->vgId);
+    pthread_mutex_unlock(&gSyncManager->mutex);
+    return NULL;
+  }
+
+  pthread_mutex_init(&pNode->mutex, NULL);
+
+  taosHashPut(gSyncManager->vgroupTable, &pInfo->vgId, sizeof(SyncGroupId), &pNode, sizeof(SSyncNode *));
+
+  pthread_mutex_unlock(&gSyncManager->mutex);
+  return NULL;
+}
+
+void syncStop(const SSyncNode* pNode) {
+  pthread_mutex_lock(&gSyncManager->mutex);
+
+  SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &pNode->vgId, sizeof(SyncGroupId));
+  if (ppNode == NULL) {
+    syncInfo("vgroup %d not exist", pNode->vgId);
+    pthread_mutex_unlock(&gSyncManager->mutex);
+    return;
+  }
+  assert(*ppNode == pNode);
+  taosTmrStop(pNode->syncTimer);
+
+  taosHashRemove(gSyncManager->vgroupTable, &pNode->vgId, sizeof(SyncGroupId));
+  pthread_mutex_unlock(&gSyncManager->mutex);
+
+  pthread_mutex_destroy(&((*ppNode)->mutex));
+  free(*ppNode);
+}
+
+int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak) {
+  SSyncMessage msg;
+
+  pthread_mutex_lock(&syncNode->mutex);
+  int32_t ret = syncRaftStep(&syncNode->raft, syncInitPropMsg(&msg, pBuf, pData, isWeak));
+  pthread_mutex_unlock(&syncNode->mutex);
+  return ret;
+}
+
+void syncReconfig(const SSyncNode* pNode, const SSyncCluster* pCfg) {}
+
+int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode) {
+  return 0;
+}
+
+int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode) {
+  return 0;
+}
+
+// process rpc rsp message from other sync server
+static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet) {
+
+}
+
+// process rpc message from other sync server
+static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
+
+}
+
+static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg) {
+  if (gSyncManager->rpcServerTable == NULL) {
+    gSyncManager->rpcServerTable = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    if (gSyncManager->rpcServerTable == NULL) {
+      syncError("init sync rpc server hash table error");
+      return -1;
+    }
+  }
+  assert(pSyncCfg->selfIndex < pSyncCfg->replica && pSyncCfg->selfIndex >= 0);
+  const SNodeInfo* pNode = &(pSyncCfg->nodeInfo[pSyncCfg->replica]);
+  char buffer[20] = {'\0'};
+  snprintf(buffer, sizeof(buffer), "%s:%d", &(pNode->nodeFqdn[0]), pNode->nodePort);
+  size_t len = strlen(buffer);
+  void** ppRpcServer = taosHashGet(gSyncManager->rpcServerTable, buffer, len);
+  if (ppRpcServer != NULL) {
+    // already inited
+    syncInfo("sync rpc server for %s already exist", buffer);
+    return 0;
+  }
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = pNode->nodePort;
+  rpcInit.label        = "sync-server";
+  rpcInit.numOfThreads = SYNC_SERVER_WORKER;
+  rpcInit.cfp          = syncProcessReqMsg;
+  rpcInit.sessions     = TSDB_MAX_VNODES << 4;
+  rpcInit.connType     = TAOS_CONN_SERVER;
+  rpcInit.idleTime     = SYNC_ACTIVITY_TIMER * 1000;
+
+  void* rpcServer = rpcOpen(&rpcInit);
+  if (rpcServer == NULL) {
+    syncInfo("rpcOpen for sync rpc server for %s fail", buffer);
+    return -1;
+  }
+
+  taosHashPut(gSyncManager->rpcServerTable, buffer, strlen(buffer), rpcServer, len);
+  syncInfo("sync rpc server for %s init success", buffer);
+
+  return 0;
+}
+
+static int syncInitRpcClient(SSyncManager* syncManager) {
+  char secret[TSDB_KEY_LEN] = "secret";
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.label        = "sync-client";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = syncProcessRsp;
+  rpcInit.sessions     = TSDB_MAX_VNODES << 4;
+  rpcInit.connType     = TAOS_CONN_CLIENT;
+  rpcInit.idleTime     = SYNC_ACTIVITY_TIMER * 1000;
+  rpcInit.user         = "t";
+  rpcInit.ckey         = "key";
+  rpcInit.secret       = secret;
+
+  syncManager->clientRpc = rpcOpen(&rpcInit);
+  if (syncManager->clientRpc == NULL) {
+    syncError("failed to init sync rpc client");
+    return -1;
+  }
+
+  syncInfo("sync inter-sync rpc client is initialized");
+  return 0;
+}
+
+static int syncOpenWorkerPool(SSyncManager* syncManager) {
+  int i;
+  pthread_attr_t thattr;
+
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); 
+
+  for (i = 0; i < TAOS_SYNC_MAX_WORKER; ++i) {
+    SSyncWorker* pWorker = &(syncManager->worker[i]);
+
+    if (pthread_create(&(pWorker->thread), &thattr, (void *)syncWorkerMain, pWorker) != 0) {
+      syncError("failed to create sync worker since %s", strerror(errno));
+
+      return -1;
+    }
+  }
+
+  pthread_attr_destroy(&thattr);
+
+  return 0;
+}
+
+static int syncCloseWorkerPool(SSyncManager* syncManager) {
+  return 0;
+}
+
+static void *syncWorkerMain(void *argv) {
+  SSyncWorker* pWorker = (SSyncWorker *)argv;
+
+  taosBlockSIGPIPE();
+  setThreadName("syncWorker");
+
+  return NULL;
+}
+
+static void syncNodeTick(void *param, void *tmrId) {
+  SyncGroupId vgId = (SyncGroupId)param;
+  SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &vgId, sizeof(SyncGroupId));
+  if (ppNode == NULL) {
+    return;
+  }
+  SSyncNode *pNode = *ppNode;
+
+  pthread_mutex_lock(&pNode->mutex);
+  syncRaftTick(&pNode->raft);
+  pthread_mutex_unlock(&pNode->mutex);
+
+  pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pNode->vgId, gSyncManager->syncTimerManager);
+}
\ No newline at end of file

source/libs/sync/src/sync_raft_impl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft.h"
+#include "raft_configuration.h"
+#include "raft_log.h"
+#include "raft_replication.h"
+#include "sync_raft_progress_tracker.h"
+#include "syncInt.h"
+
+static int convertClear(SSyncRaft* pRaft);
+static int stepFollower(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+static int stepCandidate(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+static int stepLeader(SSyncRaft* pRaft, const SSyncMessage* pMsg);
+
+static int triggerAll(SSyncRaft* pRaft);
+
+static void tickElection(SSyncRaft* pRaft);
+static void tickHeartbeat(SSyncRaft* pRaft);
+
+static void appendEntries(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n);
+static bool maybeCommit(SSyncRaft* pRaft);
+
+static void abortLeaderTransfer(SSyncRaft* pRaft);
+
+static void resetRaft(SSyncRaft* pRaft, SyncTerm term);
+
+void syncRaftBecomeFollower(SSyncRaft* pRaft, SyncTerm term, SyncNodeId leaderId) {
+  convertClear(pRaft);
+
+  pRaft->stepFp = stepFollower;
+  resetRaft(pRaft, term);
+  pRaft->tickFp = tickElection;
+  pRaft->leaderId = leaderId;
+  pRaft->state = TAOS_SYNC_STATE_FOLLOWER;
+  syncInfo("[%d:%d] became followe at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+}
+
+void syncRaftBecomePreCandidate(SSyncRaft* pRaft) {
+  convertClear(pRaft);
+
+	/**
+   * Becoming a pre-candidate changes our step functions and state,
+	 * but doesn't change anything else. In particular it does not increase
+	 * r.Term or change r.Vote.
+   **/
+  pRaft->stepFp = stepCandidate;
+  pRaft->tickFp = tickElection;
+  pRaft->state  = TAOS_SYNC_STATE_CANDIDATE;
+  pRaft->candidateState.inPreVote = true;
+  syncInfo("[%d:%d] became pre-candidate at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+}
+
+void syncRaftBecomeCandidate(SSyncRaft* pRaft) {
+  convertClear(pRaft);
+
+  pRaft->candidateState.inPreVote = false;
+  pRaft->stepFp = stepCandidate;
+  // become candidate make term+1
+  resetRaft(pRaft, pRaft->term + 1);
+  pRaft->tickFp = tickElection;
+  pRaft->voteFor = pRaft->selfId;
+  pRaft->state  = TAOS_SYNC_STATE_CANDIDATE;
+  syncInfo("[%d:%d] became candidate at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+}
+
+void syncRaftBecomeLeader(SSyncRaft* pRaft) {
+  assert(pRaft->state != TAOS_SYNC_STATE_FOLLOWER);
+
+  pRaft->stepFp = stepLeader;
+  resetRaft(pRaft, pRaft->term);
+  pRaft->leaderId = pRaft->leaderId;
+  pRaft->state  = TAOS_SYNC_STATE_LEADER;
+  // TODO: check if there is pending config log
+  int nPendingConf = syncRaftLogNumOfPendingConf(pRaft->log);
+  if (nPendingConf > 1) {
+    syncFatal("unexpected multiple uncommitted config entry");
+  }
+
+  syncInfo("[%d:%d] became leader at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term);
+
+  // after become leader, send a no-op log
+  SSyncRaftEntry* entry = (SSyncRaftEntry*)malloc(sizeof(SSyncRaftEntry));
+  if (entry == NULL) {
+    return;
+  }
+  *entry = (SSyncRaftEntry) {
+    .buffer = (SSyncBuffer) {
+      .data = NULL,
+      .len = 0,
+    }
+  };
+  appendEntries(pRaft, entry, 1);
+  //syncRaftTriggerHeartbeat(pRaft);
+}
+
+void syncRaftTriggerHeartbeat(SSyncRaft* pRaft) {
+  triggerAll(pRaft);
+}
+
+void syncRaftRandomizedElectionTimeout(SSyncRaft* pRaft) {
+  // electionTimeoutTick in [3,6] tick
+  pRaft->randomizedElectionTimeout = taosRand() % 4 + 3;
+}
+
+bool syncRaftIsPromotable(SSyncRaft* pRaft) {
+  return pRaft->selfId != SYNC_NON_NODE_ID;
+}
+
+bool syncRaftIsPastElectionTimeout(SSyncRaft* pRaft) {
+  return pRaft->electionElapsed >= pRaft->randomizedElectionTimeout;
+}
+
+int syncRaftQuorum(SSyncRaft* pRaft) {
+  return pRaft->cluster.replica / 2 + 1;
+}
+
+SSyncRaftVoteResult  syncRaftPollVote(SSyncRaft* pRaft, SyncNodeId id, 
+                                      bool preVote, bool grant, 
+                                      int* rejected, int *granted) {
+  int voterIndex = syncRaftConfigurationIndexOfNode(pRaft, id);
+  if (voterIndex == -1) {
+    return SYNC_RAFT_VOTE_PENDING;
+  }
+
+  if (grant) {
+    syncInfo("[%d:%d] received grant (pre-vote %d) from %d at term %" PRId64 "", 
+      pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term);
+  } else {
+    syncInfo("[%d:%d] received rejection (pre-vote %d) from %d at term %" PRId64 "", 
+      pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term);
+  }
+
+  syncRaftRecordVote(pRaft->tracker, voterIndex, grant);
+  return syncRaftTallyVotes(pRaft->tracker, rejected, granted);
+}
+/*
+  if (accept) {
+    syncInfo("[%d:%d] received (pre-vote %d) from %d at term %" PRId64 "", 
+      pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term);
+  } else {
+    syncInfo("[%d:%d] received rejection from %d at term %" PRId64 "", 
+      pRaft->selfGroupId, pRaft->selfId, id, pRaft->term);
+  }
+  
+  int voteIndex = syncRaftConfigurationIndexOfNode(pRaft, id);
+  assert(voteIndex < pRaft->cluster.replica && voteIndex >= 0);
+  assert(pRaft->candidateState.votes[voteIndex] == SYNC_RAFT_VOTE_RESP_UNKNOWN);
+
+  pRaft->candidateState.votes[voteIndex] = accept ? SYNC_RAFT_VOTE_RESP_GRANT : SYNC_RAFT_VOTE_RESP_REJECT;
+  int granted = 0, rejected = 0;
+  int i;
+  for (i = 0; i < pRaft->cluster.replica; ++i) {
+    if (pRaft->candidateState.votes[i] == SYNC_RAFT_VOTE_RESP_GRANT) granted++;
+    else if (pRaft->candidateState.votes[i] == SYNC_RAFT_VOTE_RESP_REJECT) rejected++;
+  }
+
+  if (rejectNum) *rejectNum = rejected;
+  return granted;
+*/
+
+static int convertClear(SSyncRaft* pRaft) {
+
+}
+
+static int stepFollower(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  
+  return 0;
+}
+
+static int stepCandidate(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  /**
+   * Only handle vote responses corresponding to our candidacy (while in
+	 * StateCandidate, we may get stale MsgPreVoteResp messages in this term from
+	 * our pre-candidate state).
+   **/
+  RaftMessageType msgType = pMsg->msgType;
+
+  if (msgType == RAFT_MSG_INTERNAL_PROP) {
+    return 0;
+  }
+
+  if (msgType == RAFT_MSG_VOTE_RESP) {
+    syncRaftHandleVoteRespMessage(pRaft, pMsg);
+    return 0;
+  } else if (msgType == RAFT_MSG_APPEND) {
+    syncRaftBecomeFollower(pRaft, pRaft->term, pMsg->from);
+    syncRaftHandleAppendEntriesMessage(pRaft, pMsg);
+  }
+  return 0;
+}
+
+static int stepLeader(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
+  convertClear(pRaft);
+  return 0;
+}
+
+/**
+ *  tickElection is run by followers and candidates per tick.
+ **/
+static void tickElection(SSyncRaft* pRaft) {
+  pRaft->electionElapsed += 1;
+
+  if (!syncRaftIsPromotable(pRaft)) {
+    return;
+  }
+
+  if (!syncRaftIsPastElectionTimeout(pRaft)) {
+    return;
+  }
+
+  // election timeout
+  pRaft->electionElapsed = 0;
+  SSyncMessage msg;
+  syncRaftStep(pRaft, syncInitElectionMsg(&msg, pRaft->selfId));
+}
+
+static void tickHeartbeat(SSyncRaft* pRaft) {
+
+}
+
+static void appendEntries(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n) {
+  SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log);
+  SyncTerm term = pRaft->term;
+  int i;
+
+  for (i = 0; i < n; ++i) {
+    entries[i].term = term;
+    entries[i].index = lastIndex + 1 + i;
+  }
+
+  syncRaftLogAppend(pRaft->log, entries, n);
+
+  SSyncRaftProgress* progress = &(pRaft->tracker->progressMap[pRaft->cluster.selfIndex]);
+  syncRaftProgressMaybeUpdate(progress, lastIndex);
+  // Regardless of maybeCommit's return, our caller will call bcastAppend.
+  maybeCommit(pRaft);
+}
+
+/**
+ * maybeCommit attempts to advance the commit index. Returns true if
+ * the commit index changed (in which case the caller should call
+ * r.bcastAppend).
+ **/
+static bool maybeCommit(SSyncRaft* pRaft) {
+  
+  return true;
+}
+
+/**
+ * trigger I/O requests for newly appended log entries or heartbeats.
+ **/
+static int triggerAll(SSyncRaft* pRaft) {
+  assert(pRaft->state == TAOS_SYNC_STATE_LEADER);
+  int i;
+
+  for (i = 0; i < pRaft->cluster.replica; ++i) {
+    if (i == pRaft->cluster.selfIndex) {
+      continue;
+    }
+
+    syncRaftReplicate(pRaft, i);
+  }
+}
+
+static void abortLeaderTransfer(SSyncRaft* pRaft) {
+  pRaft->leadTransferee = SYNC_NON_NODE_ID;
+}
+
+static void initProgress(int i, SSyncRaftProgress* progress, void* arg) {
+  syncRaftInitProgress(i, (SSyncRaft*)arg, progress);
+}
+
+static void resetRaft(SSyncRaft* pRaft, SyncTerm term) {
+  if (pRaft->term != term) {
+    pRaft->term = term;
+    pRaft->voteFor = SYNC_NON_NODE_ID;
+  }
+
+  pRaft->leaderId = SYNC_NON_NODE_ID;
+
+  pRaft->electionElapsed = 0;
+  pRaft->heartbeatElapsed = 0;
+
+  syncRaftRandomizedElectionTimeout(pRaft);
+
+  abortLeaderTransfer(pRaft);
+
+  syncRaftResetVotes(pRaft->tracker);
+  syncRaftProgressVisit(pRaft->tracker, initProgress, pRaft);
+
+  pRaft->pendingConfigIndex = 0;
+  pRaft->uncommittedSize = 0;
+}
\ No newline at end of file

source/libs/sync/src/sync_raft_inflights.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http: *www.gnu.org/licenses/>.
+ */
+
+#include "sync_raft_inflights.h"
+
+SSyncRaftInflights* syncRaftOpenInflights(int size) {
+  SSyncRaftInflights* inflights = (SSyncRaftInflights*)malloc(sizeof(SSyncRaftInflights));
+  if (inflights == NULL) {
+    return NULL;
+  }
+  SyncIndex* buffer = (SyncIndex*)malloc(sizeof(SyncIndex) * size);
+  if (buffer == NULL) {
+    free(inflights);
+    return NULL;
+  }
+  *inflights = (SSyncRaftInflights) {
+    .buffer = buffer,
+    .count  = 0,
+    .size   = 0,
+    .start  = 0,
+  };
+
+  return inflights;
+}
+
+void syncRaftCloseInflights(SSyncRaftInflights* inflights) {
+  free(inflights->buffer);
+  free(inflights);
+}
+
+/**
+ * syncRaftInflightAdd notifies the Inflights that a new message with the given index is being
+ * dispatched. syncRaftInflightFull() must be called prior to syncRaftInflightAdd() 
+ * to verify that there is room for one more message, 
+ * and consecutive calls to add syncRaftInflightAdd() must provide a
+ * monotonic sequence of indexes.
+ **/
+void syncRaftInflightAdd(SSyncRaftInflights* inflights, SyncIndex inflightIndex) {
+  assert(!syncRaftInflightFull(inflights));
+
+  int next = inflights->start + inflights->count;
+  int size = inflights->size;
+  /* is next wrapped around buffer? */
+  if (next >= size) {
+    next -= size;
+  }
+
+  inflights->buffer[next] = inflightIndex;
+  inflights->count++;
+}
+
+/**
+ * syncRaftInflightFreeLE frees the inflights smaller or equal to the given `to` flight.
+ **/
+void syncRaftInflightFreeLE(SSyncRaftInflights* inflights, SyncIndex toIndex) {
+  if (inflights->count == 0 || toIndex < inflights->buffer[inflights->start]) {
+    /* out of the left side of the window */
+    return;
+  }
+
+  int i, idx;
+  for (i = 0, idx = inflights->start; i < inflights->count; i++) {
+    if (toIndex < inflights->buffer[idx]) { // found the first large inflight
+      break;
+    }
+
+    // increase index and maybe rotate
+    int size = inflights->size;
+    idx++;
+    if (idx >= size) {
+      idx -= size;
+    }
+  }
+
+  // free i inflights and set new start index
+  inflights->count -= i;
+  inflights->start  = idx;
+  assert(inflights->count >= 0);
+  if (inflights->count == 0) {
+		// inflights is empty, reset the start index so that we don't grow the
+		// buffer unnecessarily.    
+    inflights->start = 0;
+  }
+}
+
+/** 
+ * syncRaftInflightFreeFirstOne releases the first inflight. 
+ * This is a no-op if nothing is inflight.
+ **/
+void syncRaftInflightFreeFirstOne(SSyncRaftInflights* inflights) {
+  syncRaftInflightFreeLE(inflights, inflights->buffer[inflights->start]);
+}

source/libs/sync/src/sync_raft_progress.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "raft.h"
+#include "raft_log.h"
+#include "sync_raft_progress.h"
+#include "sync_raft_progress_tracker.h"
+#include "sync.h"
+#include "syncInt.h"
+
+static void resetProgressState(SSyncRaftProgress* progress, RaftProgressState state);
+static void probeAcked(SSyncRaftProgress* progress);
+
+static void resumeProgress(SSyncRaftProgress* progress);
+
+void syncRaftInitProgress(int i, SSyncRaft* pRaft, SSyncRaftProgress* progress) {
+  SSyncRaftInflights* inflights = syncRaftOpenInflights(pRaft->tracker->maxInflight);
+  if (inflights == NULL) {
+    return;
+  }
+  *progress = (SSyncRaftProgress) {
+    .matchIndex = i == pRaft->selfIndex ? syncRaftLogLastIndex(pRaft->log) : 0,
+    .nextIndex  = syncRaftLogLastIndex(pRaft->log) + 1,
+    .inflights  = inflights,
+    .isLearner  = false,
+    .state      = PROGRESS_STATE_PROBE,
+  };
+}
+
+/**
+ * syncRaftProgressMaybeUpdate is called when an MsgAppResp arrives from the follower, with the
+ * index acked by it. The method returns false if the given n index comes from
+ * an outdated message. Otherwise it updates the progress and returns true.
+ **/
+bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastIndex) {
+  bool updated = false;
+
+  if (progress->matchIndex < lastIndex) {
+    progress->matchIndex = lastIndex;
+    updated = true;
+    probeAcked(progress);
+  }
+
+  progress->nextIndex = MAX(progress->nextIndex, lastIndex + 1);
+
+  return updated;
+}
+
+bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress,
+                                SyncIndex rejected, SyncIndex matchHint) {
+  if (progress->state == PROGRESS_STATE_REPLICATE) {
+		/** 
+     * the rejection must be stale if the progress has matched and "rejected"
+		 * is smaller than "match".
+     **/
+    if (rejected <= progress->matchIndex) {
+      syncDebug("match index is up to date,ignore");
+      return false;
+    }
+
+    /* directly decrease next to match + 1 */
+    progress->nextIndex = progress->matchIndex + 1;
+    return true;
+  }
+
+	/**
+   * The rejection must be stale if "rejected" does not match next - 1. This
+	 * is because non-replicating followers are probed one entry at a time.
+   **/
+  if (rejected != progress->nextIndex - 1) {
+    syncDebug("rejected index %" PRId64 " different from next index %" PRId64 " -> ignore"
+      , rejected, progress->nextIndex);
+    return false;
+  }
+
+  progress->nextIndex = MAX(MIN(rejected, matchHint + 1), 1);
+
+  progress->probeSent = false;
+  return true;
+}
+
+/**
+ * syncRaftProgressIsPaused returns whether sending log entries to this node has been throttled.
+ * This is done when a node has rejected recent MsgApps, is currently waiting
+ * for a snapshot, or has reached the MaxInflightMsgs limit. In normal
+ * operation, this is false. A throttled node will be contacted less frequently
+ * until it has reached a state in which it's able to accept a steady stream of
+ * log entries again.
+ **/
+bool syncRaftProgressIsPaused(SSyncRaftProgress* progress) {
+  switch (progress->state) {
+    case PROGRESS_STATE_PROBE:
+      return progress->probeSent;
+    case PROGRESS_STATE_REPLICATE:
+      return syncRaftInflightFull(progress->inflights);
+    case PROGRESS_STATE_SNAPSHOT:
+      return true;
+    default:
+      syncFatal("error sync state:%d", progress->state);
+  }
+}
+
+bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress) {
+  return syncRaftLogLastIndex(pRaft->log) + 1 == progress->nextIndex;
+}
+
+/**
+ * syncRaftProgressBecomeProbe transitions into StateProbe. Next is reset to Match+1 or,
+ * optionally and if larger, the index of the pending snapshot.
+ **/
+void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress) {
+  /**
+   * If the original state is ProgressStateSnapshot, progress knows that
+	 * the pending snapshot has been sent to this peer successfully, then
+	 * probes from pendingSnapshot + 1.
+   **/
+  if (progress->state == PROGRESS_STATE_SNAPSHOT) {
+    SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex;
+    resetProgressState(progress, PROGRESS_STATE_PROBE);
+    progress->nextIndex = MAX(progress->matchIndex + 1, pendingSnapshotIndex + 1);
+  } else {
+    resetProgressState(progress, PROGRESS_STATE_PROBE);
+    progress->nextIndex = progress->matchIndex + 1;
+  }
+}
+
+/**
+ * syncRaftProgressBecomeReplicate transitions into StateReplicate, resetting Next to Match+1.
+ **/
+void syncRaftProgressBecomeReplicate(SSyncRaftProgress* progress) {
+  resetProgressState(progress, PROGRESS_STATE_REPLICATE);
+  progress->nextIndex = progress->matchIndex + 1;
+}
+
+void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex) {
+  resetProgressState(progress, PROGRESS_STATE_SNAPSHOT);
+  progress->pendingSnapshotIndex = snapshotIndex;
+}
+
+/**
+ * ResetState moves the Progress into the specified State, resetting ProbeSent,
+ * PendingSnapshot, and Inflights.
+ **/
+static void resetProgressState(SSyncRaftProgress* progress, RaftProgressState state) {
+  progress->probeSent = false;
+  progress->pendingSnapshotIndex = 0;
+  progress->state = state;
+  syncRaftInflightReset(progress->inflights);
+}
+
+/**
+ * probeAcked is called when this peer has accepted an append. It resets
+ * ProbeSent to signal that additional append messages should be sent without
+ * further delay.
+ **/
+static void probeAcked(SSyncRaftProgress* progress) {
+  progress->probeSent = false;
+}
+
+#if 0
+
+SyncIndex syncRaftProgressNextIndex(SSyncRaft* pRaft, int i) {
+  return pRaft->leaderState.progress[i].nextIndex;
+}
+
+SyncIndex syncRaftProgressMatchIndex(SSyncRaft* pRaft, int i) {
+  return pRaft->leaderState.progress[i].matchIndex;
+}
+
+void syncRaftProgressUpdateLastSend(SSyncRaft* pRaft, int i) {
+  pRaft->leaderState.progress[i].lastSend = pRaft->io.time(pRaft);
+}
+
+void syncRaftProgressUpdateSnapshotLastSend(SSyncRaft* pRaft, int i) {
+  pRaft->leaderState.progress[i].lastSendSnapshot = pRaft->io.time(pRaft);
+}
+
+bool syncRaftProgressResetRecentRecv(SSyncRaft* pRaft, int i) {
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+  bool prev = progress->recentRecv;
+  progress->recentRecv = false;
+  return prev;
+}
+
+void syncRaftProgressMarkRecentRecv(SSyncRaft* pRaft, int i) {
+  pRaft->leaderState.progress[i].recentRecv = true;
+}
+
+bool syncRaftProgressGetRecentRecv(SSyncRaft* pRaft, int i) {
+  return pRaft->leaderState.progress[i].recentRecv;
+}
+
+void syncRaftProgressBecomeSnapshot(SSyncRaft* pRaft, int i) {
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+  resetProgressState(progress, PROGRESS_STATE_SNAPSHOT);
+  progress->pendingSnapshotIndex = raftLogSnapshotIndex(pRaft->log);
+}
+
+void syncRaftProgressBecomeProbe(SSyncRaft* pRaft, int i) {
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+
+  if (progress->state == PROGRESS_STATE_SNAPSHOT) {
+    assert(progress->pendingSnapshotIndex > 0);
+    SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex;
+    resetProgressState(progress, PROGRESS_STATE_PROBE);
+    progress->nextIndex = max(progress->matchIndex + 1, pendingSnapshotIndex);
+  } else {
+    resetProgressState(progress, PROGRESS_STATE_PROBE);
+    progress->nextIndex = progress->matchIndex + 1;
+  }
+}
+
+void syncRaftProgressBecomeReplicate(SSyncRaft* pRaft, int i) {
+  resetProgressState(pRaft->leaderState.progress, PROGRESS_STATE_REPLICATE);
+  pRaft->leaderState.progress->nextIndex = pRaft->leaderState.progress->matchIndex + 1;
+}
+
+void syncRaftProgressAbortSnapshot(SSyncRaft* pRaft, int i) {
+  SSyncRaftProgress* progress = &(pRaft->leaderState.progress[i]);
+  progress->pendingSnapshotIndex = 0;
+  progress->state = PROGRESS_STATE_PROBE;
+}
+
+RaftProgressState syncRaftProgressState(SSyncRaft* pRaft, int i) {
+  return pRaft->leaderState.progress[i].state;
+}
+
+
+
+#endif
\ No newline at end of file

source/libs/sync/src/sync_raft_progress_tracker.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync_raft_progress_tracker.h"
+
+SSyncRaftProgressTracker* syncRaftOpenProgressTracker() {
+  SSyncRaftProgressTracker* tracker = (SSyncRaftProgressTracker*)malloc(sizeof(SSyncRaftProgressTracker));
+  if (tracker == NULL) {
+    return NULL;
+  }
+
+  return tracker;
+}
+
+void syncRaftResetVotes(SSyncRaftProgressTracker* tracker) {
+  memset(tracker->votes, SYNC_RAFT_VOTE_RESP_UNKNOWN, sizeof(SyncRaftVoteResult) * TSDB_MAX_REPLICA);
+}
+
+void syncRaftProgressVisit(SSyncRaftProgressTracker* tracker, visitProgressFp visit, void* arg) {
+  int i;
+  for (i = 0; i < TSDB_MAX_REPLICA; ++i) {
+    SSyncRaftProgress* progress = &(tracker->progressMap[i]);
+    visit(i, progress, arg);
+  }
+}
+
+void syncRaftRecordVote(SSyncRaftProgressTracker* tracker, int i, bool grant) {
+  if (tracker->votes[i] != SYNC_RAFT_VOTE_RESP_UNKNOWN) {
+    return;
+  }
+
+  tracker->votes[i] = grant ? SYNC_RAFT_VOTE_RESP_GRANT : SYNC_RAFT_VOTE_RESP_REJECT;
+}
+
+/** 
+ * syncRaftTallyVotes returns the number of granted and rejected Votes, and whether the
+ * election outcome is known.
+ **/
+SyncRaftVoteResult syncRaftTallyVotes(SSyncRaftProgressTracker* tracker, int* rejected, int *granted) {
+  int i;
+  SSyncRaftProgress* progress;
+  int r, g;
+
+  for (i = 0, r = 0, g = 0; i < TSDB_MAX_REPLICA; ++i) {
+    progress = &(tracker->progressMap[i]);
+    if (progress->id == SYNC_NON_NODE_ID) {
+      continue;
+    }
+
+    if (tracker->votes[i] == SYNC_RAFT_VOTE_RESP_UNKNOWN) {
+      continue;
+    }
+
+    if (tracker->votes[i] == SYNC_RAFT_VOTE_RESP_GRANT) {
+      g++;
+    } else {
+      r++;
+    }
+  }
+
+  if (rejected) *rejected = r;
+  if (granted) *granted = g;
+  return syncRaftVoteResult(&(tracker->config.voters), tracker->votes);
+}
\ No newline at end of file

source/libs/sync/src/sync_raft_quorum_joint.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync_raft_quorum_majority.h"
+#include "sync_raft_quorum_joint.h"
+#include "sync_raft_quorum.h"
+
+/**
+ * syncRaftVoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+ * a result indicating whether the vote is pending, lost, or won. A joint quorum
+ * requires both majority quorums to vote in favor.
+ **/
+SyncRaftVoteResult syncRaftVoteResult(SSyncRaftQuorumJointConfig* config, const SyncRaftVoteResult* votes) {
+  SyncRaftVoteResult r1 = syncRaftMajorityVoteResult(&(config->majorityConfig[0]), votes);
+  SyncRaftVoteResult r2 = syncRaftMajorityVoteResult(&(config->majorityConfig[1]), votes);
+
+  if (r1 == r2) {
+    // If they agree, return the agreed state.
+    return r1;
+  }
+
+  if (r1 == SYNC_RAFT_VOTE_LOST || r2 == SYNC_RAFT_VOTE_LOST) {
+    // If either config has lost, loss is the only possible outcome.
+    return SYNC_RAFT_VOTE_LOST;
+  }
+
+  // One side won, the other one is pending, so the whole outcome is.
+  return SYNC_RAFT_VOTE_PENDING;
+}

source/libs/sync/src/sync_raft_quorum_majority.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync_raft_quorum.h"
+#include "sync_raft_quorum_majority.h"
+
+/**
+ * syncRaftMajorityVoteResult takes a mapping of voters to yes/no (true/false) votes and returns
+ * a result indicating whether the vote is pending (i.e. neither a quorum of
+ * yes/no has been reached), won (a quorum of yes has been reached), or lost (a
+ * quorum of no has been reached).
+ **/
+SyncRaftVoteResult syncRaftMajorityVoteResult(SSyncCluster* config, const SyncRaftVoteResult* votes) {
+  if (config->replica == 0) {
+    return SYNC_RAFT_VOTE_WON;
+  }
+
+  int i, g, r, missing;
+  for (i = g = r = missing = 0; i < TSDB_MAX_REPLICA; ++i) {
+    if (config->nodeInfo[i].nodeId == SYNC_NON_NODE_ID) {
+      continue;
+    }
+
+    if (votes[i] == SYNC_RAFT_VOTE_RESP_UNKNOWN) {
+      missing += 1;
+    } else if (votes[i] == SYNC_RAFT_VOTE_RESP_GRANT) {
+      g +=1;
+    } else {
+      r += 1;
+    }
+  }
+
+  int quorum = config->replica / 2 + 1;
+  if (g >= quorum) {
+    return SYNC_RAFT_VOTE_WON;
+  }
+  if (r + missing >= quorum) {
+    return SYNC_RAFT_VOTE_PENDING;
+  }
+
+  return SYNC_RAFT_VOTE_LOST;
+}
\ No newline at end of file

source/os/src/osDir.c

@@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) {
 
 bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; }
 
-bool taosMkDir(char *dirname) {
+bool taosMkDir(const char *dirname) {
   int32_t code = mkdir(dirname, 0755);
   if (code < 0 && errno == EEXIST) {
     return true;

source/util/src/mallocator.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mallocator.h"
+
+/* ------------------------ HEAP ALLOCATOR ------------------------ */
+typedef struct {
+  size_t tusage;
+} SHeapAllocator;
+
+static void * haMalloc(SMemAllocator *pma, size_t size);
+static void * haCalloc(SMemAllocator *pma, size_t nmemb, size_t size);
+static void * haRealloc(SMemAllocator *pma, void *ptr, size_t size);
+static void   haFree(SMemAllocator *pma, void *ptr);
+static size_t haUsage(SMemAllocator *pma);
+
+SMemAllocator *tdCreateHeapAllocator() {
+  SMemAllocator *pma = NULL;
+
+  pma = calloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator));
+  if (pma) {
+    pma->impl = POINTER_SHIFT(pma, sizeof(SMemAllocator));
+    pma->malloc = haMalloc;
+    pma->calloc = haCalloc;
+    pma->realloc = haRealloc;
+    pma->free = haFree;
+    pma->usage = haUsage;
+  }
+
+  return pma;
+}
+
+void tdDestroyHeapAllocator(SMemAllocator *pMemAllocator) {
+  // TODO
+}
+
+static void *haMalloc(SMemAllocator *pma, size_t size) {
+  void *          ptr;
+  size_t          tsize = size + sizeof(size_t);
+  SHeapAllocator *pha = (SHeapAllocator *)(pma->impl);
+
+  ptr = malloc(tsize);
+  if (ptr) {
+    *(size_t *)ptr = size;
+    ptr = POINTER_SHIFT(ptr, sizeof(size_t));
+    atomic_fetch_add_64(&(pha->tusage), tsize);
+  }
+
+  return ptr;
+}
+
+static void *haCalloc(SMemAllocator *pma, size_t nmemb, size_t size) {
+  void * ptr;
+  size_t tsize = nmemb * size;
+
+  ptr = haMalloc(pma, tsize);
+  if (ptr) {
+    memset(ptr, 0, tsize);
+  }
+
+  return ptr;
+}
+
+static void *haRealloc(SMemAllocator *pma, void *ptr, size_t size) {
+  size_t psize;
+  size_t tsize = size + sizeof(size_t);
+
+  if (ptr == NULL) {
+    psize = 0;
+  } else {
+    psize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
+  }
+
+  if (psize < size) {
+    // TODO
+  } else {
+    return ptr;
+  }
+}
+
+static void haFree(SMemAllocator *pma, void *ptr) { /* TODO */
+  SHeapAllocator *pha = (SHeapAllocator *)(pma->impl);
+  if (ptr) {
+    size_t tsize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)) + sizeof(size_t);
+    atomic_fetch_sub_64(&(pha->tusage), tsize);
+    free(POINTER_SHIFT(ptr, -sizeof(size_t)));
+  }
+}
+
+static size_t haUsage(SMemAllocator *pma) { return ((SHeapAllocator *)(pma->impl))->tusage; }
+
+/* ------------------------ ARENA ALLOCATOR ------------------------ */
+typedef struct {
+  size_t usage;
+} SArenaAllocator;
\ No newline at end of file