diff --git a/.travis.yml b/.travis.yml
index 6a4acce4513fc277ffebd564c07c1f5df5ff2754..4afdb6154ea12535b1a56f67b594559b1a97da06 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,7 +52,8 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
-          pip3 install numpy
+          py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev
+          pip3 install psutil
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e53cca2ea80e70a755724577ac788ecdcd97274b..8dd99f0b58fbb7f43dfadac3927f80a7f41fc565 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,6 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-SET(TD_SYNC FALSE)
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
@@ -17,8 +16,8 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
 
 INCLUDE(cmake/input.inc)
 INCLUDE(cmake/platform.inc)
-INCLUDE(cmake/env.inc)
 INCLUDE(cmake/define.inc)
+INCLUDE(cmake/env.inc)
 INCLUDE(cmake/install.inc)
 
 ADD_SUBDIRECTORY(deps)
diff --git a/alert/.gitignore b/alert/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..332fefb3ae376fb47d21c8e5d44c2d1d61f96e5a
--- /dev/null
+++ b/alert/.gitignore
@@ -0,0 +1,21 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+vendor/
+
+# Project specific files
+cmd/alert/alert
+cmd/alert/alert.log
+*.db
+*.gz
\ No newline at end of file
diff --git a/alert/README.md b/alert/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..23179669349ebf94f3774c6101c06c2e375ef059
--- /dev/null
+++ b/alert/README.md
@@ -0,0 +1,184 @@
+# Alert
+
+The Alert application reads data from [TDEngine](https://www.taosdata.com/), calculating according to predefined rules to generate alerts, and pushes alerts to downstream applications like [AlertManager](https://github.com/prometheus/alertmanager).
+
+## Install
+
+### From Binary
+
+Precompiled binaries is available at [taosdata website](https://www.taosdata.com/en/getting-started/), please download and unpack it by below shell command.
+
+```
+$ tar -xzf tdengine-alert-$version-$OS-$ARCH.tar.gz 
+```
+
+If you have no TDengine server or client installed, please execute below command to install the required driver library:
+
+```
+$ ./install_driver.sh
+```
+
+### From Source Code
+
+Two prerequisites are required to install from source.
+
+1. TDEngine server or client must be installed.
+2. Latest [Go](https://golang.org) language must be installed.
+
+When these two prerequisites are ready, please follow steps below to build the application:
+
+```
+$ mkdir taosdata
+$ cd taosdata
+$ git clone https://github.com/taosdata/tdengine.git
+$ cd tdengine/alert/cmd/alert
+$ go build
+```
+
+If `go build` fails because some of the dependency packages cannot be downloaded, please follow steps in [goproxy.io](https://goproxy.io) to configure `GOPROXY` and try `go build` again.
+
+## Configure
+
+The configuration file format of Alert application is standard `json`, below is its default content, please revise according to actual scenario.
+
+```json
+{
+  "port": 8100,
+  "database": "file:alert.db",
+  "tdengine": "root:taosdata@/tcp(127.0.0.1:0)/",
+  "log": {
+    "level": "production",
+    "path": "alert.log"
+  },
+  "receivers": {
+    "alertManager": "http://127.0.0.1:9093/api/v1/alerts",
+    "console": true
+  }
+}
+```
+
+The use of each configuration item is:
+
+* **port**: This is the `http` service port which enables other application to manage rules by `restful API`.
+* **database**: rules are stored in a `sqlite` database, this is the path of the database file (if the file does not exist, the alert application creates it automatically).
+* **tdengine**: connection string of `TDEngine` server, note in most cases the database information should be put in a rule, thus it should NOT be included here.
+* **log > level**: log level, could be `production` or `debug`.
+* **log > path**: log output file path.
+* **receivers > alertManager**: the alert application pushes alerts to `AlertManager` at this URL.
+* **receivers > console**: print out alerts to console (stdout) or not.
+
+When the configruation file is ready, the alert application can be started with below command (`alert.cfg` is the path of the configuration file):
+
+```
+$ ./alert -cfg alert.cfg
+```
+
+## Prepare an alert rule
+
+From technical aspect, an alert could be defined as: query and filter recent data from `TDEngine`, and calculating out a boolean value from these data according to a formula, and trigger an alert if the boolean value last for a certain duration.
+
+This is a rule example in `json` format:
+
+```json
+{
+  "name": "rule1",
+  "sql": "select sum(col1) as sumCol1 from test.meters where ts > now - 1h group by areaid",
+  "expr": "sumCol1 > 10",
+  "for": "10m",
+  "period": "1m",
+  "labels": {
+    "ruleName": "rule1"
+  },
+  "annotations": {
+    "summary": "sum of rule {{$labels.ruleName}} of area {{$values.areaid}} is {{$values.sumCol1}}"
+  }
+}
+```
+
+The fields of the rule is explained below:
+
+* **name**: the name of the rule, must be unique.
+* **sql**: this is the `sql` statement used to query data from `TDEngine`, columns of the query result are used in later processing, so please give the column an alias if aggregation functions are used.
+* **expr**: an expression whose result is a boolean value, arithmatic and logical calculations can be included in the expression, and builtin functions (see below) are also supported. Alerts are only triggered when the expression evaluates to `true`.
+* **for**: this item is a duration which default value is zero second. when `expr` evaluates to `true` and last at least this duration, an alert is triggered.
+* **period**: the interval for the alert application to check the rule, default is 1 minute.
+* **labels**: a label list, labels are used to generate alert information. note if the `sql` statement includes a `group by` clause, the `group by` columns are inserted into this list automatically.
+* **annotations**: the template of alert information which is in [go template](https://golang.org/pkg/text/template) syntax, labels can be referenced by `$labels.<label name>` and columns of the query result can be referenced by `$values.<column name>`.
+
+### Operators
+
+Operators which can be used in the `expr` field of a rule are list below, `()` can be to change precedence if default does not meet requirement.
+
+<table>
+<thead>
+<tr> <td>Operator</td><td>Unary/Binary</td><td>Precedence</td><td>Effect</td> </tr>
+</thead>
+<tbody>
+<tr> <td>~</td><td>Unary</td><td>6</td><td>Bitwise Not</td> </tr>
+<tr> <td>!</td><td>Unary</td><td>6</td><td>Logical Not</td> </tr>
+<tr> <td>+</td><td>Unary</td><td>6</td><td>Positive Sign</td> </tr>
+<tr> <td>-</td><td>Unary</td><td>6</td><td>Negative Sign</td> </tr>
+<tr> <td>*</td><td>Binary</td><td>5</td><td>Multiplication</td> </tr>
+<tr> <td>/</td><td>Binary</td><td>5</td><td>Division</td> </tr>
+<tr> <td>%</td><td>Binary</td><td>5</td><td>Modulus</td> </tr>
+<tr> <td><<</td><td>Binary</td><td>5</td><td>Bitwise Left Shift</td> </tr>
+<tr> <td>>></td><td>Binary</td><td>5</td><td>Bitwise Right Shift</td> </tr>
+<tr> <td>&</td><td>Binary</td><td>5</td><td>Bitwise And</td> </tr>
+<tr> <td>+</td><td>Binary</td><td>4</td><td>Addition</td> </tr>
+<tr> <td>-</td><td>Binary</td><td>4</td><td>Subtraction</td> </tr>
+<tr> <td>|</td><td>Binary</td><td>4</td><td>Bitwise Or</td> </tr>
+<tr> <td>^</td><td>Binary</td><td>4</td><td>Bitwise Xor</td> </tr>
+<tr> <td>==</td><td>Binary</td><td>3</td><td>Equal</td> </tr>
+<tr> <td>!=</td><td>Binary</td><td>3</td><td>Not Equal</td> </tr>
+<tr> <td><</td><td>Binary</td><td>3</td><td>Less Than</td> </tr>
+<tr> <td><=</td><td>Binary</td><td>3</td><td>Less Than or Equal</td> </tr>
+<tr> <td>></td><td>Binary</td><td>3</td><td>Great Than</td> </tr>
+<tr> <td>>=</td><td>Binary</td><td>3</td><td>Great Than or Equal</td> </tr>
+<tr> <td>&&</td><td>Binary</td><td>2</td><td>Logical And</td> </tr>
+<tr> <td>||</td><td>Binary</td><td>1</td><td>Logical Or</td> </tr>
+</tbody>
+</table>
+
+### Built-in Functions
+
+Built-in function can be used in the `expr` field of a rule.
+
+* **min**: returns the minimum one of its arguments, for example: `min(1, 2, 3)` returns `1`.
+* **max**: returns the maximum one of its arguments, for example: `max(1, 2, 3)` returns `3`.
+* **sum**: returns the sum of its arguments, for example: `sum(1, 2, 3)` returns `6`.
+* **avg**: returns the average of its arguments, for example: `avg(1, 2, 3)` returns `2`.
+* **sqrt**: returns the square root of its argument, for example: `sqrt(9)` returns `3`.
+* **ceil**: returns the minimum integer which greater or equal to its argument, for example: `ceil(9.1)` returns `10`.
+* **floor**: returns the maximum integer which lesser or equal to its argument, for example: `floor(9.9)` returns `9`.
+* **round**: round its argument to nearest integer, for examples: `round(9.9)` returns `10` and `round(9.1)` returns `9`.
+* **log**: returns the natural logarithm of its argument, for example: `log(10)` returns `2.302585`. 
+* **log10**: returns base 10 logarithm of its argument, for example: `log10(10)` return `1`.
+* **abs**: returns the absolute value of its argument, for example: `abs(-1)` returns `1`.
+* **if**: if the first argument is `true` returns its second argument, and returns its third argument otherwise, for examples: `if(true, 10, 100)` returns `10` and `if(false, 10, 100)` returns `100`.
+
+## Rule Management
+
+* Add / Update
+
+    * API address: http://\<server\>:\<port\>/api/update-rule
+    * Method: POST
+    * Body: the rule
+    * Example：curl -d '@rule.json' http://localhost:8100/api/update-rule
+
+* Delete
+
+    * API address: http://\<server\>:\<port\>/api/delete-rule?name=\<rule name\>
+    * Method：DELETE
+    * Example：curl -X DELETE http://localhost:8100/api/delete-rule?name=rule1
+
+* Enable / Disable
+
+    * API address: http://\<server\>:\<port\>/api/enable-rule?name=\<rule name\>&enable=[true | false]
+    * Method POST
+    * Example：curl -X POST http://localhost:8100/api/enable-rule?name=rule1&enable=true
+
+* Retrieve rule list
+
+    * API address: http://\<server\>:\<port\>/api/list-rule
+    * Method: GET
+    * Example：curl http://localhost:8100/api/list-rule
diff --git a/alert/README_cn.md b/alert/README_cn.md
new file mode 100644
index 0000000000000000000000000000000000000000..ec6e4566c8eb9fa2142c37b2b3e1b1a04f783a9a
--- /dev/null
+++ b/alert/README_cn.md
@@ -0,0 +1,181 @@
+# Alert
+
+报警监测程序，从 [TDEngine](https://www.taosdata.com/) 读取数据后，根据预定义的规则计算和生成报警，并将它们推送到 [AlertManager](https://github.com/prometheus/alertmanager) 或其它下游应用。
+
+## 安装
+
+### 使用编译好的二进制文件
+
+您可以从 [涛思数据](https://www.taosdata.com/cn/getting-started/) 官网下载最新的安装包。下载完成后，使用以下命令解压：
+
+```
+$ tar -xzf tdengine-alert-$version-$OS-$ARCH.tar.gz 
+```
+
+如果您之前没有安装过 TDengine 的服务端或客户端，您需要使用下面的命令安装 TDengine 的动态库：
+
+```
+$ ./install_driver.sh
+```
+
+### 从源码安装
+
+从源码安装需要在您用于编译的计算机上提前安装好 TDEngine 的服务端或客户端，如果您还没有安装，可以参考 TDEngine 的文档。
+
+报警监测程序使用 [Go语言](https://golang.org) 开发，请安装最新版的 Go 语言编译环境。
+
+```
+$ mkdir taosdata
+$ cd taosdata
+$ git clone https://github.com/taosdata/tdengine.git
+$ cd tdengine/alert/cmd/alert
+$ go build
+```
+
+如果由于部分包无法下载导致 `go build` 失败，请根据 [goproxy.io](https://goproxy.io) 上的说明配置好 `GOPROXY` 再重新执行 `go build`。
+
+## 配置
+
+报警监测程序的配置文件采用标准`json`格式，下面是默认的文件内容，请根据实际情况修改。
+
+```json
+{
+  "port": 8100,
+  "database": "file:alert.db",
+  "tdengine": "root:taosdata@/tcp(127.0.0.1:0)/",
+  "log": {
+    "level": "production",
+    "path": "alert.log"
+  },
+  "receivers": {
+    "alertManager": "http://127.0.0.1:9093/api/v1/alerts",
+    "console": true
+  }
+}
+```
+
+其中：
+
+* **port**：报警监测程序支持使用 `restful API` 对规则进行管理，这个参数用于配置 `http` 服务的侦听端口。
+* **database**：报警监测程序将规则保存到了一个 `sqlite` 数据库中，这个参数用于指定数据库文件的路径（不需要提前创建这个文件，如果它不存在，程序会自动创建它）。
+* **tdengine**：`TDEngine` 的连接信息，一般来说，数据库信息应该在报警规则中指定，所以这里 **不** 应包含这一部分信息。
+* **log > level**：日志的记录级别，可选 `production` 或 `debug`。
+* **log > path**：日志文件的路径。
+* **receivers > alertManager**：报警监测程序会将报警推送到 `AlertManager`，在这里指定 `AlertManager` 的接收地址。
+* **receivers > console**：是否输出到控制台 (stdout)。
+
+准备好配置文件后，可使用下面的命令启动报警监测程序（ `alert.cfg` 是配置文件的路径）：
+
+```
+$ ./alert -cfg alert.cfg
+```
+
+## 编写报警规则
+
+从技术角度，可以将报警描述为：从 `TDEngine` 中查询最近一段时间、符合一定过滤条件的数据，并基于这些数据根据定义好的计算方法得出一个结果，当结果符合某个条件且持续一定时间后，触发报警。
+
+根据上面的描述，可以很容易的知道报警规则中需要包含的大部分信息。 以下是一个完整的报警规则，采用标准 `json` 格式：
+
+```json
+{
+  "name": "rule1",
+  "sql": "select sum(col1) as sumCol1 from test.meters where ts > now - 1h group by areaid",
+  "expr": "sumCol1 > 10",
+  "for": "10m",
+  "period": "1m",
+  "labels": {
+    "ruleName": "rule1"
+  },
+  "annotations": {
+    "summary": "sum of rule {{$labels.ruleName}} of area {{$values.areaid}} is {{$values.sumCol1}}"
+  }
+}
+```
+
+其中：
+
+* **name**：用于为规则指定一个唯一的名字。
+* **sql**：从 `TDEngine` 中查询数据时使用的 `sql` 语句，查询结果中的列将被后续计算使用，所以，如果使用了聚合函数，请为这一列指定一个别名。
+* **expr**：一个计算结果为布尔型的表达式，支持算数运算、逻辑运算，并且内置了部分函数，也可以引用查询结果中的列。 当表达式计算结果为 `true` 时，进入报警状态。
+* **for**：当表达式计算结果为 `true` 的连续时长超过这个选项时，触发报警，否则报警处于“待定”状态。默认为0，表示一旦计算结果为 `true`，立即触发报警。
+* **period**：规则的检查周期，默认1分钟。
+* **labels**：人为指定的标签列表，标签可以在生成报警信息引用。如果 `sql` 中包含 `group by` 子句，则所有用于分组的字段会被自动加入这个标签列表中。
+* **annotations**：用于定义报警信息，使用 [go template](https://golang.org/pkg/text/template) 语法，其中，可以通过 `$labels.<label name>` 引用标签，也可以通过 `$values.<column name>` 引用查询结果中的列。
+
+### 运算符
+
+以下是 `expr` 字段中支持的运算符，您可以使用 `()` 改变运算的优先级。
+
+<table>
+<thead>
+<tr> <td>运算符</td><td>单目/双目</td><td>优先级</td><td>作用</td> </tr>
+</thead>
+<tbody>
+<tr> <td>~</td><td>单目</td><td>6</td><td>按位取反</td> </tr>
+<tr> <td>!</td><td>单目</td><td>6</td><td>逻辑非</td> </tr>
+<tr> <td>+</td><td>单目</td><td>6</td><td>正号</td> </tr>
+<tr> <td>-</td><td>单目</td><td>6</td><td>负号</td> </tr>
+<tr> <td>*</td><td>双目</td><td>5</td><td>乘法</td> </tr>
+<tr> <td>/</td><td>双目</td><td>5</td><td>除法</td> </tr>
+<tr> <td>%</td><td>双目</td><td>5</td><td>取模（余数）</td> </tr>
+<tr> <td><<</td><td>双目</td><td>5</td><td>按位左移</td> </tr>
+<tr> <td>>></td><td>双目</td><td>5</td><td>按位右移</td> </tr>
+<tr> <td>&</td><td>双目</td><td>5</td><td>按位与</td> </tr>
+<tr> <td>+</td><td>双目</td><td>4</td><td>加法</td> </tr>
+<tr> <td>-</td><td>双目</td><td>4</td><td>减法</td> </tr>
+<tr> <td>|</td><td>双目</td><td>4</td><td>按位或</td> </tr>
+<tr> <td>^</td><td>双目</td><td>4</td><td>按位异或</td> </tr>
+<tr> <td>==</td><td>双目</td><td>3</td><td>等于</td> </tr>
+<tr> <td>!=</td><td>双目</td><td>3</td><td>不等于</td> </tr>
+<tr> <td><</td><td>双目</td><td>3</td><td>小于</td> </tr>
+<tr> <td><=</td><td>双目</td><td>3</td><td>小于或等于</td> </tr>
+<tr> <td>></td><td>双目</td><td>3</td><td>大于</td> </tr>
+<tr> <td>>=</td><td>双目</td><td>3</td><td>大于或等于</td> </tr>
+<tr> <td>&&</td><td>双目</td><td>2</td><td>逻辑与</td> </tr>
+<tr> <td>||</td><td>双目</td><td>1</td><td>逻辑或</td> </tr>
+</tbody>
+</table>
+
+### 内置函数
+
+目前支持以下内置函数，可以在报警规则的 `expr` 字段中使用这些函数：
+
+* **min**：取多个值中的最小值，例如 `min(1, 2, 3)` 返回 `1`。
+* **max**：取多个值中的最大值，例如 `max(1, 2, 3)` 返回 `3`。
+* **sum**：求和，例如 `sum(1, 2, 3)` 返回 `6`。
+* **avg**：求算术平均值，例如 `avg(1, 2, 3)` 返回 `2`。
+* **sqrt**：计算平方根，例如 `sqrt(9)` 返回 `3`。
+* **ceil**：上取整，例如 `ceil(9.1)` 返回 `10`。
+* **floor**：下取整，例如 `floor(9.9)` 返回 `9`。
+* **round**：四舍五入，例如 `round(9.9)` 返回 `10`， `round(9.1)` 返回 `9`。
+* **log**：计算自然对数，例如 `log(10)` 返回 `2.302585`。
+* **log10**：计算以10为底的对数，例如 `log10(10)` 返回 `1`。
+* **abs**：计算绝对值，例如 `abs(-1)` 返回 `1`。
+* **if**：如果第一个参数为 `true`，返回第二个参数，否则返回第三个参数，例如 `if(true, 10, 100)` 返回 `10`， `if(false, 10, 100)` 返回 `100`。
+
+## 规则管理
+
+* 添加或修改
+
+    * API地址：http://\<server\>:\<port\>/api/update-rule
+    * Method：POST
+    * Body：规则定义
+    * 示例：curl -d '@rule.json' http://localhost:8100/api/update-rule
+
+* 删除
+
+    * API地址：http://\<server\>:\<port\>/api/delete-rule?name=\<rule name\>
+    * Method：DELETE
+    * 示例：curl -X DELETE http://localhost:8100/api/delete-rule?name=rule1
+
+* 挂起或恢复
+
+    * API地址：http://\<server\>:\<port\>/api/enable-rule?name=\<rule name\>&enable=[true | false]
+    * Method：POST
+    * 示例：curl -X POST http://localhost:8100/api/enable-rule?name=rule1&enable=true
+
+* 获取列表
+
+    * API地址：http://\<server\>:\<port\>/api/list-rule
+    * Method：GET
+    * 示例：curl http://localhost:8100/api/list-rule
diff --git a/alert/app/app.go b/alert/app/app.go
new file mode 100644
index 0000000000000000000000000000000000000000..760f93d839e217cd552bcf06e230675b16b80463
--- /dev/null
+++ b/alert/app/app.go
@@ -0,0 +1,175 @@
+package app
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/taosdata/alert/models"
+	"github.com/taosdata/alert/utils"
+	"github.com/taosdata/alert/utils/log"
+)
+
+func Init() error {
+	if e := initRule(); e != nil {
+		return e
+	}
+
+	http.HandleFunc("/api/list-rule", onListRule)
+	http.HandleFunc("/api/list-alert", onListAlert)
+	http.HandleFunc("/api/update-rule", onUpdateRule)
+	http.HandleFunc("/api/enable-rule", onEnableRule)
+	http.HandleFunc("/api/delete-rule", onDeleteRule)
+
+	return nil
+}
+
+func Uninit() error {
+	uninitRule()
+	return nil
+}
+
+func onListRule(w http.ResponseWriter, r *http.Request) {
+	var res []*Rule
+	rules.Range(func(k, v interface{}) bool {
+		res = append(res, v.(*Rule))
+		return true
+	})
+
+	w.Header().Add("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(res)
+}
+
+func onListAlert(w http.ResponseWriter, r *http.Request) {
+	var alerts []*Alert
+	rn := r.URL.Query().Get("rule")
+	rules.Range(func(k, v interface{}) bool {
+		if len(rn) > 0 && rn != k.(string) {
+			return true
+		}
+
+		rule := v.(*Rule)
+		rule.Alerts.Range(func(k, v interface{}) bool {
+			alert := v.(*Alert)
+			// TODO: not go-routine safe
+			if alert.State != AlertStateWaiting {
+				alerts = append(alerts, v.(*Alert))
+			}
+			return true
+		})
+		return true
+	})
+	w.Header().Add("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(alerts)
+}
+
+func onUpdateRule(w http.ResponseWriter, r *http.Request) {
+	data, e := ioutil.ReadAll(r.Body)
+	if e != nil {
+		log.Error("failed to read request body: ", e.Error())
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	rule, e := newRule(string(data))
+	if e != nil {
+		log.Error("failed to parse rule: ", e.Error())
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if e = doUpdateRule(rule, string(data)); e != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func doUpdateRule(rule *Rule, ruleStr string) error {
+	if _, ok := rules.Load(rule.Name); ok {
+		if len(utils.Cfg.Database) > 0 {
+			e := models.UpdateRule(rule.Name, ruleStr)
+			if e != nil {
+				log.Errorf("[%s]: update failed: %s", rule.Name, e.Error())
+				return e
+			}
+		}
+		log.Infof("[%s]: update succeeded.", rule.Name)
+	} else {
+		if len(utils.Cfg.Database) > 0 {
+			e := models.AddRule(&models.Rule{
+				Name:    rule.Name,
+				Content: ruleStr,
+			})
+			if e != nil {
+				log.Errorf("[%s]: add failed: %s", rule.Name, e.Error())
+				return e
+			}
+		}
+		log.Infof("[%s]: add succeeded.", rule.Name)
+	}
+
+	rules.Store(rule.Name, rule)
+	return nil
+}
+
+func onEnableRule(w http.ResponseWriter, r *http.Request) {
+	var rule *Rule
+	name := r.URL.Query().Get("name")
+	enable := strings.ToLower(r.URL.Query().Get("enable")) == "true"
+
+	if x, ok := rules.Load(name); ok {
+		rule = x.(*Rule)
+	} else {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	if rule.isEnabled() == enable {
+		return
+	}
+
+	if len(utils.Cfg.Database) > 0 {
+		if e := models.EnableRule(name, enable); e != nil {
+			if enable {
+				log.Errorf("[%s]: enable failed: ", name, e.Error())
+			} else {
+				log.Errorf("[%s]: disable failed: ", name, e.Error())
+			}
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	}
+
+	if enable {
+		rule = rule.clone()
+		rule.setNextRunTime(time.Now())
+		rules.Store(rule.Name, rule)
+		log.Infof("[%s]: enable succeeded.", name)
+	} else {
+		rule.setState(RuleStateDisabled)
+		log.Infof("[%s]: disable succeeded.", name)
+	}
+}
+
+func onDeleteRule(w http.ResponseWriter, r *http.Request) {
+	name := r.URL.Query().Get("name")
+	if len(name) == 0 {
+		return
+	}
+	if e := doDeleteRule(name); e != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func doDeleteRule(name string) error {
+	if len(utils.Cfg.Database) > 0 {
+		if e := models.DeleteRule(name); e != nil {
+			log.Errorf("[%s]: delete failed: %s", name, e.Error())
+			return e
+		}
+	}
+	rules.Delete(name)
+	log.Infof("[%s]: delete succeeded.", name)
+	return nil
+}
diff --git a/alert/app/expr/expr.go b/alert/app/expr/expr.go
new file mode 100644
index 0000000000000000000000000000000000000000..42cb506f0e4f66d238cb3d37eeac3752d68b1989
--- /dev/null
+++ b/alert/app/expr/expr.go
@@ -0,0 +1,792 @@
+package expr
+
+import (
+	"errors"
+	"io"
+	"math"
+	"strconv"
+	"strings"
+	"text/scanner"
+)
+
+var (
+	// compile errors
+	ErrorExpressionSyntax     = errors.New("expression syntax error")
+	ErrorUnrecognizedFunction = errors.New("unrecognized function")
+	ErrorArgumentCount        = errors.New("too many/few arguments")
+	ErrorInvalidFloat         = errors.New("invalid float")
+	ErrorInvalidInteger       = errors.New("invalid integer")
+
+	// eval errors
+	ErrorUnsupportedDataType     = errors.New("unsupported data type")
+	ErrorInvalidOperationFloat   = errors.New("invalid operation for float")
+	ErrorInvalidOperationInteger = errors.New("invalid operation for integer")
+	ErrorInvalidOperationBoolean = errors.New("invalid operation for boolean")
+	ErrorOnlyIntegerAllowed      = errors.New("only integers is allowed")
+	ErrorDataTypeMismatch        = errors.New("data type mismatch")
+)
+
+// binary operator precedence
+//    5             *   /   %  <<  >>  &
+//    4             +   -   |  ^
+//    3             ==  !=  <  <=  >   >=
+//    2             &&
+//    1             ||
+const (
+	opOr         = -(iota + 1000) // ||
+	opAnd                         // &&
+	opEqual                       // ==
+	opNotEqual                    // !=
+	opGTE                         // >=
+	opLTE                         // <=
+	opLeftShift                   // <<
+	opRightShift                  // >>
+)
+
+type lexer struct {
+	scan scanner.Scanner
+	tok  rune
+}
+
+func (l *lexer) init(src io.Reader) {
+	l.scan.Error = func(s *scanner.Scanner, msg string) {
+		panic(errors.New(msg))
+	}
+	l.scan.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings
+	l.scan.Init(src)
+	l.tok = l.next()
+}
+
+func (l *lexer) next() rune {
+	l.tok = l.scan.Scan()
+
+	switch l.tok {
+	case '|':
+		if l.scan.Peek() == '|' {
+			l.tok = opOr
+			l.scan.Scan()
+		}
+
+	case '&':
+		if l.scan.Peek() == '&' {
+			l.tok = opAnd
+			l.scan.Scan()
+		}
+
+	case '=':
+		if l.scan.Peek() == '=' {
+			l.tok = opEqual
+			l.scan.Scan()
+		} else {
+			// TODO: error
+		}
+
+	case '!':
+		if l.scan.Peek() == '=' {
+			l.tok = opNotEqual
+			l.scan.Scan()
+		} else {
+			// TODO: error
+		}
+
+	case '<':
+		if tok := l.scan.Peek(); tok == '<' {
+			l.tok = opLeftShift
+			l.scan.Scan()
+		} else if tok == '=' {
+			l.tok = opLTE
+			l.scan.Scan()
+		}
+
+	case '>':
+		if tok := l.scan.Peek(); tok == '>' {
+			l.tok = opRightShift
+			l.scan.Scan()
+		} else if tok == '=' {
+			l.tok = opGTE
+			l.scan.Scan()
+		}
+	}
+	return l.tok
+}
+
+func (l *lexer) token() rune {
+	return l.tok
+}
+
+func (l *lexer) text() string {
+	switch l.tok {
+	case opOr:
+		return "||"
+	case opAnd:
+		return "&&"
+	case opEqual:
+		return "=="
+	case opNotEqual:
+		return "!="
+	case opLeftShift:
+		return "<<"
+	case opLTE:
+		return "<="
+	case opRightShift:
+		return ">>"
+	case opGTE:
+		return ">="
+	default:
+		return l.scan.TokenText()
+	}
+}
+
+type Expr interface {
+	Eval(env func(string) interface{}) interface{}
+}
+
+type unaryExpr struct {
+	op      rune
+	subExpr Expr
+}
+
+func (ue *unaryExpr) Eval(env func(string) interface{}) interface{} {
+	val := ue.subExpr.Eval(env)
+	switch v := val.(type) {
+	case float64:
+		if ue.op != '-' {
+			panic(ErrorInvalidOperationFloat)
+		}
+		return -v
+	case int64:
+		switch ue.op {
+		case '-':
+			return -v
+		case '~':
+			return ^v
+		default:
+			panic(ErrorInvalidOperationInteger)
+		}
+	case bool:
+		if ue.op != '!' {
+			panic(ErrorInvalidOperationBoolean)
+		}
+		return !v
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+type binaryExpr struct {
+	op  rune
+	lhs Expr
+	rhs Expr
+}
+
+func (be *binaryExpr) Eval(env func(string) interface{}) interface{} {
+	lval := be.lhs.Eval(env)
+	rval := be.rhs.Eval(env)
+
+	switch be.op {
+	case '*':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv * rv
+			case int64:
+				return lv * float64(rv)
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) * rv
+			case int64:
+				return lv * rv
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case '/':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				if rv == 0 {
+					return math.Inf(int(lv))
+				} else {
+					return lv / rv
+				}
+			case int64:
+				if rv == 0 {
+					return math.Inf(int(lv))
+				} else {
+					return lv / float64(rv)
+				}
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				if rv == 0 {
+					return math.Inf(int(lv))
+				} else {
+					return float64(lv) / rv
+				}
+			case int64:
+				if rv == 0 {
+					return math.Inf(int(lv))
+				} else {
+					return lv / rv
+				}
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case '%':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return math.Mod(lv, rv)
+			case int64:
+				return math.Mod(lv, float64(rv))
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return math.Mod(float64(lv), rv)
+			case int64:
+				if rv == 0 {
+					return math.Inf(int(lv))
+				} else {
+					return lv % rv
+				}
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case opLeftShift:
+		switch lv := lval.(type) {
+		case int64:
+			switch rv := rval.(type) {
+			case int64:
+				return lv << rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case opRightShift:
+		switch lv := lval.(type) {
+		case int64:
+			switch rv := rval.(type) {
+			case int64:
+				return lv >> rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case '&':
+		switch lv := lval.(type) {
+		case int64:
+			switch rv := rval.(type) {
+			case int64:
+				return lv & rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case '+':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv + rv
+			case int64:
+				return lv + float64(rv)
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) + rv
+			case int64:
+				return lv + rv
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case '-':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv - rv
+			case int64:
+				return lv - float64(rv)
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) - rv
+			case int64:
+				return lv - rv
+			case bool:
+				panic(ErrorInvalidOperationBoolean)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case '|':
+		switch lv := lval.(type) {
+		case int64:
+			switch rv := rval.(type) {
+			case int64:
+				return lv | rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case '^':
+		switch lv := lval.(type) {
+		case int64:
+			switch rv := rval.(type) {
+			case int64:
+				return lv ^ rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case opEqual:
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv == rv
+			case int64:
+				return lv == float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) == rv
+			case int64:
+				return lv == rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			switch rv := rval.(type) {
+			case float64:
+			case int64:
+			case bool:
+				return lv == rv
+			}
+		}
+
+	case opNotEqual:
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv != rv
+			case int64:
+				return lv != float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) != rv
+			case int64:
+				return lv != rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			switch rv := rval.(type) {
+			case float64:
+			case int64:
+			case bool:
+				return lv != rv
+			}
+		}
+
+	case '<':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv < rv
+			case int64:
+				return lv < float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) < rv
+			case int64:
+				return lv < rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case opLTE:
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv <= rv
+			case int64:
+				return lv <= float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) <= rv
+			case int64:
+				return lv <= rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case '>':
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv > rv
+			case int64:
+				return lv > float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) > rv
+			case int64:
+				return lv > rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case opGTE:
+		switch lv := lval.(type) {
+		case float64:
+			switch rv := rval.(type) {
+			case float64:
+				return lv >= rv
+			case int64:
+				return lv >= float64(rv)
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case int64:
+			switch rv := rval.(type) {
+			case float64:
+				return float64(lv) >= rv
+			case int64:
+				return lv >= rv
+			case bool:
+				panic(ErrorDataTypeMismatch)
+			}
+		case bool:
+			panic(ErrorInvalidOperationBoolean)
+		}
+
+	case opAnd:
+		switch lv := lval.(type) {
+		case bool:
+			switch rv := rval.(type) {
+			case bool:
+				return lv && rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+
+	case opOr:
+		switch lv := lval.(type) {
+		case bool:
+			switch rv := rval.(type) {
+			case bool:
+				return lv || rv
+			default:
+				panic(ErrorOnlyIntegerAllowed)
+			}
+		default:
+			panic(ErrorOnlyIntegerAllowed)
+		}
+	}
+
+	return nil
+}
+
+type funcExpr struct {
+	name string
+	args []Expr
+}
+
+func (fe *funcExpr) Eval(env func(string) interface{}) interface{} {
+	argv := make([]interface{}, 0, len(fe.args))
+	for _, arg := range fe.args {
+		argv = append(argv, arg.Eval(env))
+	}
+	return funcs[fe.name].call(argv)
+}
+
+type floatExpr struct {
+	val float64
+}
+
+func (fe *floatExpr) Eval(env func(string) interface{}) interface{} {
+	return fe.val
+}
+
+type intExpr struct {
+	val int64
+}
+
+func (ie *intExpr) Eval(env func(string) interface{}) interface{} {
+	return ie.val
+}
+
+type boolExpr struct {
+	val bool
+}
+
+func (be *boolExpr) Eval(env func(string) interface{}) interface{} {
+	return be.val
+}
+
+type stringExpr struct {
+	val string
+}
+
+func (se *stringExpr) Eval(env func(string) interface{}) interface{} {
+	return se.val
+}
+
+type varExpr struct {
+	name string
+}
+
+func (ve *varExpr) Eval(env func(string) interface{}) interface{} {
+	return env(ve.name)
+}
+
+func Compile(src string) (expr Expr, err error) {
+	defer func() {
+		switch x := recover().(type) {
+		case nil:
+		case error:
+			err = x
+		default:
+		}
+	}()
+
+	lexer := lexer{}
+	lexer.init(strings.NewReader(src))
+	expr = parseBinary(&lexer, 0)
+	if lexer.token() != scanner.EOF {
+		panic(ErrorExpressionSyntax)
+	}
+	return expr, nil
+}
+
+func precedence(op rune) int {
+	switch op {
+	case opOr:
+		return 1
+	case opAnd:
+		return 2
+	case opEqual, opNotEqual, '<', '>', opGTE, opLTE:
+		return 3
+	case '+', '-', '|', '^':
+		return 4
+	case '*', '/', '%', opLeftShift, opRightShift, '&':
+		return 5
+	}
+	return 0
+}
+
+// binary = unary ('+' binary)*
+func parseBinary(lexer *lexer, lastPrec int) Expr {
+	lhs := parseUnary(lexer)
+
+	for {
+		op := lexer.token()
+		prec := precedence(op)
+		if prec <= lastPrec {
+			break
+		}
+		lexer.next() // consume operator
+		rhs := parseBinary(lexer, prec)
+		lhs = &binaryExpr{op: op, lhs: lhs, rhs: rhs}
+	}
+
+	return lhs
+}
+
+// unary = '+|-' expr | primary
+func parseUnary(lexer *lexer) Expr {
+	flag := false
+	for tok := lexer.token(); ; tok = lexer.next() {
+		if tok == '-' {
+			flag = !flag
+		} else if tok != '+' {
+			break
+		}
+	}
+	if flag {
+		return &unaryExpr{op: '-', subExpr: parsePrimary(lexer)}
+	}
+
+	flag = false
+	for tok := lexer.token(); tok == '!'; tok = lexer.next() {
+		flag = !flag
+	}
+	if flag {
+		return &unaryExpr{op: '!', subExpr: parsePrimary(lexer)}
+	}
+
+	flag = false
+	for tok := lexer.token(); tok == '~'; tok = lexer.next() {
+		flag = !flag
+	}
+	if flag {
+		return &unaryExpr{op: '~', subExpr: parsePrimary(lexer)}
+	}
+
+	return parsePrimary(lexer)
+}
+
+// primary = id
+//         | id '(' expr ',' ... ',' expr ')'
+//         | num
+//         | '(' expr ')'
+func parsePrimary(lexer *lexer) Expr {
+	switch lexer.token() {
+	case '+', '-', '!', '~':
+		return parseUnary(lexer)
+
+	case '(':
+		lexer.next() // consume '('
+		node := parseBinary(lexer, 0)
+		if lexer.token() != ')' {
+			panic(ErrorExpressionSyntax)
+		}
+		lexer.next() // consume ')'
+		return node
+
+	case scanner.Ident:
+		id := strings.ToLower(lexer.text())
+		if lexer.next() != '(' {
+			if id == "true" {
+				return &boolExpr{val: true}
+			} else if id == "false" {
+				return &boolExpr{val: false}
+			} else {
+				return &varExpr{name: id}
+			}
+		}
+		node := funcExpr{name: id}
+		for lexer.next() != ')' {
+			arg := parseBinary(lexer, 0)
+			node.args = append(node.args, arg)
+			if lexer.token() != ',' {
+				break
+			}
+		}
+		if lexer.token() != ')' {
+			panic(ErrorExpressionSyntax)
+		}
+
+		if fn, ok := funcs[id]; !ok {
+			panic(ErrorUnrecognizedFunction)
+		} else if fn.minArgs >= 0 && len(node.args) < fn.minArgs {
+			panic(ErrorArgumentCount)
+		} else if fn.maxArgs >= 0 && len(node.args) > fn.maxArgs {
+			panic(ErrorArgumentCount)
+		}
+
+		lexer.next() // consume it
+		return &node
+
+	case scanner.Int:
+		val, e := strconv.ParseInt(lexer.text(), 0, 64)
+		if e != nil {
+			panic(ErrorInvalidFloat)
+		}
+		lexer.next()
+		return &intExpr{val: val}
+
+	case scanner.Float:
+		val, e := strconv.ParseFloat(lexer.text(), 0)
+		if e != nil {
+			panic(ErrorInvalidInteger)
+		}
+		lexer.next()
+		return &floatExpr{val: val}
+
+	case scanner.String:
+		panic(errors.New("strings are not allowed in expression at present"))
+		val := lexer.text()
+		lexer.next()
+		return &stringExpr{val: val}
+
+	default:
+		panic(ErrorExpressionSyntax)
+	}
+}
diff --git a/alert/app/expr/expr_test.go b/alert/app/expr/expr_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..0228268771a681dc5c71ad0b74d994d1f9f5338a
--- /dev/null
+++ b/alert/app/expr/expr_test.go
@@ -0,0 +1,198 @@
+package expr
+
+import "testing"
+
+func TestIntArithmetic(t *testing.T) {
+	cases := []struct {
+		expr     string
+		expected int64
+	}{
+		{"+10", 10},
+		{"-10", -10},
+		{"3 + 4 + 5 + 6 * 7 + 8", 62},
+		{"3 + 4 + (5 + 6) * 7 + 8", 92},
+		{"3 + 4 + (5 + 6) * 7 / 11 + 8", 22},
+		{"3 + 4 + -5 * 6 / 7 % 8", 3},
+		{"10 - 5", 5},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(nil); res.(int64) != c.expected {
+			t.Errorf("result for expression '%s' is %v, but expected is %v", c.expr, res, c.expected)
+		}
+	}
+}
+
+func TestFloatArithmetic(t *testing.T) {
+	cases := []struct {
+		expr     string
+		expected float64
+	}{
+		{"+10.5", 10.5},
+		{"-10.5", -10.5},
+		{"3.1 + 4.2 + 5 + 6 * 7 + 8", 62.3},
+		{"3.1 + 4.2 + (5 + 6) * 7 + 8.3", 92.6},
+		{"3.1 + 4.2 + (5.1 + 5.9) * 7 / 11 + 8", 22.3},
+		{"3.3 + 4.2 - 4.0 * 7.5 / 3", -2.5},
+		{"3.3 + 4.2 - 4 * 7.0 / 2", -6.5},
+		{"3.5/2.0", 1.75},
+		{"3.5/2", 1.75},
+		{"7 / 3.5", 2},
+		{"3.5 % 2.0", 1.5},
+		{"3.5 % 2", 1.5},
+		{"7 % 2.5", 2},
+		{"7.3 - 2", 5.3},
+		{"7 - 2.3", 4.7},
+		{"1 + 1.5", 2.5},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(nil); res.(float64) != c.expected {
+			t.Errorf("result for expression '%s' is %v, but expected is %v", c.expr, res, c.expected)
+		}
+	}
+}
+
+func TestVariable(t *testing.T) {
+	variables := map[string]interface{}{
+		"a": int64(6),
+		"b": int64(7),
+	}
+	env := func(key string) interface{} {
+		return variables[key]
+	}
+
+	cases := []struct {
+		expr     string
+		expected int64
+	}{
+		{"3 + 4 + (+5) + a * b + 8", 62},
+		{"3 + 4 + (5 + a) * b + 8", 92},
+		{"3 + 4 + (5 + a) * b / 11 + 8", 22},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(env); res.(int64) != c.expected {
+			t.Errorf("result for expression '%s' is %v, but expected is %v", c.expr, res, c.expected)
+		}
+	}
+}
+
+func TestFunction(t *testing.T) {
+	variables := map[string]interface{}{
+		"a": int64(6),
+		"b": 7.0,
+	}
+
+	env := func(key string) interface{} {
+		return variables[key]
+	}
+
+	cases := []struct {
+		expr     string
+		expected float64
+	}{
+		{"sum(3, 4,  5, a * b, 8)", 62},
+		{"sum(3, 4, (5 + a) * b, 8)", 92},
+		{"sum(3, 4, (5 + a) * b / 11, 8)", 22},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(env); res.(float64) != c.expected {
+			t.Errorf("result for expression '%s' is %v, but expected is %v", c.expr, res, c.expected)
+		}
+	}
+}
+
+func TestLogical(t *testing.T) {
+	cases := []struct {
+		expr     string
+		expected bool
+	}{
+		{"true", true},
+		{"false", false},
+		{"true == true", true},
+		{"true == false", false},
+		{"true != true", false},
+		{"true != false", true},
+		{"5 > 3", true},
+		{"5 < 3", false},
+		{"5.2 > 3", true},
+		{"5.2 < 3", false},
+		{"5 > 3.1", true},
+		{"5 < 3.1", false},
+		{"5.1 > 3.3", true},
+		{"5.1 < 3.3", false},
+		{"5 >= 3", true},
+		{"5 <= 3", false},
+		{"5.2 >= 3", true},
+		{"5.2 <= 3", false},
+		{"5 >= 3.1", true},
+		{"5 <= 3.1", false},
+		{"5.1 >= 3.3", true},
+		{"5.1 <= 3.3", false},
+		{"5 != 3", true},
+		{"5.2 != 3.2", true},
+		{"5.2 != 3", true},
+		{"5 != 3.2", true},
+		{"5 == 3", false},
+		{"5.2 == 3.2", false},
+		{"5.2 == 3", false},
+		{"5 == 3.2", false},
+		{"!(5 > 3)", false},
+		{"5>3 && 3>1", true},
+		{"5<3 || 3<1", false},
+		{"4<=4 || 3<1", true},
+		{"4<4 || 3>=1", true},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(nil); res.(bool) != c.expected {
+			t.Errorf("result for expression '%s' is %v, but expected is %v", c.expr, res, c.expected)
+		}
+	}
+}
+
+func TestBitwise(t *testing.T) {
+	cases := []struct {
+		expr     string
+		expected int64
+	}{
+		{"0x0C & 0x04", 0x04},
+		{"0x08 | 0x04", 0x0C},
+		{"0x0C ^ 0x04", 0x08},
+		{"0x01 << 2", 0x04},
+		{"0x04 >> 2", 0x01},
+		{"~0x04", ^0x04},
+	}
+
+	for _, c := range cases {
+		expr, e := Compile(c.expr)
+		if e != nil {
+			t.Errorf("failed to compile expression '%s': %s", c.expr, e.Error())
+		}
+		if res := expr.Eval(nil); res.(int64) != c.expected {
+			t.Errorf("result for expression '%s' is 0x%X, but expected is 0x%X", c.expr, res, c.expected)
+		}
+	}
+}
diff --git a/alert/app/expr/funcs.go b/alert/app/expr/funcs.go
new file mode 100644
index 0000000000000000000000000000000000000000..97ed0e1c647666dbc1cb4c3b6445bdfae8a034c6
--- /dev/null
+++ b/alert/app/expr/funcs.go
@@ -0,0 +1,198 @@
+package expr
+
+import (
+	"math"
+)
+
+type builtInFunc struct {
+	minArgs, maxArgs int
+	call             func([]interface{}) interface{}
+}
+
+func fnMin(args []interface{}) interface{} {
+	res := args[0]
+	for _, arg := range args[1:] {
+		switch v1 := res.(type) {
+		case int64:
+			switch v2 := arg.(type) {
+			case int64:
+				if v2 < v1 {
+					res = v2
+				}
+			case float64:
+				res = math.Min(float64(v1), v2)
+			default:
+				panic(ErrorUnsupportedDataType)
+			}
+		case float64:
+			switch v2 := arg.(type) {
+			case int64:
+				res = math.Min(v1, float64(v2))
+			case float64:
+				res = math.Min(v1, v2)
+			default:
+				panic(ErrorUnsupportedDataType)
+			}
+		default:
+			panic(ErrorUnsupportedDataType)
+		}
+	}
+	return res
+}
+
+func fnMax(args []interface{}) interface{} {
+	res := args[0]
+	for _, arg := range args[1:] {
+		switch v1 := res.(type) {
+		case int64:
+			switch v2 := arg.(type) {
+			case int64:
+				if v2 > v1 {
+					res = v2
+				}
+			case float64:
+				res = math.Max(float64(v1), v2)
+			default:
+				panic(ErrorUnsupportedDataType)
+			}
+		case float64:
+			switch v2 := arg.(type) {
+			case int64:
+				res = math.Max(v1, float64(v2))
+			case float64:
+				res = math.Max(v1, v2)
+			default:
+				panic(ErrorUnsupportedDataType)
+			}
+		default:
+			panic(ErrorUnsupportedDataType)
+		}
+	}
+	return res
+}
+
+func fnSum(args []interface{}) interface{} {
+	res := float64(0)
+	for _, arg := range args {
+		switch v := arg.(type) {
+		case int64:
+			res += float64(v)
+		case float64:
+			res += v
+		default:
+			panic(ErrorUnsupportedDataType)
+		}
+	}
+	return res
+}
+
+func fnAvg(args []interface{}) interface{} {
+	return fnSum(args).(float64) / float64(len(args))
+}
+
+func fnSqrt(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return math.Sqrt(float64(v))
+	case float64:
+		return math.Sqrt(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnFloor(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return v
+	case float64:
+		return math.Floor(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnCeil(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return v
+	case float64:
+		return math.Ceil(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnRound(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return v
+	case float64:
+		return math.Round(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnLog(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return math.Log(float64(v))
+	case float64:
+		return math.Log(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnLog10(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		return math.Log10(float64(v))
+	case float64:
+		return math.Log10(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnAbs(args []interface{}) interface{} {
+	switch v := args[0].(type) {
+	case int64:
+		if v < 0 {
+			return -v
+		}
+		return v
+	case float64:
+		return math.Abs(v)
+	default:
+		panic(ErrorUnsupportedDataType)
+	}
+}
+
+func fnIf(args []interface{}) interface{} {
+	v, ok := args[0].(bool)
+	if !ok {
+		panic(ErrorUnsupportedDataType)
+	}
+	if v {
+		return args[1]
+	} else {
+		return args[2]
+	}
+}
+
+var funcs = map[string]builtInFunc{
+	"min":   builtInFunc{minArgs: 1, maxArgs: -1, call: fnMin},
+	"max":   builtInFunc{minArgs: 1, maxArgs: -1, call: fnMax},
+	"sum":   builtInFunc{minArgs: 1, maxArgs: -1, call: fnSum},
+	"avg":   builtInFunc{minArgs: 1, maxArgs: -1, call: fnAvg},
+	"sqrt":  builtInFunc{minArgs: 1, maxArgs: 1, call: fnSqrt},
+	"ceil":  builtInFunc{minArgs: 1, maxArgs: 1, call: fnCeil},
+	"floor": builtInFunc{minArgs: 1, maxArgs: 1, call: fnFloor},
+	"round": builtInFunc{minArgs: 1, maxArgs: 1, call: fnRound},
+	"log":   builtInFunc{minArgs: 1, maxArgs: 1, call: fnLog},
+	"log10": builtInFunc{minArgs: 1, maxArgs: 1, call: fnLog10},
+	"abs":   builtInFunc{minArgs: 1, maxArgs: 1, call: fnAbs},
+	"if":    builtInFunc{minArgs: 3, maxArgs: 3, call: fnIf},
+}
diff --git a/alert/app/expr/funcs_test.go b/alert/app/expr/funcs_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..b960888b25f609e5c43fbf6cdd90e9b29ada319c
--- /dev/null
+++ b/alert/app/expr/funcs_test.go
@@ -0,0 +1,329 @@
+package expr
+
+import (
+	"math"
+	"testing"
+)
+
+func TestMax(t *testing.T) {
+	cases := []struct {
+		args     []interface{}
+		expected float64
+	}{
+		{[]interface{}{int64(1), int64(2), int64(3), int64(4), int64(5)}, 5},
+		{[]interface{}{int64(1), int64(2), float64(3), int64(4), float64(5)}, 5},
+		{[]interface{}{int64(-1), int64(-2), float64(-3), int64(-4), float64(-5)}, -1},
+		{[]interface{}{int64(-1), int64(-1), float64(-1), int64(-1), float64(-1)}, -1},
+		{[]interface{}{int64(-1), int64(0), float64(-1), int64(-1), float64(-1)}, 0},
+	}
+
+	for _, c := range cases {
+		r := fnMax(c.args)
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("max(%v) = %v, want %v", c.args, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("max(%v) = %v, want %v", c.args, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type max(%v)", c.args)
+		}
+	}
+}
+
+func TestMin(t *testing.T) {
+	cases := []struct {
+		args     []interface{}
+		expected float64
+	}{
+		{[]interface{}{int64(1), int64(2), int64(3), int64(4), int64(5)}, 1},
+		{[]interface{}{int64(5), int64(4), float64(3), int64(2), float64(1)}, 1},
+		{[]interface{}{int64(-1), int64(-2), float64(-3), int64(-4), float64(-5)}, -5},
+		{[]interface{}{int64(-1), int64(-1), float64(-1), int64(-1), float64(-1)}, -1},
+		{[]interface{}{int64(1), int64(0), float64(1), int64(1), float64(1)}, 0},
+	}
+
+	for _, c := range cases {
+		r := fnMin(c.args)
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("min(%v) = %v, want %v", c.args, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("min(%v) = %v, want %v", c.args, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type min(%v)", c.args)
+		}
+	}
+}
+
+func TestSumAvg(t *testing.T) {
+	cases := []struct {
+		args     []interface{}
+		expected float64
+	}{
+		{[]interface{}{int64(1)}, 1},
+		{[]interface{}{int64(1), int64(2), int64(3), int64(4), int64(5)}, 15},
+		{[]interface{}{int64(5), int64(4), float64(3), int64(2), float64(1)}, 15},
+		{[]interface{}{int64(-1), int64(-2), float64(-3), int64(-4), float64(-5)}, -15},
+		{[]interface{}{int64(-1), int64(-1), float64(-1), int64(-1), float64(-1)}, -5},
+		{[]interface{}{int64(1), int64(0), float64(1), int64(1), float64(1)}, 4},
+	}
+
+	for _, c := range cases {
+		r := fnSum(c.args)
+		switch v := r.(type) {
+		case float64:
+			if v != c.expected {
+				t.Errorf("sum(%v) = %v, want %v", c.args, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type sum(%v)", c.args)
+		}
+	}
+
+	for _, c := range cases {
+		r := fnAvg(c.args)
+		expected := c.expected / float64(len(c.args))
+		switch v := r.(type) {
+		case float64:
+			if v != expected {
+				t.Errorf("avg(%v) = %v, want %v", c.args, v, expected)
+			}
+		default:
+			t.Errorf("unknown result type avg(%v)", c.args)
+		}
+	}
+}
+
+func TestSqrt(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(0), 0},
+		{int64(1), 1},
+		{int64(256), 16},
+		{10.0, math.Sqrt(10)},
+		{10000.0, math.Sqrt(10000)},
+	}
+
+	for _, c := range cases {
+		r := fnSqrt([]interface{}{c.arg})
+		switch v := r.(type) {
+		case float64:
+			if v != c.expected {
+				t.Errorf("sqrt(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type sqrt(%v)", c.arg)
+		}
+	}
+}
+
+func TestFloor(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(0), 0},
+		{int64(1), 1},
+		{int64(-1), -1},
+		{10.4, 10},
+		{-10.4, -11},
+		{10.8, 10},
+		{-10.8, -11},
+	}
+
+	for _, c := range cases {
+		r := fnFloor([]interface{}{c.arg})
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("floor(%v) = %v, want %v", c.arg, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("floor(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type floor(%v)", c.arg)
+		}
+	}
+}
+
+func TestCeil(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(0), 0},
+		{int64(1), 1},
+		{int64(-1), -1},
+		{10.4, 11},
+		{-10.4, -10},
+		{10.8, 11},
+		{-10.8, -10},
+	}
+
+	for _, c := range cases {
+		r := fnCeil([]interface{}{c.arg})
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("ceil(%v) = %v, want %v", c.arg, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("ceil(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type ceil(%v)", c.arg)
+		}
+	}
+}
+
+func TestRound(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(0), 0},
+		{int64(1), 1},
+		{int64(-1), -1},
+		{10.4, 10},
+		{-10.4, -10},
+		{10.8, 11},
+		{-10.8, -11},
+	}
+
+	for _, c := range cases {
+		r := fnRound([]interface{}{c.arg})
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("round(%v) = %v, want %v", c.arg, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("round(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type round(%v)", c.arg)
+		}
+	}
+}
+
+func TestLog(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(1), math.Log(1)},
+		{0.1, math.Log(0.1)},
+		{10.4, math.Log(10.4)},
+		{10.8, math.Log(10.8)},
+	}
+
+	for _, c := range cases {
+		r := fnLog([]interface{}{c.arg})
+		switch v := r.(type) {
+		case float64:
+			if v != c.expected {
+				t.Errorf("log(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type log(%v)", c.arg)
+		}
+	}
+}
+
+func TestLog10(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(1), math.Log10(1)},
+		{0.1, math.Log10(0.1)},
+		{10.4, math.Log10(10.4)},
+		{10.8, math.Log10(10.8)},
+		{int64(100), math.Log10(100)},
+	}
+
+	for _, c := range cases {
+		r := fnLog10([]interface{}{c.arg})
+		switch v := r.(type) {
+		case float64:
+			if v != c.expected {
+				t.Errorf("log10(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type log10(%v)", c.arg)
+		}
+	}
+}
+
+func TestAbs(t *testing.T) {
+	cases := []struct {
+		arg      interface{}
+		expected float64
+	}{
+		{int64(1), 1},
+		{int64(0), 0},
+		{int64(-1), 1},
+		{10.4, 10.4},
+		{-10.4, 10.4},
+	}
+
+	for _, c := range cases {
+		r := fnAbs([]interface{}{c.arg})
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("abs(%v) = %v, want %v", c.arg, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("abs(%v) = %v, want %v", c.arg, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type abs(%v)", c.arg)
+		}
+	}
+}
+
+func TestIf(t *testing.T) {
+	cases := []struct {
+		args     []interface{}
+		expected float64
+	}{
+		{[]interface{}{true, int64(10), int64(20)}, 10},
+		{[]interface{}{false, int64(10), int64(20)}, 20},
+		{[]interface{}{true, 10.3, 20.6}, 10.3},
+		{[]interface{}{false, 10.3, 20.6}, 20.6},
+		{[]interface{}{true, int64(10), 20.6}, 10},
+		{[]interface{}{false, int64(10), 20.6}, 20.6},
+	}
+
+	for _, c := range cases {
+		r := fnIf(c.args)
+		switch v := r.(type) {
+		case int64:
+			if v != int64(c.expected) {
+				t.Errorf("if(%v) = %v, want %v", c.args, v, int64(c.expected))
+			}
+		case float64:
+			if v != c.expected {
+				t.Errorf("if(%v) = %v, want %v", c.args, v, c.expected)
+			}
+		default:
+			t.Errorf("unknown result type if(%v)", c.args)
+		}
+	}
+}
diff --git a/alert/app/route.go b/alert/app/route.go
new file mode 100644
index 0000000000000000000000000000000000000000..0fec3bab3406bcdc7922ec58987f4c1131f39c57
--- /dev/null
+++ b/alert/app/route.go
@@ -0,0 +1,44 @@
+package app
+
+import (
+	"regexp"
+	"strings"
+)
+
+type RouteMatchCriteria struct {
+	Tag   string         `yaml:"tag"`
+	Value string         `yaml:"match"`
+	Re    *regexp.Regexp `yaml:"-"`
+}
+
+func (c *RouteMatchCriteria) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var v map[string]string
+	if e := unmarshal(&v); e != nil {
+		return e
+	}
+
+	for k, a := range v {
+		c.Tag = k
+		c.Value = a
+		if strings.HasPrefix(a, "re:") {
+			re, e := regexp.Compile(a[3:])
+			if e != nil {
+				return e
+			}
+			c.Re = re
+		}
+	}
+
+	return nil
+}
+
+type Route struct {
+	Continue       bool                 `yaml:"continue"`
+	Receiver       string               `yaml:"receiver"`
+	GroupWait      Duration             `yaml:"group_wait"`
+	GroupInterval  Duration             `yaml:"group_interval"`
+	RepeatInterval Duration             `yaml:"repeat_interval"`
+	GroupBy        []string             `yaml:"group_by"`
+	Match          []RouteMatchCriteria `yaml:"match"`
+	Routes         []*Route             `yaml:"routes"`
+}
diff --git a/alert/app/rule.go b/alert/app/rule.go
new file mode 100644
index 0000000000000000000000000000000000000000..d1ab9776fed4cd0d2e9134e0e6bfb46872567376
--- /dev/null
+++ b/alert/app/rule.go
@@ -0,0 +1,627 @@
+package app
+
+import (
+	"bytes"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"text/scanner"
+	"text/template"
+	"time"
+
+	"github.com/taosdata/alert/app/expr"
+	"github.com/taosdata/alert/models"
+	"github.com/taosdata/alert/utils"
+	"github.com/taosdata/alert/utils/log"
+)
+
+type Duration struct{ time.Duration }
+
+func (d Duration) MarshalJSON() ([]byte, error) {
+	return json.Marshal(d.String())
+}
+
+func (d *Duration) doUnmarshal(v interface{}) error {
+	switch value := v.(type) {
+	case float64:
+		*d = Duration{time.Duration(value)}
+		return nil
+	case string:
+		if duration, e := time.ParseDuration(value); e != nil {
+			return e
+		} else {
+			*d = Duration{duration}
+		}
+		return nil
+	default:
+		return errors.New("invalid duration")
+	}
+}
+
+func (d *Duration) UnmarshalJSON(b []byte) error {
+	var v interface{}
+	if e := json.Unmarshal(b, &v); e != nil {
+		return e
+	}
+	return d.doUnmarshal(v)
+}
+
+const (
+	AlertStateWaiting = iota
+	AlertStatePending
+	AlertStateFiring
+)
+
+type Alert struct {
+	State         uint8                  `json:"-"`
+	LastRefreshAt time.Time              `json:"-"`
+	StartsAt      time.Time              `json:"startsAt,omitempty"`
+	EndsAt        time.Time              `json:"endsAt,omitempty"`
+	Values        map[string]interface{} `json:"values"`
+	Labels        map[string]string      `json:"labels"`
+	Annotations   map[string]string      `json:"annotations"`
+}
+
+func (alert *Alert) doRefresh(firing bool, rule *Rule) bool {
+	switch {
+	case (!firing) && (alert.State == AlertStateWaiting):
+		return false
+
+	case (!firing) && (alert.State == AlertStatePending):
+		alert.State = AlertStateWaiting
+		return false
+
+	case (!firing) && (alert.State == AlertStateFiring):
+		alert.State = AlertStateWaiting
+		alert.EndsAt = time.Now()
+
+	case firing && (alert.State == AlertStateWaiting):
+		alert.StartsAt = time.Now()
+		if rule.For.Nanoseconds() > 0 {
+			alert.State = AlertStatePending
+			return false
+		}
+		alert.State = AlertStateFiring
+
+	case firing && (alert.State == AlertStatePending):
+		if time.Now().Sub(alert.StartsAt) < rule.For.Duration {
+			return false
+		}
+		alert.StartsAt = alert.StartsAt.Add(rule.For.Duration)
+		alert.State = AlertStateFiring
+
+	case firing && (alert.State == AlertStateFiring):
+	}
+
+	return true
+}
+
+func (alert *Alert) refresh(rule *Rule, values map[string]interface{}) {
+	alert.LastRefreshAt = time.Now()
+
+	defer func() {
+		switch x := recover().(type) {
+		case nil:
+		case error:
+			rule.setState(RuleStateError)
+			log.Errorf("[%s]: failed to evaluate: %s", rule.Name, x.Error())
+		default:
+			rule.setState(RuleStateError)
+			log.Errorf("[%s]: failed to evaluate: unknown error", rule.Name)
+		}
+	}()
+
+	alert.Values = values
+	res := rule.Expr.Eval(func(key string) interface{} {
+		// ToLower is required as column name in result is in lower case
+		return alert.Values[strings.ToLower(key)]
+	})
+
+	val, ok := res.(bool)
+	if !ok {
+		rule.setState(RuleStateError)
+		log.Errorf("[%s]: result type is not bool", rule.Name)
+		return
+	}
+
+	if !alert.doRefresh(val, rule) {
+		return
+	}
+
+	buf := bytes.Buffer{}
+	alert.Annotations = map[string]string{}
+	for k, v := range rule.Annotations {
+		if e := v.Execute(&buf, alert); e != nil {
+			log.Errorf("[%s]: failed to generate annotation '%s': %s", rule.Name, k, e.Error())
+		} else {
+			alert.Annotations[k] = buf.String()
+		}
+		buf.Reset()
+	}
+
+	buf.Reset()
+	if e := json.NewEncoder(&buf).Encode(alert); e != nil {
+		log.Errorf("[%s]: failed to serialize alert to JSON: %s", rule.Name, e.Error())
+	} else {
+		chAlert <- buf.String()
+	}
+}
+
+const (
+	RuleStateNormal = iota
+	RuleStateError
+	RuleStateDisabled
+	RuleStateRunning = 0x04
+)
+
+type Rule struct {
+	Name           string                        `json:"name"`
+	State          uint32                        `json:"state"`
+	SQL            string                        `json:"sql"`
+	GroupByCols    []string                      `json:"-"`
+	For            Duration                      `json:"for"`
+	Period         Duration                      `json:"period"`
+	NextRunTime    time.Time                     `json:"-"`
+	RawExpr        string                        `json:"expr"`
+	Expr           expr.Expr                     `json:"-"`
+	Labels         map[string]string             `json:"labels"`
+	RawAnnotations map[string]string             `json:"annotations"`
+	Annotations    map[string]*template.Template `json:"-"`
+	Alerts         sync.Map                      `json:"-"`
+}
+
+func (rule *Rule) clone() *Rule {
+	return &Rule{
+		Name:           rule.Name,
+		State:          RuleStateNormal,
+		SQL:            rule.SQL,
+		GroupByCols:    rule.GroupByCols,
+		For:            rule.For,
+		Period:         rule.Period,
+		NextRunTime:    time.Time{},
+		RawExpr:        rule.RawExpr,
+		Expr:           rule.Expr,
+		Labels:         rule.Labels,
+		RawAnnotations: rule.RawAnnotations,
+		Annotations:    rule.Annotations,
+		// don't copy alerts
+	}
+}
+
+func (rule *Rule) setState(s uint32) {
+	for {
+		old := atomic.LoadUint32(&rule.State)
+		new := old&0xffffffc0 | s
+		if atomic.CompareAndSwapUint32(&rule.State, old, new) {
+			break
+		}
+	}
+}
+
+func (rule *Rule) state() uint32 {
+	return atomic.LoadUint32(&rule.State) & 0xffffffc0
+}
+
+func (rule *Rule) isEnabled() bool {
+	state := atomic.LoadUint32(&rule.State)
+	return state&RuleStateDisabled == 0
+}
+
+func (rule *Rule) setNextRunTime(tm time.Time) {
+	rule.NextRunTime = tm.Round(rule.Period.Duration)
+	if rule.NextRunTime.Before(tm) {
+		rule.NextRunTime = rule.NextRunTime.Add(rule.Period.Duration)
+	}
+}
+
+func parseGroupBy(sql string) (cols []string, err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			err = e.(error)
+		}
+	}()
+
+	s := scanner.Scanner{
+		Error: func(s *scanner.Scanner, msg string) {
+			panic(errors.New(msg))
+		},
+		Mode: scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats,
+	}
+
+	s.Init(strings.NewReader(sql))
+	if s.Scan() != scanner.Ident || strings.ToLower(s.TokenText()) != "select" {
+		err = errors.New("only select statement is allowed.")
+		return
+	}
+
+	hasGroupBy := false
+	for t := s.Scan(); t != scanner.EOF; t = s.Scan() {
+		if t != scanner.Ident {
+			continue
+		}
+		if strings.ToLower(s.TokenText()) != "group" {
+			continue
+		}
+		if s.Scan() != scanner.Ident {
+			continue
+		}
+		if strings.ToLower(s.TokenText()) == "by" {
+			hasGroupBy = true
+			break
+		}
+	}
+
+	if !hasGroupBy {
+		return
+	}
+
+	for {
+		if s.Scan() != scanner.Ident {
+			err = errors.New("SQL statement syntax error.")
+			return
+		}
+		col := strings.ToLower(s.TokenText())
+		cols = append(cols, col)
+		if s.Scan() != ',' {
+			break
+		}
+	}
+
+	return
+}
+
+func (rule *Rule) parseGroupBy() (err error) {
+	cols, e := parseGroupBy(rule.SQL)
+	if e == nil {
+		rule.GroupByCols = cols
+	}
+	return nil
+}
+
+func (rule *Rule) getAlert(values map[string]interface{}) *Alert {
+	sb := strings.Builder{}
+	for _, name := range rule.GroupByCols {
+		value := values[name]
+		if value == nil {
+		} else {
+			sb.WriteString(fmt.Sprint(value))
+		}
+		sb.WriteByte('_')
+	}
+
+	var alert *Alert
+	key := sb.String()
+
+	if v, ok := rule.Alerts.Load(key); ok {
+		alert = v.(*Alert)
+	}
+	if alert == nil {
+		alert = &Alert{Labels: map[string]string{}}
+		for k, v := range rule.Labels {
+			alert.Labels[k] = v
+		}
+		for _, name := range rule.GroupByCols {
+			value := values[name]
+			if value == nil {
+				alert.Labels[name] = ""
+			} else {
+				alert.Labels[name] = fmt.Sprint(value)
+			}
+		}
+		rule.Alerts.Store(key, alert)
+	}
+
+	return alert
+}
+
+func (rule *Rule) preRun(tm time.Time) bool {
+	if tm.Before(rule.NextRunTime) {
+		return false
+	}
+	rule.setNextRunTime(tm)
+
+	for {
+		state := atomic.LoadUint32(&rule.State)
+		if state != RuleStateNormal {
+			return false
+		}
+		if atomic.CompareAndSwapUint32(&rule.State, state, RuleStateRunning) {
+			break
+		}
+	}
+	return true
+}
+
+func (rule *Rule) run(db *sql.DB) {
+	rows, e := db.Query(rule.SQL)
+	if e != nil {
+		log.Errorf("[%s]: failed to query TDengine: %s", rule.Name, e.Error())
+		return
+	}
+
+	cols, e := rows.ColumnTypes()
+	if e != nil {
+		log.Errorf("[%s]: unable to get column information: %s", rule.Name, e.Error())
+		return
+	}
+
+	for rows.Next() {
+		values := make([]interface{}, 0, len(cols))
+		for range cols {
+			var v interface{}
+			values = append(values, &v)
+		}
+		rows.Scan(values...)
+
+		m := make(map[string]interface{})
+		for i, col := range cols {
+			name := strings.ToLower(col.Name())
+			m[name] = *(values[i].(*interface{}))
+		}
+
+		alert := rule.getAlert(m)
+		alert.refresh(rule, m)
+	}
+
+	now := time.Now()
+	rule.Alerts.Range(func(k, v interface{}) bool {
+		alert := v.(*Alert)
+		if now.Sub(alert.LastRefreshAt) > rule.Period.Duration*10 {
+			rule.Alerts.Delete(k)
+		}
+		return true
+	})
+}
+
+func (rule *Rule) postRun() {
+	for {
+		old := atomic.LoadUint32(&rule.State)
+		new := old & ^uint32(RuleStateRunning)
+		if atomic.CompareAndSwapUint32(&rule.State, old, new) {
+			break
+		}
+	}
+}
+
+func newRule(str string) (*Rule, error) {
+	rule := Rule{}
+
+	e := json.NewDecoder(strings.NewReader(str)).Decode(&rule)
+	if e != nil {
+		return nil, e
+	}
+
+	if rule.Period.Nanoseconds() <= 0 {
+		rule.Period = Duration{time.Minute}
+	}
+	rule.setNextRunTime(time.Now())
+
+	if rule.For.Nanoseconds() < 0 {
+		rule.For = Duration{0}
+	}
+
+	if e = rule.parseGroupBy(); e != nil {
+		return nil, e
+	}
+
+	if expr, e := expr.Compile(rule.RawExpr); e != nil {
+		return nil, e
+	} else {
+		rule.Expr = expr
+	}
+
+	rule.Annotations = map[string]*template.Template{}
+	for k, v := range rule.RawAnnotations {
+		v = reValue.ReplaceAllStringFunc(v, func(s string) string {
+			// as column name in query result is always in lower case,
+			// we need to convert value reference in annotations to
+			// lower case
+			return strings.ToLower(s)
+		})
+		text := "{{$labels := .Labels}}{{$values := .Values}}" + v
+		tmpl, e := template.New(k).Parse(text)
+		if e != nil {
+			return nil, e
+		}
+		rule.Annotations[k] = tmpl
+	}
+
+	return &rule, nil
+}
+
+const (
+	batchSize = 1024
+)
+
+var (
+	rules   sync.Map
+	wg      sync.WaitGroup
+	chStop  = make(chan struct{})
+	chAlert = make(chan string, batchSize)
+	reValue = regexp.MustCompile(`\$values\.[_a-zA-Z0-9]+`)
+)
+
+func runRules() {
+	defer wg.Done()
+
+	db, e := sql.Open("taosSql", utils.Cfg.TDengine)
+	if e != nil {
+		log.Fatal("failed to connect to TDengine: ", e.Error())
+	}
+	defer db.Close()
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+LOOP:
+	for {
+		var tm time.Time
+		select {
+		case <-chStop:
+			close(chAlert)
+			break LOOP
+		case tm = <-ticker.C:
+		}
+
+		rules.Range(func(k, v interface{}) bool {
+			rule := v.(*Rule)
+			if !rule.preRun(tm) {
+				return true
+			}
+
+			wg.Add(1)
+			go func(rule *Rule) {
+				defer wg.Done()
+				defer rule.postRun()
+				rule.run(db)
+			}(rule)
+
+			return true
+		})
+	}
+}
+
+func doPushAlerts(alerts []string) {
+	defer wg.Done()
+
+	if len(utils.Cfg.Receivers.AlertManager) == 0 {
+		return
+	}
+
+	buf := bytes.Buffer{}
+	buf.WriteByte('[')
+	for i, alert := range alerts {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		buf.WriteString(alert)
+	}
+	buf.WriteByte(']')
+
+	log.Debug(buf.String())
+
+	resp, e := http.DefaultClient.Post(utils.Cfg.Receivers.AlertManager, "application/json", &buf)
+	if e != nil {
+		log.Errorf("failed to push alerts to downstream: %s", e.Error())
+		return
+	}
+	resp.Body.Close()
+}
+
+func pushAlerts() {
+	defer wg.Done()
+
+	ticker := time.NewTicker(time.Millisecond * 100)
+	defer ticker.Stop()
+
+	alerts := make([]string, 0, batchSize)
+
+LOOP:
+	for {
+		select {
+		case alert := <-chAlert:
+			if utils.Cfg.Receivers.Console {
+				fmt.Print(alert)
+			}
+			if len(alert) == 0 {
+				if len(alerts) > 0 {
+					wg.Add(1)
+					doPushAlerts(alerts)
+				}
+				break LOOP
+			}
+			if len(alerts) == batchSize {
+				wg.Add(1)
+				go doPushAlerts(alerts)
+				alerts = make([]string, 0, batchSize)
+			}
+			alerts = append(alerts, alert)
+
+		case <-ticker.C:
+			if len(alerts) > 0 {
+				wg.Add(1)
+				go doPushAlerts(alerts)
+				alerts = make([]string, 0, batchSize)
+			}
+		}
+	}
+}
+
+func loadRuleFromDatabase() error {
+	allRules, e := models.LoadAllRule()
+	if e != nil {
+		log.Error("failed to load rules from database:", e.Error())
+		return e
+	}
+
+	count := 0
+	for _, r := range allRules {
+		rule, e := newRule(r.Content)
+		if e != nil {
+			log.Errorf("[%s]: parse failed: %s", r.Name, e.Error())
+			continue
+		}
+		if !r.Enabled {
+			rule.setState(RuleStateDisabled)
+		}
+		rules.Store(rule.Name, rule)
+		count++
+	}
+	log.Infof("total %d rules loaded", count)
+	return nil
+}
+
+func loadRuleFromFile() error {
+	f, e := os.Open(utils.Cfg.RuleFile)
+	if e != nil {
+		log.Error("failed to load rules from file:", e.Error())
+		return e
+	}
+	defer f.Close()
+
+	var allRules []Rule
+	e = json.NewDecoder(f).Decode(&allRules)
+	if e != nil {
+		log.Error("failed to parse rule file:", e.Error())
+		return e
+	}
+
+	for i := 0; i < len(allRules); i++ {
+		rule := &allRules[i]
+		rules.Store(rule.Name, rule)
+	}
+	log.Infof("total %d rules loaded", len(allRules))
+
+	return nil
+}
+
+func initRule() error {
+	if len(utils.Cfg.Database) > 0 {
+		if e := loadRuleFromDatabase(); e != nil {
+			return e
+		}
+	} else {
+		if e := loadRuleFromFile(); e != nil {
+			return e
+		}
+	}
+
+	wg.Add(2)
+	go runRules()
+	go pushAlerts()
+
+	return nil
+}
+
+func uninitRule() error {
+	close(chStop)
+	wg.Wait()
+	return nil
+}
diff --git a/alert/app/rule_test.go b/alert/app/rule_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..0adf100e9cda5358c34f7406f699ea486202e0f7
--- /dev/null
+++ b/alert/app/rule_test.go
@@ -0,0 +1,81 @@
+package app
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/taosdata/alert/utils/log"
+)
+
+func TestParseGroupBy(t *testing.T) {
+	cases := []struct {
+		sql  string
+		cols []string
+	}{
+		{
+			sql:  "select * from a",
+			cols: []string{},
+		},
+		{
+			sql:  "select * from a group by abc",
+			cols: []string{"abc"},
+		},
+		{
+			sql:  "select * from a group by abc, def",
+			cols: []string{"abc", "def"},
+		},
+		{
+			sql:  "select * from a Group by abc, def order by abc",
+			cols: []string{"abc", "def"},
+		},
+	}
+
+	for _, c := range cases {
+		cols, e := parseGroupBy(c.sql)
+		if e != nil {
+			t.Errorf("failed to parse sql '%s': %s", c.sql, e.Error())
+		}
+		for i := range cols {
+			if i >= len(c.cols) {
+				t.Errorf("count of group by columns of '%s' is wrong", c.sql)
+			}
+			if c.cols[i] != cols[i] {
+				t.Errorf("wrong group by columns for '%s'", c.sql)
+			}
+		}
+	}
+}
+
+func TestManagement(t *testing.T) {
+	const format = `{"name":"rule%d", "sql":"select count(*) as count from meters", "expr":"count>2"}`
+
+	log.Init()
+
+	for i := 0; i < 5; i++ {
+		s := fmt.Sprintf(format, i)
+		rule, e := newRule(s)
+		if e != nil {
+			t.Errorf("failed to create rule: %s", e.Error())
+		}
+		e = doUpdateRule(rule, s)
+		if e != nil {
+			t.Errorf("failed to add or update rule: %s", e.Error())
+		}
+	}
+
+	for i := 0; i < 5; i++ {
+		name := fmt.Sprintf("rule%d", i)
+		if _, ok := rules.Load(name); !ok {
+			t.Errorf("rule '%s' does not exist", name)
+		}
+	}
+
+	name := "rule1"
+	if e := doDeleteRule(name); e != nil {
+		t.Errorf("failed to delete rule: %s", e.Error())
+	}
+
+	if _, ok := rules.Load(name); ok {
+		t.Errorf("rule '%s' should not exist any more", name)
+	}
+}
diff --git a/alert/cmd/alert/alert.cfg b/alert/cmd/alert/alert.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..036f288fc6750f31e11fd8bab4b8d3c43197c772
--- /dev/null
+++ b/alert/cmd/alert/alert.cfg
@@ -0,0 +1,13 @@
+{
+	"port": 8100,
+	"database": "file:alert.db",
+	"tdengine": "root:taosdata@/tcp(127.0.0.1:0)/",
+	"log": {
+		"level": "debug",
+		"path": ""
+	},
+	"receivers": {
+		"alertManager": "http://127.0.0.1:9093/api/v1/alerts",
+		"console": true
+	}
+}
diff --git a/alert/cmd/alert/install_driver.sh b/alert/cmd/alert/install_driver.sh
new file mode 100644
index 0000000000000000000000000000000000000000..1171a9953846475289531c2e07e89e5465c65959
--- /dev/null
+++ b/alert/cmd/alert/install_driver.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# This file is used to install TDengine client library on linux systems.
+
+set -e
+#set -x
+
+# -----------------------Variables definition---------------------
+script_dir=$(dirname $(readlink -f "$0"))
+# Dynamic directory
+lib_link_dir="/usr/lib"
+
+#install main path
+install_main_dir="/usr/local/taos"
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo"
+fi
+
+function clean_driver() {
+    ${csudo} rm -f /usr/lib/libtaos.so || :
+}
+
+function install_driver() {
+    echo -e "${GREEN}Start to install TDengine client driver ...${NC}"
+    
+    #create install main dir and all sub dir
+    ${csudo} mkdir -p ${install_main_dir}  
+    ${csudo} mkdir -p ${install_main_dir}/driver
+
+    ${csudo} rm -f ${lib_link_dir}/libtaos.*         || :
+    ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/*  
+    ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
+    ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
+ 
+    echo
+    echo -e "\033[44;32;1mTDengine client driver is successfully installed!${NC}"
+}
+
+install_driver
diff --git a/alert/cmd/alert/main.go b/alert/cmd/alert/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..d3874396b4d49b46be642aa4e742e36b6c9568ef
--- /dev/null
+++ b/alert/cmd/alert/main.go
@@ -0,0 +1,183 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/taosdata/alert/app"
+	"github.com/taosdata/alert/models"
+	"github.com/taosdata/alert/utils"
+	"github.com/taosdata/alert/utils/log"
+
+	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/taosdata/driver-go/taosSql"
+)
+
+type httpHandler struct {
+}
+
+func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	path := r.URL.Path
+	http.DefaultServeMux.ServeHTTP(w, r)
+	duration := time.Now().Sub(start)
+	log.Debugf("[%s]\t%s\t%s", r.Method, path, duration)
+}
+
+func serveWeb() *http.Server {
+	log.Info("Listening at port: ", utils.Cfg.Port)
+
+	srv := &http.Server{
+		Addr:    ":" + strconv.Itoa(int(utils.Cfg.Port)),
+		Handler: &httpHandler{},
+	}
+	go func() {
+		if e := srv.ListenAndServe(); e != nil {
+			log.Error(e.Error())
+		}
+	}()
+
+	return srv
+}
+
+func copyFile(dst, src string) error {
+	if dst == src {
+		return nil
+	}
+
+	in, e := os.Open(src)
+	if e != nil {
+		return e
+	}
+	defer in.Close()
+
+	out, e := os.Create(dst)
+	if e != nil {
+		return e
+	}
+	defer out.Close()
+
+	_, e = io.Copy(out, in)
+	return e
+}
+
+func doSetup(cfgPath string) error {
+	exePath, e := os.Executable()
+	if e != nil {
+		fmt.Fprintf(os.Stderr, "failed to get executable path: %s\n", e.Error())
+		return e
+	}
+
+	if !filepath.IsAbs(cfgPath) {
+		dir := filepath.Dir(exePath)
+		cfgPath = filepath.Join(dir, cfgPath)
+	}
+
+	e = copyFile("/etc/taos/alert.cfg", cfgPath)
+	if e != nil {
+		fmt.Fprintf(os.Stderr, "failed copy configuration file: %s\n", e.Error())
+		return e
+	}
+
+	f, e := os.Create("/etc/systemd/system/alert.service")
+	if e != nil {
+		fmt.Printf("failed to create alert service: %s\n", e.Error())
+		return e
+	}
+	defer f.Close()
+
+	const content = `[Unit]
+Description=Alert (TDengine Alert Service)
+After=syslog.target
+After=network.target
+
+[Service]
+RestartSec=2s
+Type=simple
+WorkingDirectory=/var/lib/taos/
+ExecStart=%s -cfg /etc/taos/alert.cfg
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+`
+	_, e = fmt.Fprintf(f, content, exePath)
+	if e != nil {
+		fmt.Printf("failed to create alert.service: %s\n", e.Error())
+		return e
+	}
+
+	return nil
+}
+
+const version = "TDengine alert v1.0.0"
+
+func main() {
+	var (
+		cfgPath     string
+		setup       bool
+		showVersion bool
+	)
+	flag.StringVar(&cfgPath, "cfg", "alert.cfg", "path of configuration file")
+	flag.BoolVar(&setup, "setup", false, "setup the service as a daemon")
+	flag.BoolVar(&showVersion, "version", false, "show version information")
+	flag.Parse()
+
+	if showVersion {
+		fmt.Println(version)
+		return
+	}
+
+	if setup {
+		if runtime.GOOS == "linux" {
+			doSetup(cfgPath)
+		} else {
+			fmt.Fprintln(os.Stderr, "can only run as a daemon mode in linux.")
+		}
+		return
+	}
+
+	if e := utils.LoadConfig(cfgPath); e != nil {
+		fmt.Fprintln(os.Stderr, "failed to load configuration")
+		return
+	}
+
+	if e := log.Init(); e != nil {
+		fmt.Fprintln(os.Stderr, "failed to initialize logger:", e.Error())
+		return
+	}
+	defer log.Sync()
+
+	if e := models.Init(); e != nil {
+		log.Fatal("failed to initialize database:", e.Error())
+	}
+
+	if e := app.Init(); e != nil {
+		log.Fatal("failed to initialize application:", e.Error())
+	}
+	// start web server
+	srv := serveWeb()
+
+	// wait `Ctrl-C` or `Kill` to exit, `Kill` does not work on Windows
+	interrupt := make(chan os.Signal)
+	signal.Notify(interrupt, os.Interrupt)
+	signal.Notify(interrupt, os.Kill)
+	<-interrupt
+	fmt.Println("'Ctrl + C' received, exiting...")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	srv.Shutdown(ctx)
+	cancel()
+
+	app.Uninit()
+	models.Uninit()
+}
diff --git a/alert/cmd/alert/rule.json b/alert/cmd/alert/rule.json
new file mode 100644
index 0000000000000000000000000000000000000000..2859ff9c90d225f0c73674c5338125d952350f9c
--- /dev/null
+++ b/alert/cmd/alert/rule.json
@@ -0,0 +1,13 @@
+{
+    "name": "CarTooFast",
+    "period": "10s",
+    "sql": "select avg(speed) as avgspeed from test.cars where ts > now - 5m group by id",
+    "expr": "avgSpeed > 100",
+    "for": "0s",
+    "labels": {
+        "ruleName": "CarTooFast"
+    },
+    "annotations": {
+        "summary": "car {{$values.id}} is too fast, its average speed is {{$values.avgSpeed}}km/h"
+    }
+}
\ No newline at end of file
diff --git a/alert/go.mod b/alert/go.mod
new file mode 100644
index 0000000000000000000000000000000000000000..e0bd1d1941d8654e747194891f7693b4d79d2fbc
--- /dev/null
+++ b/alert/go.mod
@@ -0,0 +1,12 @@
+module github.com/taosdata/alert
+
+go 1.14
+
+require (
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/taosdata/driver-go master
+	go.uber.org/zap v1.14.1
+	google.golang.org/appengine v1.6.5 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+)
diff --git a/alert/models/models.go b/alert/models/models.go
new file mode 100644
index 0000000000000000000000000000000000000000..5d6de5572cc0ebb14ae4bca9f97ea1f64f34c00d
--- /dev/null
+++ b/alert/models/models.go
@@ -0,0 +1,131 @@
+package models
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/taosdata/alert/utils"
+	"github.com/taosdata/alert/utils/log"
+)
+
+var db *sqlx.DB
+
+func Init() error {
+	xdb, e := sqlx.Connect("sqlite3", utils.Cfg.Database)
+	if e == nil {
+		db = xdb
+	}
+	return upgrade()
+}
+
+func Uninit() error {
+	db.Close()
+	return nil
+}
+
+func getStringOption(tx *sqlx.Tx, name string) (string, error) {
+	const qs = "SELECT * FROM `option` WHERE `name`=?"
+
+	var (
+		e error
+		o struct {
+			Name  string `db:"name"`
+			Value string `db:"value"`
+		}
+	)
+
+	if tx != nil {
+		e = tx.Get(&o, qs, name)
+	} else {
+		e = db.Get(&o, qs, name)
+	}
+
+	if e != nil {
+		return "", e
+	}
+
+	return o.Value, nil
+}
+
+func getIntOption(tx *sqlx.Tx, name string) (int, error) {
+	s, e := getStringOption(tx, name)
+	if e != nil {
+		return 0, e
+	}
+	v, e := strconv.ParseInt(s, 10, 64)
+	return int(v), e
+}
+
+func setOption(tx *sqlx.Tx, name string, value interface{}) error {
+	const qs = "REPLACE INTO `option`(`name`, `value`) VALUES(?, ?);"
+
+	var (
+		e  error
+		sv string
+	)
+
+	switch v := value.(type) {
+	case time.Time:
+		sv = v.Format(time.RFC3339)
+	default:
+		sv = fmt.Sprint(value)
+	}
+
+	if tx != nil {
+		_, e = tx.Exec(qs, name, sv)
+	} else {
+		_, e = db.Exec(qs, name, sv)
+	}
+
+	return e
+}
+
+var upgradeScripts = []struct {
+	ver   int
+	stmts []string
+}{
+	{
+		ver: 0,
+		stmts: []string{
+			"CREATE TABLE `option`( `name` VARCHAR(63) PRIMARY KEY, `value` VARCHAR(255) NOT NULL) WITHOUT ROWID;",
+			"CREATE TABLE `rule`( `name` VARCHAR(63) PRIMARY KEY, `enabled` TINYINT(1) NOT NULL, `created_at` DATETIME NOT NULL, `updated_at` DATETIME NOT NULL, `content` TEXT(65535) NOT NULL);",
+		},
+	},
+}
+
+func upgrade() error {
+	const dbVersion = "database version"
+
+	ver, e := getIntOption(nil, dbVersion)
+	if e != nil { // regards all errors as schema not created
+		ver = -1 // set ver to -1 to execute all statements
+	}
+
+	tx, e := db.Beginx()
+	if e != nil {
+		return e
+	}
+
+	for _, us := range upgradeScripts {
+		if us.ver <= ver {
+			continue
+		}
+		log.Info("upgrading database to version: ", us.ver)
+		for _, s := range us.stmts {
+			if _, e = tx.Exec(s); e != nil {
+				tx.Rollback()
+				return e
+			}
+		}
+		ver = us.ver
+	}
+
+	if e = setOption(tx, dbVersion, ver); e != nil {
+		tx.Rollback()
+		return e
+	}
+
+	return tx.Commit()
+}
diff --git a/alert/models/rule.go b/alert/models/rule.go
new file mode 100644
index 0000000000000000000000000000000000000000..53d45f45dafae3db5d871219a51214963c0c2747
--- /dev/null
+++ b/alert/models/rule.go
@@ -0,0 +1,75 @@
+package models
+
+import "time"
+
+const (
+	sqlSelectAllRule = "SELECT * FROM `rule`;"
+	sqlSelectRule    = "SELECT * FROM `rule` WHERE `name` = ?;"
+	sqlInsertRule    = "INSERT INTO `rule`(`name`, `enabled`, `created_at`, `updated_at`, `content`) VALUES(:name, :enabled, :created_at, :updated_at, :content);"
+	sqlUpdateRule    = "UPDATE `rule` SET `content` = :content, `updated_at` = :updated_at WHERE `name` = :name;"
+	sqlEnableRule    = "UPDATE `rule` SET `enabled` = :enabled, `updated_at` = :updated_at WHERE `name` = :name;"
+	sqlDeleteRule    = "DELETE FROM `rule` WHERE `name` = ?;"
+)
+
+type Rule struct {
+	Name      string    `db:"name"`
+	Enabled   bool      `db:"enabled"`
+	CreatedAt time.Time `db:"created_at"`
+	UpdatedAt time.Time `db:"updated_at"`
+	Content   string    `db:"content"`
+}
+
+func AddRule(r *Rule) error {
+	r.CreatedAt = time.Now()
+	r.Enabled = true
+	r.UpdatedAt = r.CreatedAt
+	_, e := db.NamedExec(sqlInsertRule, r)
+	return e
+}
+
+func UpdateRule(name string, content string) error {
+	r := Rule{
+		Name:      name,
+		UpdatedAt: time.Now(),
+		Content:   content,
+	}
+	_, e := db.NamedExec(sqlUpdateRule, &r)
+	return e
+}
+
+func EnableRule(name string, enabled bool) error {
+	r := Rule{
+		Name:      name,
+		Enabled:   enabled,
+		UpdatedAt: time.Now(),
+	}
+
+	if res, e := db.NamedExec(sqlEnableRule, &r); e != nil {
+		return e
+	} else if n, e := res.RowsAffected(); n != 1 {
+		return e
+	}
+
+	return nil
+}
+
+func DeleteRule(name string) error {
+	_, e := db.Exec(sqlDeleteRule, name)
+	return e
+}
+
+func GetRuleByName(name string) (*Rule, error) {
+	r := Rule{}
+	if e := db.Get(&r, sqlSelectRule, name); e != nil {
+		return nil, e
+	}
+	return &r, nil
+}
+
+func LoadAllRule() ([]Rule, error) {
+	var rules []Rule
+	if e := db.Select(&rules, sqlSelectAllRule); e != nil {
+		return nil, e
+	}
+	return rules, nil
+}
diff --git a/alert/release.sh b/alert/release.sh
new file mode 100755
index 0000000000000000000000000000000000000000..44b383a57e1bc0cd387cb2fefeb2d580f3b5e19e
--- /dev/null
+++ b/alert/release.sh
@@ -0,0 +1,45 @@
+set -e
+
+# releash.sh  -c [arm | arm64 | x64 | x86] 
+#             -o [linux | darwin | windows]  
+
+# set parameters by default value
+cpuType=x64    # [arm | arm64 | x64 | x86]
+osType=linux   # [linux | darwin | windows]
+
+while getopts "h:c:o:" arg
+do
+  case $arg in
+    c)
+      #echo "cpuType=$OPTARG"
+      cpuType=$(echo $OPTARG)
+      ;;
+    o)
+      #echo "osType=$OPTARG"
+      osType=$(echo $OPTARG)
+      ;;
+    h)
+      echo "Usage: `basename $0` -c [arm | arm64 | x64 | x86] -o [linux | darwin | windows]"
+      exit 0
+      ;;
+    ?) #unknown option 
+      echo "unknown argument"
+      exit 1
+      ;;
+  esac
+done
+
+
+startdir=$(pwd)
+scriptdir=$(dirname $(readlink -f $0))
+cd ${scriptdir}/cmd/alert
+version=$(grep 'const version =' main.go | awk '{print $NF}')
+version=${version%\"}
+
+echo "cpuType=${cpuType}"
+echo "osType=${osType}"
+echo "version=${version}"
+
+GOOS=${osType} GOARCH=${cpuType} go build
+
+GZIP=-9 tar -zcf ${startdir}/tdengine-alert-${version}-${osType}-${cpuType}.tar.gz alert alert.cfg install_driver.sh driver/
diff --git a/alert/test/prepare.sim b/alert/test/prepare.sim
new file mode 100644
index 0000000000000000000000000000000000000000..2b786e8a080784e7bfffd63d41d0abf1d52637b4
--- /dev/null
+++ b/alert/test/prepare.sim
@@ -0,0 +1,17 @@
+sql connect
+sleep 100
+
+sql drop database if exists test
+sql create database test
+sql use test
+
+print ====== create super table
+sql create table cars (ts timestamp, speed int) tags(id int)
+
+print ====== create tables
+$i = 0
+while $i < 5
+    $tb = car . $i
+    sql create table $tb using cars tags( $i )
+    $i = $i + 1
+endw
diff --git a/alert/test/rule.json b/alert/test/rule.json
new file mode 100644
index 0000000000000000000000000000000000000000..8394b26593ee2fd432014be6b8537b4edec73b99
--- /dev/null
+++ b/alert/test/rule.json
@@ -0,0 +1,13 @@
+{
+    "name": "test1",
+    "period": "10s",
+    "sql": "select avg(speed) as avgspeed from test.cars group by id",
+    "expr": "avgSpeed >= 3",
+    "for": "0s",
+    "labels": {
+        "ruleName": "test1"
+    },
+    "annotations": {
+        "summary": "speed of car(id = {{$labels.id}}) is too high: {{$values.avgSpeed}}"
+    }
+}
\ No newline at end of file
diff --git a/alert/test/step1.sim b/alert/test/step1.sim
new file mode 100644
index 0000000000000000000000000000000000000000..f8c3d3bdd533ebc4d4c5efdd1e4e813f212ebcca
--- /dev/null
+++ b/alert/test/step1.sim
@@ -0,0 +1,10 @@
+sql connect
+sleep 100
+
+print ====== insert 10 records to table 0
+$i = 10
+while $i > 0
+    $ms = $i . s
+    sql insert into test.car0 values(now - $ms , 1)
+    $i = $i - 1
+endw
diff --git a/alert/test/step2.sim b/alert/test/step2.sim
new file mode 100644
index 0000000000000000000000000000000000000000..911d8290ba07c68440b725e7a570f90f9b5d7e74
--- /dev/null
+++ b/alert/test/step2.sim
@@ -0,0 +1,5 @@
+sql connect
+sleep 100
+
+print ====== insert another records table 0
+sql insert into test.car0 values(now , 100)
diff --git a/alert/test/step3.sim b/alert/test/step3.sim
new file mode 100644
index 0000000000000000000000000000000000000000..d88a171905f85a7a2754033605e701b327ff947a
--- /dev/null
+++ b/alert/test/step3.sim
@@ -0,0 +1,12 @@
+sql connect
+sleep 100
+
+print ====== insert 10 records to table 0, 1, 2
+$i = 10
+while $i > 0
+    $ms = $i . s
+    sql insert into test.car0 values(now - $ms , 1)
+    sql insert into test.car1 values(now - $ms , $i )
+    sql insert into test.car2 values(now - $ms , 10)
+    $i = $i - 1
+endw
diff --git a/alert/test/test.sh b/alert/test/test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..d6cc10e6e7b645fe13867767497c1a0f2881cdcb
--- /dev/null
+++ b/alert/test/test.sh
@@ -0,0 +1,91 @@
+# wait until $1 alerts are generates, and at most wait $2 seconds
+# return 0 if wait succeeded, 1 if wait timeout
+function waitAlert() {
+  local i=0
+
+  while [ $i -lt $2 ]; do
+    local c=$(wc -l alert.out | awk '{print $1}')
+
+    if [ $c -ge $1 ]; then
+      return 0
+    fi
+
+    let "i=$i+1"
+    sleep 1s
+  done
+
+  return 1
+}
+
+# prepare environment
+kill -INT `ps aux | grep 'alert -cfg' | grep -v grep | awk '{print $2}'`
+
+rm -f alert.db
+rm -f alert.out
+../cmd/alert/alert -cfg ../cmd/alert/alert.cfg > alert.out &
+
+../../td/debug/build/bin/tsim -c /etc/taos -f ./prepare.sim
+
+# add a rule to alert application
+curl -d '@rule.json' http://localhost:8100/api/update-rule
+
+# step 1: add some data but not trigger an alert
+../../td/debug/build/bin/tsim -c /etc/taos -f ./step1.sim
+
+# wait 20 seconds, should not get an alert
+waitAlert 1 20
+res=$?
+if [ $res -eq 0 ]; then
+  echo 'should not have alerts here'
+  exit 1
+fi
+
+# step 2: trigger an alert
+../../td/debug/build/bin/tsim -c /etc/taos -f ./step2.sim
+
+# wait 30 seconds for the alert
+waitAlert 1 30 
+res=$?
+if [ $res -eq 1 ]; then
+  echo 'there should be an alert now'
+  exit 1
+fi
+
+# compare whether the generate alert meet expectation
+diff <(uniq alert.out | sed -n 1p | jq -cS 'del(.startsAt, .endsAt)') <(jq -cSn '{"values":{"avgspeed":10,"id":0},"labels":{"id":"0","ruleName":"test1"},"annotations":{"summary":"speed of car(id = 0) is too high: 10"}}')
+if [ $? -ne 0 ]; then
+  echo 'the generated alert does not meet expectation'
+  exit 1
+fi
+
+# step 3: add more data, trigger another 3 alerts
+../../td/debug/build/bin/tsim -c /etc/taos -f ./step3.sim
+
+# wait 30 seconds for the alerts
+waitAlert 4 30 
+res=$?
+if [ $res -eq 1 ]; then
+  echo 'there should be 4 alerts now'
+  exit 1
+fi
+
+# compare whether the generate alert meet expectation
+diff <(uniq alert.out | sed -n 2p | jq -cS 'del(.startsAt, .endsAt)') <(jq -cSn '{"annotations":{"summary":"speed of car(id = 0) is too high: 5.714285714285714"},"labels":{"id":"0","ruleName":"test1"},"values":{"avgspeed":5.714285714285714,"id":0}}')
+if [ $? -ne 0 ]; then
+  echo 'the generated alert does not meet expectation'
+  exit 1
+fi
+
+diff <(uniq alert.out | sed -n 3p | jq -cS 'del(.startsAt, .endsAt)') <(jq -cSn '{"annotations":{"summary":"speed of car(id = 1) is too high: 5.5"},"labels":{"id":"1","ruleName":"test1"},"values":{"avgspeed":5.5,"id":1}}')
+if [ $? -ne 0 ]; then
+  echo 'the generated alert does not meet expectation'
+  exit 1
+fi
+
+diff <(uniq alert.out | sed -n 4p | jq -cS 'del(.startsAt, .endsAt)') <(jq -cSn '{"annotations":{"summary":"speed of car(id = 2) is too high: 10"},"labels":{"id":"2","ruleName":"test1"},"values":{"avgspeed":10,"id":2}}')
+if [ $? -ne 0 ]; then
+  echo 'the generated alert does not meet expectation'
+  exit 1
+fi
+
+kill -INT `ps aux | grep 'alert -cfg' | grep -v grep | awk '{print $2}'`
diff --git a/alert/utils/config.go b/alert/utils/config.go
new file mode 100644
index 0000000000000000000000000000000000000000..71b9dd16e1f22784dbb01247485772aef4b49374
--- /dev/null
+++ b/alert/utils/config.go
@@ -0,0 +1,41 @@
+package utils
+
+import (
+	"encoding/json"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	Port     uint16 `json:"port,omitempty" yaml:"port,omitempty"`
+	Database string `json:"database,omitempty" yaml:"database,omitempty"`
+	RuleFile string `json:"ruleFile,omitempty" yaml:"ruleFile,omitempty"`
+	Log      struct {
+		Level string `json:"level,omitempty" yaml:"level,omitempty"`
+		Path  string `json:"path,omitempty" yaml:"path,omitempty"`
+	} `json:"log" yaml:"log"`
+	TDengine  string `json:"tdengine,omitempty" yaml:"tdengine,omitempty"`
+	Receivers struct {
+		AlertManager string `json:"alertManager,omitempty" yaml:"alertManager,omitempty"`
+		Console      bool   `json:"console"`
+	} `json:"receivers" yaml:"receivers"`
+}
+
+var Cfg Config
+
+func LoadConfig(path string) error {
+	f, e := os.Open(path)
+	if e != nil {
+		return e
+	}
+	defer f.Close()
+
+	e = yaml.NewDecoder(f).Decode(&Cfg)
+	if e != nil {
+		f.Seek(0, 0)
+		e = json.NewDecoder(f).Decode(&Cfg)
+	}
+
+	return e
+}
diff --git a/alert/utils/log/log.go b/alert/utils/log/log.go
new file mode 100644
index 0000000000000000000000000000000000000000..5f49c213ae90cd9cf1e2d20a03fa4442f8b22782
--- /dev/null
+++ b/alert/utils/log/log.go
@@ -0,0 +1,94 @@
+package log
+
+import (
+	"github.com/taosdata/alert/utils"
+	"go.uber.org/zap"
+)
+
+var logger *zap.SugaredLogger
+
+func Init() error {
+	var cfg zap.Config
+
+	if utils.Cfg.Log.Level == "debug" {
+		cfg = zap.NewDevelopmentConfig()
+	} else {
+		cfg = zap.NewProductionConfig()
+	}
+
+	if len(utils.Cfg.Log.Path) > 0 {
+		cfg.OutputPaths = []string{utils.Cfg.Log.Path}
+	}
+
+	l, e := cfg.Build()
+	if e != nil {
+		return e
+	}
+
+	logger = l.Sugar()
+	return nil
+}
+
+// Debug package logger
+func Debug(args ...interface{}) {
+	logger.Debug(args...)
+}
+
+// Debugf package logger
+func Debugf(template string, args ...interface{}) {
+	logger.Debugf(template, args...)
+}
+
+// Info package logger
+func Info(args ...interface{}) {
+	logger.Info(args...)
+}
+
+// Infof package logger
+func Infof(template string, args ...interface{}) {
+	logger.Infof(template, args...)
+}
+
+// Warn package logger
+func Warn(args ...interface{}) {
+	logger.Warn(args...)
+}
+
+// Warnf package logger
+func Warnf(template string, args ...interface{}) {
+	logger.Warnf(template, args...)
+}
+
+// Error package logger
+func Error(args ...interface{}) {
+	logger.Error(args...)
+}
+
+// Errorf package logger
+func Errorf(template string, args ...interface{}) {
+	logger.Errorf(template, args...)
+}
+
+// Fatal package logger
+func Fatal(args ...interface{}) {
+	logger.Fatal(args...)
+}
+
+// Fatalf package logger
+func Fatalf(template string, args ...interface{}) {
+	logger.Fatalf(template, args...)
+}
+
+// Panic package logger
+func Panic(args ...interface{}) {
+	logger.Panic(args...)
+}
+
+// Panicf package logger
+func Panicf(template string, args ...interface{}) {
+	logger.Panicf(template, args...)
+}
+
+func Sync() error {
+	return logger.Sync()
+}
diff --git a/cmake/define.inc b/cmake/define.inc
index 93bf6026106e4e2c6e788d2949446ab54b26813f..1bb1692a2b233851f8856c68f5f6a6fef5254bb6 100755
--- a/cmake/define.inc
+++ b/cmake/define.inc
@@ -1,10 +1,6 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
 IF (TD_ACCOUNT)
   ADD_DEFINITIONS(-D_ACCT)
 ENDIF ()
@@ -32,3 +28,107 @@ ENDIF ()
 IF (TD_RANDOM_NETWORK_FAIL)
   ADD_DEFINITIONS(-DTAOS_RANDOM_NETWORK_FAIL)
 ENDIF ()
+
+IF (TD_ARM)
+  ADD_DEFINITIONS(-D_TD_ARM_)
+  IF (TD_ARM_64)
+    ADD_DEFINITIONS(-D_TD_ARM_64_)
+  ENDIF ()
+  IF (TD_ARM_32)
+    ADD_DEFINITIONS(-D_TD_ARM_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_MIPS)
+  ADD_DEFINITIONS(-D_TD_MIPS_)
+  IF (TD_MIPS_64)
+    ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  ENDIF ()
+  IF (TD_MIPS_32)
+    ADD_DEFINITIONS(-D_TD_MIPS_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_LINUX_64)
+  IF (NOT TD_ARM)
+    IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ELSE ()
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ENDIF ()
+  ELSE ()
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  ENDIF ()
+  FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
+  IF (ICONV_INCLUDE_EXIST)
+    ADD_DEFINITIONS(-DUSE_LIBICONV)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_32)
+  ADD_DEFINITIONS(-D_TD_LINUX_32)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  IF (NOT TD_ARM)
+    EXIT ()
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX)
+  ADD_DEFINITIONS(-DLINUX)
+  ADD_DEFINITIONS(-D_TD_LINUX)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+  
+  IF (${COVER} MATCHES "true")
+    MESSAGE(STATUS "Test coverage mode, add extra flags")
+    SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+    SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+    SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+  ENDIF ()
+ENDIF ()
+
+IF (TD_APLHINE)
+  SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
+  link_libraries(/usr/lib/libargp.a)
+  ADD_DEFINITIONS(-D_ALPINE)
+ENDIF ()
+
+IF (TD_DARWIN_64)
+  ADD_DEFINITIONS(-D_TD_DARWIN_64)
+  ADD_DEFINITIONS(-DDARWIN)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+ENDIF ()
+
+IF (TD_WINDOWS)
+  ADD_DEFINITIONS(-DWINDOWS)
+  ADD_DEFINITIONS(-D__CLEANUP_C)
+  ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
+  ADD_DEFINITIONS(-DPTW32_BUILD)
+  ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
+  SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+  IF (NOT TD_GODLL)
+    SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(DEBUG_FLAGS "/Zi /W3 /GL")
+    SET(RELEASE_FLAGS "/W0 /GL")
+  ENDIF ()
+ENDIF () 
+
+IF (TD_WINDOWS_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_64)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+ENDIF ()
+
+IF (TD_WINDOWS_32)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_32)
+ENDIF ()
+
diff --git a/cmake/env.inc b/cmake/env.inc
index fbfa40e5ae11b4eddfa46c0c619dfd426676a047..67b934119d0e2de95c2ec60741cc4b78b7dbb267 100755
--- a/cmake/env.inc
+++ b/cmake/env.inc
@@ -9,7 +9,6 @@ SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
 SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
 
-MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
 MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
 MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
diff --git a/cmake/platform.inc b/cmake/platform.inc
index 2e0e2d6af08fa529f7435b2e39fa4cdb4d293fae..a0668de7b592f59ac817fd10ba1f33205b378aae 100755
--- a/cmake/platform.inc
+++ b/cmake/platform.inc
@@ -14,6 +14,7 @@ PROJECT(TDengine)
 #
 
 # Set macro definitions according to os platform
+SET(TD_LINUX FALSE)
 SET(TD_LINUX_64 FALSE)
 SET(TD_LINUX_32 FALSE)
 SET(TD_ARM FALSE)
@@ -22,8 +23,11 @@ SET(TD_ARM_32 FALSE)
 SET(TD_MIPS FALSE)
 SET(TD_MIPS_64 FALSE)
 SET(TD_MIPS_32 FALSE)
-SET(TD_DARWIN_64 FALSE)
+SET(TD_WINDOWS FALSE)
 SET(TD_WINDOWS_64 FALSE)
+SET(TD_WINDOWS_32 FALSE)
+SET(TD_DARWIN_64 FALSE)
+SET(TD_APLHINE FALSE)
 
 # if generate ARM version:
 #   cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
@@ -31,24 +35,21 @@ IF (${CPUTYPE} MATCHES "aarch32")
   SET(TD_ARM TRUE)
   SET(TD_ARM_32 TRUE)
   SET(TD_PAGMODE_LITE TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_32_)
+  MESSAGE(STATUS "input cpuType: aarch32")
 ELSEIF (${CPUTYPE} MATCHES "aarch64")
   SET(TD_ARM TRUE)
   SET(TD_ARM_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_64_)
+  MESSAGE(STATUS "input cpuType: aarch64")
 ELSEIF (${CPUTYPE} MATCHES "mips64")
   SET(TD_MIPS TRUE)
   SET(TD_MIPS_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_MIPS_)
-  ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  MESSAGE(STATUS "input cpuType: mips64")
 ELSEIF (${CPUTYPE} MATCHES "x64")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x64")
 ELSEIF (${CPUTYPE} MATCHES "x86")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x86")
 ELSE ()
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
 ENDIF ()
 
 #
@@ -59,16 +60,13 @@ execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_
 MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+  SET(TD_LINUX TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_LINUX_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
-    ADD_DEFINITIONS(-D_M_X64)
     MESSAGE(STATUS "The current platform is Linux 64-bit")
   ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
+    SET(TD_LINUX_32 TRUE)
     IF (TD_ARM)
-      SET(TD_LINUX_32 TRUE)
-      SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
-      #ADD_DEFINITIONS(-D_M_IX86)
       MESSAGE(STATUS "The current platform is Linux 32-bit")
     ELSE ()
       MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
@@ -78,22 +76,26 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
     EXIT ()
   ENDIF ()
+
+  IF (${TD_OS_INFO} MATCHES "Alpine")
+    SET(TD_APLHINE TRUE)
+    MESSAGE(STATUS "The current OS is Alpine, append extra flags")
+  ENDIF()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_DARWIN_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
     MESSAGE(STATUS "The current platform is Darwin 64-bit")
   ELSE ()
     MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+  SET(TD_WINDOWS TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_WINDOWS_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
-    ADD_DEFINITIONS(-D_M_X64)
     MESSAGE(STATUS "The current platform is Windows 64-bit")
   ELSE ()
+    SET(TD_WINDOWS_32 TRUE)
     MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
@@ -101,74 +103,3 @@ ELSE()
   MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
   EXIT ()
 ENDIF ()
-
-IF (TD_LINUX_64)
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    IF (NOT TD_ARM)
-      IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ELSE ()
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ENDIF ()
-    ELSE ()
-      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ENDIF ()
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, append extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_libraries(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_LINUX_32)
-    IF (NOT TD_ARM)
-      EXIT ()
-    ENDIF ()
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, add extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_library(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_WINDOWS_64)
-    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
-    IF (NOT TD_GODLL)
-      SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
-      SET(DEBUG_FLAGS "/Zi /W3 /GL")
-      SET(RELEASE_FLAGS "/W0 /GL")
-    ENDIF ()
-    ADD_DEFINITIONS(-DWINDOWS)
-    ADD_DEFINITIONS(-D__CLEANUP_C)
-    ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
-    ADD_DEFINITIONS(-DPTW32_BUILD)
-    ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
-  ELSEIF (TD_DARWIN_64)
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    ADD_DEFINITIONS(-DDARWIN)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-  ELSE ()
-    MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
-    EXIT ()
-  ENDIF ()
diff --git a/documentation/webdocs/markdowndocs/Super Table-ch.md b/documentation/webdocs/markdowndocs/Super Table-ch.md
index 5f592ec3393e50b25316fb29edd6abb13364e06a..f45f2f91b1ae6cf7e5ab3c8270de5c3c6bca6c97 100644
--- a/documentation/webdocs/markdowndocs/Super Table-ch.md	
+++ b/documentation/webdocs/markdowndocs/Super Table-ch.md	
@@ -4,7 +4,7 @@ TDengine要求每个数据采集点单独建表。独立建表的模式能够避
 
 ## 什么是超级表
 
-STable是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。
+超级表是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。
 
 TDengine扩展标准SQL语法用于定义STable，使用关键词tags指定标签信息。语法如下：
 
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md
index 343ce80422ea3d973907adbe438bfdd3021b2ddb..e494a1be45a49a0dc83d32c9d5be8d2316c7970b 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md	
+++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md	
@@ -556,7 +556,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
 
 - **APERCENTILE**
     ```mysql
@@ -565,7 +565,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
     
 - **LAST_ROW**
     ```mysql
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL.md b/documentation/webdocs/markdowndocs/TAOS SQL.md
index c0d35e9afcbf5853273f0db4c2181a0075f1e844..f94047fb6ac6a7d5a0cc25cb4789d9e686ed1baf 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL.md	
+++ b/documentation/webdocs/markdowndocs/TAOS SQL.md	
@@ -289,10 +289,10 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable. 
 
 
-- **WAVG**
+- **TWA**
   
     ```mysql
-    SELECT WAVG(field_name) FROM tb_name WHERE clause
+    SELECT TWA(field_name) FROM tb_name WHERE clause
     ```
     Function: return the time-weighted average value of a specific column  
     Return Data Type: `double`  
@@ -324,7 +324,7 @@ TDengine supports aggregations over numerical values, they are listed below:
 
 - **LEASTSQUARES**
     ```mysql
-    SELECT LEASTSQUARES(field_name) FROM tb_name [WHERE clause]
+    SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
     ```
     Function: performs a linear fit to the primary timestamp and the specified column. 
     Return Data Type: return a string of the coefficient and the interception of the fitted line.  
@@ -417,6 +417,15 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable.  
     Note: The range of `P` is `[0, 100]`. When `P=0` , `PERCENTILE` returns the equal value as `MIN`; when `P=100`, `PERCENTILE` returns the equal value as `MAX`. 
 
+- **APERCENTILE**
+    ```mysql
+    SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: the value of the specified column below which `P` percent of the data points fall, it returns approximate value of percentile.
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. 
+    Applied to: table/STable.  
+    Note: The range of `P` is `[0, 100]`. When `P=0` , `APERCENTILE` returns the equal value as `MIN`; when `P=100`, `APERCENTILE` returns the equal value as `MAX`. `APERCENTILE` has a much better performance than `PERCENTILE`.
 
 - **LAST_ROW**
     ```mysql
diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 289f2dbf91c08d047fbb9352186b8418646bbb49..b48cf23995950cb8cf084c6db3e1cdd86afa9c93 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -16,9 +16,9 @@
 
 TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括：
 
-- _taosd_：TDengine服务端可执行文件
-- _taos_： TDengine Shell可执行文件
-- _taosdump_：数据导出工具
+- _taosd_：TDengine服务端可执行文件。
+- _taos_： TDengine Shell可执行文件。
+- _taosdump_：数据导出工具。
 - *rmtaos*： 卸载TDengine的脚本, 该脚本会删除全部的程序和数据文件。请务必谨慎执行，如非必须不建议使用。
 
 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录
diff --git a/documentation20/Connections with other Tools-ch.md b/documentation20/Connections with other Tools-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..34be7d1e76363f535af9b03b91aa6f618405c602
--- /dev/null
+++ b/documentation20/Connections with other Tools-ch.md	
@@ -0,0 +1,149 @@
+# 与其他工具的连接
+
+
+## Grafana
+
+TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine中数据表中内容可以在仪表盘(DashBoard)上进行可视化展现。
+
+### 安装Grafana
+
+目前TDengine支持Grafana 5.2.4以上的版本。用户可以根据当前的操作系统，到Grafana官网下载安装包，并执行安装。下载地址如下：https://grafana.com/grafana/download。
+
+### 配置Grafana
+
+TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafana目录下。
+
+以CentOS 7.2操作系统为例，将tdengine目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。
+
+### 使用 Grafana
+
+#### 配置数据源
+
+用户可以直接通过 localhost:3000 的网址，登录 Grafana 服务器(用户名/密码:admin/admin)，通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示：
+
+![img](../assets/add_datasource1.jpg)
+
+点击 `Add data source` 可进入新增数据源页面，在查询框中输入 TDengine 可选择添加，如下图所示：
+
+![img](../assets/add_datasource2.jpg)
+
+进入数据源配置页面，按照默认提示修改相应配置即可：
+
+![img](../assets/add_datasource3.jpg)
+
+* Host： TDengine 集群的中任意一台服务器的 IP 地址与 TDengine RESTful 接口的端口号(6020)，默认 http://localhost:6020。
+* User：TDengine 用户名。
+* Password：TDengine 用户密码。
+
+点击 `Save & Test` 进行测试，成功会有如下提示：
+
+![img](../assets/add_datasource4.jpg)
+
+#### 创建 Dashboard
+
+回到主界面创建 Dashboard，点击 Add Query 进入面板查询页面：
+
+![img](../assets/create_dashboard1.jpg)
+
+如上图所示，在 Query 中选中 `TDengine` 数据源，在下方查询框可输入相应 sql 进行查询，具体说明如下：
+
+* INPUT SQL：输入要查询的语句（该 SQL 语句的结果集应为两列多行），例如：`select avg(mem_system) from log.dn where  ts >= $from and ts < $to interval($interval)` ，其中，from、to 和 interval 为 TDengine插件的内置变量，表示从Grafana插件面板获取的查询范围和时间间隔。除了内置变量外，`也支持可以使用自定义模板变量`。
+* ALIAS BY：可设置当前查询别名。 
+* GENERATE SQL： 点击该按钮会自动替换相应变量，并生成最终执行的语句。
+  
+
+按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下：
+
+![img](../assets/create_dashboard2.jpg)
+
+> 关于如何使用Grafana创建相应的监测界面以及更多有关使用Grafana的信息，请参考Grafana官方的[文档](https://grafana.com/docs/)。
+
+#### 导入 Dashboard
+
+在 Grafana 插件目录 /usr/local/taos/connector/grafana/tdengine/dashboard/ 下提供了一个 `tdengine-grafana.json` 可导入的 dashboard。
+
+点击左侧 `Import` 按钮，并上传 `tdengine-grafana.json` 文件：
+
+![img](../assets/import_dashboard1.jpg)
+
+导入完成之后可看到如下效果：
+
+![img](../assets/import_dashboard2.jpg)
+
+
+## Matlab
+
+MatLab可以通过安装包内提供的JDBC Driver直接连接到TDengine获取数据到本地工作空间。
+
+### MatLab的JDBC接口适配
+
+MatLab的适配有下面几个步骤，下面以Windows10上适配MatLab2017a为例：
+
+- 将TDengine安装包内的驱动程序JDBCDriver-1.0.0-dist.jar拷贝到${matlab_root}\MATLAB\R2017a\java\jar\toolbox
+- 将TDengine安装包内的taos.lib文件拷贝至${matlab_ root _dir}\MATLAB\R2017a\lib\win64
+- 将新添加的驱动jar包加入MatLab的classpath。在${matlab_ root _dir}\MATLAB\R2017a\toolbox\local\classpath.txt文件中添加下面一行
+
+​          `$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar`
+
+- 在${user_home}\AppData\Roaming\MathWorks\MATLAB\R2017a\下添加一个文件javalibrarypath.txt, 并在该文件中添加taos.dll的路径，比如您的taos.dll是在安装时拷贝到了C:\Windows\System32下，那么就应该在javalibrarypath.txt中添加如下一行：
+
+​          `C:\Windows\System32`
+
+### 在MatLab中连接TDengine获取数据
+
+在成功进行了上述配置后，打开MatLab。
+
+- 创建一个连接：
+
+  `conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’)`
+
+- 执行一次查询：
+
+  `sql0 = [‘select * from tb’]`
+
+  `data = select(conn, sql0);`
+
+- 插入一条记录:
+
+  `sql1 = [‘insert into tb values (now, 1)’]`
+
+  `exec(conn, sql1)`
+
+更多例子细节请参考安装包内examples\Matlab\TDengineDemo.m文件。
+
+## R 
+
+R语言支持通过JDBC接口来连接TDengine数据库。首先需要安装R语言的JDBC包。启动R语言环境，然后执行以下命令安装R语言的JDBC支持库：
+
+```R
+install.packages('RJDBC', repos='http://cran.us.r-project.org')
+```
+
+安装完成以后，通过执行`library('RJDBC')`命令加载 _RJDBC_ 包：
+
+然后加载TDengine的JDBC驱动：
+
+```R
+drv<-JDBC("com.taosdata.jdbc.TSDBDriver","JDBCDriver-2.0.0-dist.jar", identifier.quote="\"")
+```
+如果执行成功，不会出现任何错误信息。之后通过以下命令尝试连接数据库：
+
+```R
+conn<-dbConnect(drv,"jdbc:TSDB://192.168.0.1:0/?user=root&password=taosdata","root","taosdata")
+```
+
+注意将上述命令中的IP地址替换成正确的IP地址。如果没有任务错误的信息，则连接数据库成功，否则需要根据错误提示调整连接的命令。TDengine支持以下的 _RJDBC_ 包中函数：
+
+
+-	dbWriteTable(conn, "test", iris, overwrite=FALSE, append=TRUE)：将数据框iris写入表test中，overwrite必须设置为false，append必须设为TRUE,且数据框iris要与表test的结构一致。
+-	dbGetQuery(conn, "select count(*) from test")：查询语句
+-	dbSendUpdate(conn, "use db")：执行任何非查询sql语句。例如dbSendUpdate(conn, "use db")， 写入数据dbSendUpdate(conn, "insert into t1 values(now, 99)")等。
+-	dbReadTable(conn, "test")：读取表test中数据
+-	dbDisconnect(conn)：关闭连接
+-	dbRemoveTable(conn, "test")：删除表test
+
+TDengine客户端暂不支持如下函数：
+- dbExistsTable(conn, "test")：是否存在表test
+- dbListTables(conn)：显示连接中的所有表
+
+
diff --git a/documentation20/Connections with other Tools.md b/documentation20/Connections with other Tools.md
new file mode 100644
index 0000000000000000000000000000000000000000..8be05698497184aee2c41a60e32f39b636e2070e
--- /dev/null
+++ b/documentation20/Connections with other Tools.md	
@@ -0,0 +1,167 @@
+# Connect with other tools
+
+## Telegraf
+
+TDengine is easy to integrate with [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/), an open-source server agent for collecting and sending metrics and events, without more development.
+
+### Install Telegraf
+
+At present, TDengine supports Telegraf newer than version 1.7.4. Users can go to the [download link] and choose the proper package to install on your system.
+
+### Configure Telegraf
+
+Telegraf is configured by changing items in the configuration file */etc/telegraf/telegraf.conf*.
+
+
+In **output plugins** section，add _[[outputs.http]]_ iterm： 
+
+- _url_: http://ip:6020/telegraf/udb, in which _ip_ is the IP address of any node in TDengine cluster. Port 6020 is the RESTful APT port used by TDengine. _udb_ is the name of the database to save data, which needs to create beforehand.
+- _method_: "POST" 
+- _username_: username to login TDengine
+- _password_: password to login TDengine 
+- _data_format_: "json"
+- _json_timestamp_units_: "1ms"
+
+In **agent** part：
+
+- hostname: used to distinguish different machines. Need to be unique.
+- metric_batch_size: 30，the maximum number of records allowed to write in Telegraf. The larger the value is, the less frequent requests are sent. For TDengine, the value should be less than 50.
+
+Please refer to the [Telegraf docs](https://docs.influxdata.com/telegraf/v1.11/) for more information.
+
+## Grafana
+
+[Grafana] is an open-source system for time-series data display. It is easy to integrate TDengine and Grafana to build a monitor system. Data saved in TDengine can be fetched and shown on the Grafana dashboard.
+
+### Install Grafana
+
+For now, TDengine only supports Grafana newer than version 5.2.4. Users can go to the [Grafana download page] for the proper package to download.
+
+### Configure Grafana
+
+TDengine Grafana plugin is in the _/usr/local/taos/connector/grafana_ directory.
+Taking Centos 7.2 as an example, just copy TDengine directory to _/var/lib/grafana/plugins_ directory and restart Grafana.
+
+### Use Grafana
+
+Users can log in the Grafana server (username/password:admin/admin) through localhost:3000 to configure TDengine as the data source. As is shown in the picture below, TDengine as a data source option is shown in the box:
+
+
+![img](../assets/clip_image001.png)
+
+When choosing TDengine as the data source, the Host in HTTP configuration should be configured as the IP address of any node of a TDengine cluster. The port should be set as 6020. For example, when TDengine and Grafana are on the same machine, it should be configured as _http://localhost:6020. 
+
+
+Besides, users also should set the username and password used to log into TDengine. Then click _Save&Test_ button to save.
+
+![img](../assets/clip_image001-2474914.png)
+
+Then, TDengine as a data source should show in the Grafana data source list.
+
+![img](../assets/clip_image001-2474939.png)
+
+
+Then, users can create Dashboards in Grafana using TDengine as the data source:
+
+
+![img](../assets/clip_image001-2474961.png)
+
+ 
+
+Click _Add Query_ button to add a query and input the SQL command you want to run in the _INPUT SQL_ text box. The SQL command should expect a two-row, multi-column result, such as _SELECT count(*) FROM sys.cpu WHERE ts>=from and ts<​to interval(interval)_, in which, _from_, _to_ and _inteval_ are TDengine inner variables representing query time range and time interval.
+
+
+_ALIAS BY_ field is to set the query alias. Click _GENERATE SQL_ to send the command to TDengine:
+
+![img](../assets/clip_image001-2474987.png)
+
+Please refer to the [Grafana official document] for more information about Grafana.
+
+
+## Matlab
+
+Matlab can connect to and retrieve data from TDengine by TDengine JDBC Driver.
+
+### MatLab and TDengine JDBC adaptation
+
+Several steps are required to adapt Matlab to TDengine. Taking adapting Matlab2017a on Windows10 as an example:
+
+1. Copy the file _JDBCDriver-1.0.0-dist.jar_ in TDengine package to the directory _${matlab_root}\MATLAB\R2017a\java\jar\toolbox_
+2. Copy the file _taos.lib_ in TDengine package to _${matlab_ root _dir}\MATLAB\R2017a\lib\win64_
+3. Add the .jar package just copied to the Matlab classpath. Append the line below as the end of the file of _${matlab_ root _dir}\MATLAB\R2017a\toolbox\local\classpath.txt_
+
+​          `$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar`
+
+4. Create a file called _javalibrarypath.txt_ in directory _${user_home}\AppData\Roaming\MathWorks\MATLAB\R2017a\_, and add the _taos.dll_ path in the file. For example, if the file _taos.dll_ is in the directory of _C:\Windows\System32_，then add the following line in file *javalibrarypath.txt*:
+
+​          `C:\Windows\System32`
+
+### TDengine operations in Matlab
+
+After correct configuration, open Matlab:
+
+- build a connection：
+
+  `conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’)`
+
+- Query：
+
+  `sql0 = [‘select * from tb’]`
+
+  `data = select(conn, sql0);`
+
+- Insert a record:
+
+  `sql1 = [‘insert into tb values (now, 1)’]`
+
+  `exec(conn, sql1)`
+
+Please refer to the file _examples\Matlab\TDengineDemo.m_ for more information.
+
+## R 
+
+Users can use R language to access the TDengine server with the JDBC interface. At first, install JDBC package in R:
+
+```R
+install.packages('rJDBC', repos='http://cran.us.r-project.org')
+```
+
+Then use _library_ function to load the package:
+
+```R
+library('RJDBC')
+```
+
+Then load the TDengine JDBC driver:
+
+```R
+drv<-JDBC("com.taosdata.jdbc.TSDBDriver","JDBCDriver-1.0.0-dist.jar", identifier.quote="\"")
+```
+If succeed, no error message will display. Then use the following command to try a database connection:
+
+```R
+conn<-dbConnect(drv,"jdbc:TSDB://192.168.0.1:0/?user=root&password=taosdata","root","taosdata")
+```
+
+Please replace the IP address in the command above to the correct one. If no error message is shown, then the connection is established successfully. TDengine supports below functions in _RJDBC_ package:
+
+
+-	_dbWriteTable(conn, "test", iris, overwrite=FALSE, append=TRUE)_: write the data in a data frame _iris_ to the table _test_ in the TDengine server. Parameter _overwrite_ must be _false_. _append_ must be _TRUE_ and the schema of the data frame _iris_ should be the same as the table _test_.
+-	_dbGetQuery(conn, "select count(*) from test")_: run a query command
+-	_dbSendUpdate(conn, "use db")_: run any non-query command.
+-	_dbReadTable(conn, "test"_): read all the data in table _test_
+-	_dbDisconnect(conn)_: close a connection
+-	_dbRemoveTable(conn, "test")_: remove table _test_
+
+Below functions are **not supported** currently:
+- _dbExistsTable(conn, "test")_: if talbe _test_ exists
+- _dbListTables(conn)_: list all tables in the connection
+
+
+[Telegraf]: www.taosdata.com
+[download link]: https://portal.influxdata.com/downloads
+[Telegraf document]: www.taosdata.com
+[Grafana]: https://grafana.com
+[Grafana download page]: https://grafana.com/grafana/download
+[Grafana official document]: https://grafana.com/docs/
+
diff --git a/documentation20/Connector.md b/documentation20/Connector.md
new file mode 100644
index 0000000000000000000000000000000000000000..6d981478dfbad6220961514515e3c1bb51e13c06
--- /dev/null
+++ b/documentation20/Connector.md
@@ -0,0 +1,823 @@
+# TDengine connectors
+
+TDengine provides many connectors for development, including C/C++, JAVA, Python, RESTful, Go, Node.JS, etc.
+
+NOTE: All APIs which require a SQL string as parameter, including but not limit to `taos_query`, `taos_query_a`, `taos_subscribe` in the C/C++ Connector and their counterparts in other connectors, can ONLY process one SQL statement at a time. If more than one SQL statements are provided, their behaviors are undefined.
+
+## C/C++ API
+
+C/C++ APIs are similar to the MySQL APIs. Applications should include TDengine head file _taos.h_ to use C/C++ APIs by adding the following line in code:
+```C
+#include <taos.h>
+```
+Make sure TDengine library _libtaos.so_ is installed and use _-ltaos_ option to link the library when compiling. In most cases, if the return value of an API is integer, it return _0_ for success and other values as an error code for failure; if the return value is pointer, then _NULL_ is used for failure.
+
+
+### Fundamental API
+
+Fundamentatal APIs prepare runtime environment for other APIs, for example, create a database connection.
+
+- `void taos_init()`
+
+  Initialize the runtime environment for TDengine client. The API is not necessary since it is called int _taos_connect_ by default.
+
+
+- `void taos_cleanup()`
+
+  Cleanup runtime environment, client should call this API before exit.
+
+
+- `int taos_options(TSDB_OPTION option, const void * arg, ...)`
+
+   Set client options. The parameter _option_ supports values of _TSDB_OPTION_CONFIGDIR_ (configuration directory), _TSDB_OPTION_SHELL_ACTIVITY_TIMER_, _TSDB_OPTION_LOCALE_ (client locale) and _TSDB_OPTION_TIMEZONE_ (client timezone).
+
+
+- `char* taos_get_client_info()`
+
+  Retrieve version information of client.
+
+
+- `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)`
+
+  Open a connection to a TDengine server. The parameters are:
+  
+  * ip: IP address of the server
+  * user: username
+  * pass: password
+  * db: database to use, **NULL** for no database to use after connection. Otherwise, the database should exist before connection or a connection error is reported.
+  * port: port number to connect
+
+  The handle returned by this API should be kept for future use.
+
+
+- `char *taos_get_server_info(TAOS *taos)`
+
+  Retrieve version information of server.
+
+
+- `int taos_select_db(TAOS *taos, const char *db)`
+
+  Set default database to `db`.
+
+
+- `void taos_close(TAOS *taos)`
+
+  Close a connection to a TDengine server by the handle returned by _taos_connect_`
+
+
+### C/C++ sync API
+
+Sync APIs are those APIs waiting for responses from the server after sending a request. TDengine has the following sync APIs:
+
+- `TAOS_RES* taos_query(TAOS *taos, const char *sql)`
+
+  The API used to run a SQL command. The command can be DQL, DML or DDL. The parameter _taos_ is the handle returned by _taos_connect_. Return value _NULL_ means failure.
+
+
+- `int taos_result_precision(TAOS_RES *res)`
+
+  Get the timestamp precision of the result set, return value _0_ means milli-second, _1_ mean micro-second and _2_ means nano-second.
+
+
+- `TAOS_ROW taos_fetch_row(TAOS_RES *res)`
+
+  Fetch a row of return results through _res_.
+
+
+- `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)`
+
+  Fetch multiple rows from the result set, return value is row count.
+
+
+- `int taos_num_fields(TAOS_RES *res)` and `int taos_field_count(TAOS_RES* res)`
+
+  These two APIs are identical, both return the number of fields in the return result.
+
+
+- `int* taos_fetch_lengths(TAOS_RES *res)`
+
+  Get the field lengths of the result set, return value is an array whose length is the field count.
+
+
+- `int taos_affected_rows(TAOS_RES *res)`
+
+  Get affected row count of the executed statement.
+
+
+- `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)`
+
+  Fetch the description of each field. The description includes the property of data type, field name, and bytes. The API should be used with _taos_num_fields_ to fetch a row of data. The structure of `TAOS_FIELD` is:
+
+  ```c
+  typedef struct taosField {
+    char     name[65];  // field name
+    uint8_t  type;      // data type
+    int16_t  bytes;     // length of the field in bytes
+  } TAOS_FIELD;
+  ```
+
+
+- `void taos_stop_query(TAOS_RES *res)`
+
+  Stop the execution of a query.
+
+
+- `void taos_free_result(TAOS_RES *res)`
+
+  Free the resources used by a result set. Make sure to call this API after fetching results or memory leak would happen.
+
+
+- `char *taos_errstr(TAOS_RES *res)`
+
+  Return the reason of the last API call failure. The return value is a string.
+
+
+- `int *taos_errno(TAOS_RES *res)`
+
+  Return the error code of the last API call failure. The return value is an integer.
+
+
+**Note**: The connection to a TDengine server is not multi-thread safe. So a connection can only be used by one thread.
+
+
+### C/C++ async API
+
+In addition to sync APIs, TDengine also provides async APIs, which are more efficient. Async APIs are returned right away without waiting for a response from the server, allowing the application to continute with other tasks without blocking. So async APIs are more efficient, especially useful when in a poor network.
+
+All async APIs require callback functions. The callback functions have the format:
+```C
+void fp(void *param, TAOS_RES * res, TYPE param3)
+```
+The first two parameters of the callback function are the same for all async APIs. The third parameter is different for different APIs. Generally, the first parameter is the handle provided to the API for action. The second parameter is a result handle.
+
+- `void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param);`
+
+  The async version of _taos_query_.
+  
+  * taos: the handle returned by _taos_connect_.
+  * sql: the SQL command to run.
+  * fp: user defined callback function. The third parameter of the callback function _code_ is _0_ (for success) or a negative number (for failure, call taos_errstr to get the error as a string).  Applications mainly handle the second parameter, the returned result set.
+  * param: user provided parameter which is required by the callback function. 
+
+
+- `void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param);`
+
+  The async API to fetch a batch of rows, which should only be used with a _taos_query_a_ call.
+  
+  * res: result handle returned by _taos_query_a_.
+  * fp: the callback function. _param_ is a user-defined structure to pass to _fp_. The parameter _numOfRows_ is the number of result rows in the current fetch cycle. In the callback function, applications should call _taos_fetch_row_ to get records from the result handle. After getting a batch of results, applications should continue to call _taos_fetch_rows_a_ API to handle the next batch, until the _numOfRows_ is _0_ (for no more data to fetch) or _-1_ (for failure).
+
+
+- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);`
+
+  The async API to fetch a result row.
+
+  * res: result handle.
+  * fp: the callback function. _param_ is a user-defined structure to pass to _fp_. The third parameter of the callback function is a single result row, which is different from that of _taos_fetch_rows_a_ API. With this API, it is not necessary to call _taos_fetch_row_ to retrieve each result row, which is handier than _taos_fetch_rows_a_ but less efficient.
+
+
+Applications may apply operations on multiple tables. However, **it is important to make sure the operations on the same table are serialized**. That means after sending an insert request in a table to the server, no operations on the table are allowed before a response is received.
+
+
+### C/C++ parameter binding API
+
+TDengine also provides parameter binding APIs, like MySQL, only question mark `?` can be used to represent a parameter in these APIs.
+
+- `TAOS_STMT* taos_stmt_init(TAOS *taos)`
+
+  Create a TAOS_STMT to represent the prepared statement for other APIs.
+
+- `int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length)`
+
+  Parse SQL statement _sql_ and bind result to _stmt_ , if _length_ larger than 0, its value is used to determine the length of _sql_, the API auto detects the actual length of _sql_ otherwise.
+
+- `int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind)`
+
+  Bind values to parameters. _bind_ points to an array, the element count and sequence of the array must be identical as the parameters of the SQL statement. The usage of _TAOS_BIND_ is same as _MYSQL_BIND_ in MySQL, its definition is as below:
+
+  ```c
+  typedef struct TAOS_BIND {
+    int            buffer_type;
+    void *         buffer;
+    unsigned long  buffer_length;  // not used in TDengine
+    unsigned long *length;
+    int *          is_null;
+    int            is_unsigned;    // not used in TDengine
+    int *          error;          // not used in TDengine
+  } TAOS_BIND;
+  ```
+
+- `int taos_stmt_add_batch(TAOS_STMT *stmt)`
+
+  Add bound parameters to batch, client can call `taos_stmt_bind_param` again after calling this API. Note this API only support _insert_ / _import_ statements, it returns an error in other cases.
+
+- `int taos_stmt_execute(TAOS_STMT *stmt)`
+
+  Execute the prepared statement. This API can only be called once for a statement at present.
+
+- `TAOS_RES* taos_stmt_use_result(TAOS_STMT *stmt)`
+
+  Acquire the result set of an executed statement. The usage of the result is same as `taos_use_result`, `taos_free_result` must be called after one you are done with the result set to release resources.
+
+- `int taos_stmt_close(TAOS_STMT *stmt)`
+
+  Close the statement, release all resources.
+
+
+### C/C++ continuous query interface
+
+TDengine provides APIs for continuous query driven by time, which run queries periodically in the background. There are only two APIs:
+
+
+- `TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES * res, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *));`
+
+  The API is used to create a continuous query.
+  * _taos_: the connection handle returned by _taos_connect_.
+  * _sqlstr_: the SQL string to run. Only query commands are allowed.
+  * _fp_: the callback function to run after a query. TDengine passes query result `row`, query state `res` and user provided parameter `param` to this function. In this callback, `taos_num_fields` and `taos_fetch_fields` could be used to fetch field information.
+  * _param_: a parameter passed to _fp_
+  * _stime_: the time of the stream starts in the form of epoch milliseconds. If _0_ is given, the start time is set as the current time.
+  * _callback_: a callback function to run when the continuous query stops automatically.
+
+  The API is expected to return a handle for success. Otherwise, a NULL pointer is returned.
+
+
+- `void taos_close_stream (TAOS_STREAM *tstr)`
+
+  Close the continuous query by the handle returned by _taos_open_stream_. Make sure to call this API when the continuous query is not needed anymore.
+  
+
+### C/C++ subscription API
+
+For the time being, TDengine supports subscription on one or multiple tables. It is implemented through periodic pulling from a TDengine server. 
+
+* `TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval)`
+
+  The API is used to start a subscription session, it returns the subscription object on success and `NULL` in case of failure, the parameters are:
+  * **taos**: The database connnection, which must be established already.
+  * **restart**: `Zero` to continue a subscription if it already exits, other value to start from the beginning.
+  * **topic**: The unique identifier of a subscription.
+  * **sql**: A sql statement for data query, it can only be a `select` statement, can only query for raw data, and can only query data in ascending order of the timestamp field.
+  * **fp**: A callback function to receive query result, only used in asynchronization mode and should be `NULL` in synchronization mode, please refer below for its prototype.
+  * **param**: User provided additional parameter for the callback function.
+  * **interval**: Pulling interval in millisecond. Under asynchronization mode, API will call the callback function `fp` in this interval, system performance will be impacted if this interval is too short. Under synchronization mode, if the duration between two call to `taos_consume` is less than this interval, the second call blocks until the duration exceed this interval.
+
+* `typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code)`
+
+  Prototype of the callback function, the parameters are:
+  * tsub: The subscription object.
+  * res: The query result.
+  * param: User provided additional parameter when calling `taos_subscribe`.
+  * code: Error code in case of failures.
+
+* `TAOS_RES *taos_consume(TAOS_SUB *tsub)`
+
+  The API used to get the new data from a TDengine server. It should be put in an loop. The parameter `tsub` is the handle returned by `taos_subscribe`. This API should only be called in synchronization mode. If the duration between two call to `taos_consume` is less than pulling interval, the second call blocks until the duration exceed the interval. The API returns the new rows if new data arrives, or empty rowset otherwise, and if there's an error, it returns `NULL`.
+  
+* `void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress)`
+
+  Stop a subscription session by the handle returned by `taos_subscribe`. If `keepProgress` is **not** zero, the subscription progress information is kept and can be reused in later call to `taos_subscribe`, the information is removed otherwise.
+
+
+##  Java Connector
+
+TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
+
+由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
+
+* libtaos.so 
+    在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
+    
+* taos.dll
+    在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
+    
+> 注意：在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。
+
+TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点：
+
+* TDengine 不提供针对单条数据记录的删除和修改的操作，驱动中也没有支持相关方法。
+* 由于不支持删除和修改，所以也不支持事务操作。
+* 目前不支持表间的 union 操作。
+* 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。
+
+
+## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
+
+| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
+| --- | --- | --- |
+| 1.0.3 | 1.6.1.x 及以上 | 1.8.x |
+| 1.0.2 | 1.6.1.x 及以上 | 1.8.x |
+| 1.0.1 | 1.6.1.x 及以上 | 1.8.x |
+| 2.0.0 | 2.0.0.x 及以上 | 1.8.x |
+
+## TDengine DataType 和 Java DataType
+
+TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下：
+
+| TDengine DataType | Java DataType |
+| --- | --- |
+| TIMESTAMP | java.sql.Timestamp |
+| INT | java.lang.Integer |
+| BIGINT | java.lang.Long |
+| FLOAT | java.lang.Float |
+| DOUBLE | java.lang.Double |
+| SMALLINT, TINYINT |java.lang.Short  |
+| BOOL | java.lang.Boolean |
+| BINARY, NCHAR | java.lang.String |
+
+## 如何获取 TAOS-JDBCDriver
+
+### maven 仓库
+
+目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库，且各大仓库都已同步。
+* [sonatype][8]
+* [mvnrepository][9]
+* [maven.aliyun][10]
+
+maven 项目中使用如下 pom.xml 配置即可：
+
+```xml
+<dependencies>
+    <dependency>
+        <groupId>com.taosdata.jdbc</groupId>
+        <artifactId>taos-jdbcdriver</artifactId>
+        <version>2.0.0</version>
+    </dependency>
+</dependencies>
+```
+
+### 源码编译打包
+
+下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。
+
+
+## 使用说明
+
+### 获取连接
+
+如下所示配置即可获取 TDengine Connection：
+```java
+Class.forName("com.taosdata.jdbc.TSDBDriver");
+String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata";
+Connection conn = DriverManager.getConnection(jdbcUrl);
+```
+> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。
+
+TDengine 的 JDBC URL 规范格式为：
+`jdbc:TSDB://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
+
+其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下：
+
+* user：登录 TDengine 用户名，默认值 root。
+* password：用户登录密码，默认值 taosdata。
+* charset：客户端使用的字符集，默认值为系统字符集。
+* cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* locale：客户端语言环境，默认值系统当前 locale。
+* timezone：客户端使用的时区，默认值为系统当前时区。
+
+以上参数可以在 3 处配置，`优先级由高到低`分别如下：
+1. JDBC URL 参数
+    如上所述，可以在 JDBC URL 的参数中指定。
+2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
+```java
+public Connection getConn() throws Exception{
+  Class.forName("com.taosdata.jdbc.TSDBDriver");
+  String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata";
+  Properties connProps = new Properties();
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+  return conn;
+}
+```
+
+3. 客户端配置文件 taos.cfg
+
+    linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
+```properties
+# client default username
+# defaultUser           root
+
+# client default password
+# defaultPass           taosdata
+
+# default system charset
+# charset               UTF-8
+
+# system locale
+# locale                en_US.UTF-8
+```
+> 更多详细配置请参考[客户端配置][13]
+
+### 创建数据库和表
+
+```java
+Statement stmt = conn.createStatement();
+
+// create database
+stmt.executeUpdate("create database if not exists db");
+
+// use database
+stmt.executeUpdate("use db");
+
+// create table
+stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)");
+```
+> 注意：如果不使用 `use db` 指定数据库，则后续对表的操作都需要增加数据库名称作为前缀，如 db.tb。
+
+### 插入数据
+
+```java
+// insert data
+int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now + 1s, 20, 9.3)");
+
+System.out.println("insert " + affectedRows + " rows.");
+```
+> now 为系统内部函数，默认为服务器当前时间。
+> `now + 1s` 代表服务器当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。
+
+### 查询数据
+
+```java
+// query data
+ResultSet resultSet = stmt.executeQuery("select * from tb");
+
+Timestamp ts = null;
+int temperature = 0;
+float humidity = 0;
+while(resultSet.next()){
+
+    ts = resultSet.getTimestamp(1);
+    temperature = resultSet.getInt(2);
+    humidity = resultSet.getFloat("humidity");
+
+    System.out.printf("%s, %d, %s\n", ts, temperature, humidity);
+}
+```
+> 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
+
+
+### 关闭资源
+
+```java
+resultSet.close();
+stmt.close();
+conn.close();
+```
+> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。
+## 与连接池使用
+
+**HikariCP**
+
+* 引入相应 HikariCP maven 依赖：
+```xml
+<dependency>
+    <groupId>com.zaxxer</groupId>
+    <artifactId>HikariCP</artifactId>
+    <version>3.4.1</version>
+</dependency>
+```
+
+* 使用示例如下：
+```java
+ public static void main(String[] args) throws SQLException {
+    HikariConfig config = new HikariConfig();
+    config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log");
+    config.setUsername("root");
+    config.setPassword("taosdata");
+
+    config.setMinimumIdle(3);           //minimum number of idle connection
+    config.setMaximumPoolSize(10);      //maximum number of connection in the pool
+    config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool
+    config.setIdleTimeout(60000);       // max idle time for recycle idle connection 
+    config.setConnectionTestQuery("describe log.dn"); //validation query
+    config.setValidationTimeout(3000);   //validation query timeout
+
+    HikariDataSource ds = new HikariDataSource(config); //create datasource
+    
+    Connection  connection = ds.getConnection(); // get connection
+    Statement statement = connection.createStatement(); // get statement
+    
+    //query or insert 
+    // ...
+    
+    connection.close(); // put back to conneciton pool
+}
+```
+> 通过 HikariDataSource.getConnection() 获取连接后，使用完成后需要调用 close() 方法，实际上它并不会关闭连接，只是放回连接池中。
+> 更多 HikariCP 使用问题请查看[官方说明][5]
+
+## Python Connector
+
+### Install TDengine Python client
+
+Users can find python client packages in our source code directory _src/connector/python_. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use _pip_ command to install:
+
+```cmd
+pip install src/connector/python/python2/
+```
+
+or
+
+```
+pip install src/connector/python/python3/
+```
+
+If _pip_ command is not installed on the system, users can choose to install pip or just copy the _taos_ directory in the python client directory to the application directory to use.
+
+### Python client interfaces
+
+To use TDengine Python client, import TDengine module at first:
+
+```python
+import taos 
+```
+
+Users can get module information from Python help interface or refer to our [python code example](). We list the main classes and methods below:
+
+- _TDengineConnection_ class
+
+  Run `help(taos.TDengineConnection)` in python terminal for details.
+
+- _TDengineCursor_ class
+
+  Run `help(taos.TDengineCursor)` in python terminal for details.
+
+- connect method
+
+  Open a connection. Run `help(taos.connect)` in python terminal for details.
+
+## RESTful Connector
+
+TDengine also provides RESTful API to satisfy developing on different platforms. Unlike other databases, TDengine RESTful API applies operations to the database through the SQL command in the body of HTTP POST request. What users are required to provide is just a URL.
+
+
+For the time being, TDengine RESTful API uses a _\<TOKEN\>_ generated from username and password for identification. Safer identification methods will be provided in the future.
+
+
+### HTTP URL encoding 
+
+To use TDengine RESTful API, the URL should have the following encoding format:
+```
+http://<ip>:<PORT>/rest/sql
+```
+- _ip_: IP address of any node in a TDengine cluster
+- _PORT_: TDengine HTTP service port. It is 6020 by default.
+
+For example, the URL encoding _http://192.168.0.1:6020/rest/sql_ used to send HTTP request to a TDengine server with IP address as 192.168.0.1.
+
+It is required to add a token in an HTTP request header for identification.
+
+```
+Authorization: Basic <TOKEN>
+```
+
+The HTTP request body contains the SQL command to run. If the SQL command contains a table name, it should also provide the database name it belongs to in the form of `<db_name>.<tb_name>`. Otherwise, an error code is returned.
+
+For example, use _curl_ command to send a HTTP request:
+
+```
+curl -H 'Authorization: Basic <TOKEN>' -d '<SQL>' <ip>:<PORT>/rest/sql
+```
+
+or use
+
+```
+curl -u username:password -d '<SQL>' <ip>:<PORT>/rest/sql
+```
+
+where `TOKEN` is the encryted string of `{username}:{password}` using the Base64 algorithm, e.g. `root:taosdata` will be encoded as `cm9vdDp0YW9zZGF0YQ==`
+
+### HTTP response
+
+The HTTP resonse is in JSON format as below:
+
+```
+{
+    "status": "succ",
+    "head": ["column1","column2", …],
+    "data": [
+        ["2017-12-12 23:44:25.730", 1],
+        ["2017-12-12 22:44:25.728", 4]
+    ],
+    "rows": 2
+} 
+```
+Specifically,
+- _status_: the result of the operation, success or failure
+- _head_: description of returned result columns
+- _data_: the returned data array. If no data is returned, only an _affected_rows_ field is listed
+- _rows_: the number of rows returned
+
+### Example
+
+- Use _curl_ command to query all the data in table _t1_ of database _demo_:
+
+  `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql`
+
+The return value is like:
+
+```
+{
+    "status": "succ",
+    "head": ["column1","column2","column3"],
+    "data": [
+        ["2017-12-12 23:44:25.730", 1, 2.3],
+        ["2017-12-12 22:44:25.728", 4, 5.6]
+    ],
+    "rows": 2
+}
+```
+
+- Use HTTP to create a database：
+
+  `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql`
+
+    The return value should be:
+
+```
+{
+    "status": "succ",
+    "head": ["affected_rows"],
+    "data": [[1]],
+    "rows": 1,
+}
+```
+
+## Go Connector
+
+TDengine provides a GO client package `taosSql`. `taosSql` implements a kind of  interface of GO `database/sql/driver`. User can access TDengine by importing the package in their program with the following instructions, detailed usage please refer to `https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`
+
+```Go
+import (
+    "database/sql"
+    _ github.com/taosdata/driver-go/taoSql“
+)
+```
+### API
+
+* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB`
+
+   Open DB, generally DRIVER_NAME will be used as a constant with default value `taosSql`, dataSourceName is a combined String with format `user:password@/tcp(host:port)/dbname`. If user wants to access TDengine with multiple goroutine concurrently, the better way is to create an sql.Open object in each goroutine to access TDengine.
+
+  **Note**:	When calling this api, only a few initial work are done, instead the validity check happened during executing `Query` or `Exec`, at this time the connection will be created, and system will check if `user、password、host、port` is valid. Additionaly the most of features are implemented in the taosSql dependency lib `libtaos`, from this view, sql.Open is lightweight.
+
+*  `func (db *DB) Exec(query string, args ...interface{}) (Result, error)`
+
+  Execute non-Query related SQLs, the execution result is stored with type of Result.
+
+
+*  `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)`
+
+  Execute Query related SQLs, the execution result is *Raw, the detailed usage can refer GO interface `database/sql/driver`
+
+## Node.js Connector
+
+TDengine also provides a node.js connector package that is installable through [npm](https://www.npmjs.com/). The package is also in our source code at *src/connector/nodejs/*. The following instructions are also available [here](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)
+
+To get started, just type in the following to install the connector through [npm](https://www.npmjs.com/).
+
+```cmd
+npm install td-connector
+```
+
+It is highly suggested you use npm. If you don't have it installed, you can also just copy the nodejs folder from *src/connector/nodejs/* into your node project folder.
+
+To interact with TDengine, we make use of the [node-gyp](https://github.com/nodejs/node-gyp) library. To install, you will need to install the following depending on platform (the following instructions are quoted from node-gyp)
+
+### On Unix
+
+- `python` (`v2.7` recommended, `v3.x.x` is **not** supported)
+- `make`
+- A proper C/C++ compiler toolchain, like [GCC](https://gcc.gnu.org)
+
+### On macOS
+
+- `python` (`v2.7` recommended, `v3.x.x` is **not** supported) (already installed on macOS)
+
+- Xcode
+
+  - You also need to install the 
+
+    ```
+    Command Line Tools
+    ```
+
+     via Xcode. You can find this under the menu 
+
+    ```
+    Xcode -> Preferences -> Locations
+    ```
+
+     (or by running 
+
+    ```
+    xcode-select --install
+    ```
+
+     in your Terminal) 
+
+    - This step will install `gcc` and the related toolchain containing `make`
+
+### On Windows
+
+#### Option 1
+
+Install all the required tools and configurations using Microsoft's [windows-build-tools](https://github.com/felixrieseberg/windows-build-tools) using `npm install --global --production windows-build-tools` from an elevated PowerShell or CMD.exe (run as Administrator).
+
+#### Option 2
+
+Install tools and configuration manually:
+
+- Install Visual C++ Build Environment: [Visual Studio Build Tools](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) (using "Visual C++ build tools" workload) or [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) (using the "Desktop development with C++" workload)
+- Install [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` is not supported), and run `npm config set python python2.7` (or see below for further instructions on specifying the proper Python version and path.)
+- Launch cmd, `npm config set msvs_version 2017`
+
+If the above steps didn't work for you, please visit [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) for additional tips.
+
+To target native ARM64 Node.js on Windows 10 on ARM, add the  components "Visual C++ compilers and libraries for ARM64" and "Visual  C++ ATL for ARM64".
+
+### Usage
+
+The following is a short summary of the basic usage of the connector, the  full api and documentation can be found [here](http://docs.taosdata.com/node)
+
+#### Connection
+
+To use the connector, first require the library ```td-connector```. Running the function ```taos.connect``` with the connection options passed in as an object will return a TDengine connection object. The required connection option is ```host```, other options if not set, will be the default values as shown below.
+
+A cursor also needs to be initialized in order to interact with TDengine from Node.js.
+
+```javascript
+const taos = require('td-connector');
+var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0})
+var cursor = conn.cursor(); // Initializing a new cursor
+```
+
+To close a connection, run
+
+```javascript
+conn.close();
+```
+
+#### Queries
+
+We can now start executing simple queries through the ```cursor.query``` function, which returns a TaosQuery object.
+
+```javascript
+var query = cursor.query('show databases;')
+```
+
+We can get the results of the queries through the ```query.execute()``` function, which returns a promise that resolves with a TaosResult object, which contains the raw data and additional functionalities such as pretty printing the results.
+
+```javascript
+var promise = query.execute();
+promise.then(function(result) {
+  result.pretty(); //logs the results to the console as if you were in the taos shell
+});
+```
+
+You can also query by binding parameters to a query by filling in the question marks in a string as so. The query will automatically parse what was binded and convert it to the proper format for use with TDengine
+
+```javascript
+var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5);
+query.execute().then(function(result) {
+  result.pretty();
+})
+```
+
+The TaosQuery object can also be immediately executed upon creation by passing true as the second argument, returning a promise instead of a TaosQuery.
+
+```javascript
+var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true)
+promise.then(function(result) {
+  result.pretty();
+})
+```
+#### Async functionality
+
+Async queries can be performed using the same functions such as `cursor.execute`, `cursor.query`, but now with `_a` appended to them.
+
+Say you want to execute an two async query on two seperate tables, using `cursor.query_a`, you can do that and get a TaosQuery object, which upon executing with the `execute_a` function, returns a promise that resolves with a TaosResult object.
+
+```javascript
+var promise1 = cursor.query_a('select count(*), avg(v1), avg(v2) from meter1;').execute_a()
+var promise2 = cursor.query_a('select count(*), avg(v1), avg(v2) from meter2;').execute_a();
+promise1.then(function(result) {
+  result.pretty();
+})
+promise2.then(function(result) {
+  result.pretty();
+})
+```
+
+
+### Example
+
+An example of using the NodeJS connector to create a table with weather data and create and execute queries can be found [here](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js) (The preferred method for using the connector)
+
+An example of using the NodeJS connector to achieve the same things but without all the object wrappers that wrap around the data returned to achieve higher functionality can be found [here](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)
+
diff --git a/documentation20/Contributor_License_Agreement.md b/documentation20/Contributor_License_Agreement.md
new file mode 100644
index 0000000000000000000000000000000000000000..8c158da4c5958384064b9993de6643be86b94fee
--- /dev/null
+++ b/documentation20/Contributor_License_Agreement.md
@@ -0,0 +1,35 @@
+# TaosData Contributor License Agreement
+
+This TaosData Contributor License Agreement (CLA) applies to any contribution you make to any TaosData projects. If you are representing your employing organization to sign this agreement, please warrant that you have the authority to grant the agreement.
+
+## Terms
+
+**"TaosData"**, **"we"**, **"our"** and **"us"** means TaosData, inc.
+
+**"You"** and **"your"** means you or the organization you are on behalf of to sign this agreement.
+
+**"Contribution"** means any original work you, or the organization you represent submit to TaosData for any project in any manner.
+
+## Copyright License
+
+All rights of your Contribution submitted to TaosData in any manner are granted to TaosData and recipients of software distributed by TaosData. You waive any rights that my affect our ownership of the copyright and grant to us a perpetual, worldwide, transferable, non-exclusive, no-charge, royalty-free, irrevocable, and sublicensable license to use, reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Contributions and any derivative work created based on a Contribution.
+
+## Patent License
+
+With respect to any patents you own or that you can license without payment to any third party, you grant to us and to any recipient of software distributed by us, a perpetual, worldwide, transferable, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have make, use, sell, offer to sell, import, and otherwise transfer the Contribution in whole or in part, alone or included in any product under any patent you own, or license from a third party, that is necessarily infringed by the Contribution or by combination of the Contribution with any Work.
+
+## Your Representations and Warranties
+
+You represent and warrant that:
+
+- the Contribution you submit is an original work that you can legally grant the rights set out in this agreement.
+
+- the Contribution you submit and licenses you granted does not and will not, infringe the rights of any third party.
+
+- you are not aware of any pending or threatened claims, suits, actions, or charges pertaining to the contributions. You also warrant to notify TaosData immediately if you become aware of any such actual or potential claims, suits, actions, allegations or charges.
+
+## Support
+
+You are not obligated to support your Contribution except you volunteer to provide support. If you want, you can provide for a fee.
+
+**I agree and accept on behalf of myself and behalf of my organization:**
\ No newline at end of file
diff --git a/documentation20/Documentation-ch.md b/documentation20/Documentation-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..b6dbad07dc15a93f0cef3c47d48ef0a763eac531
--- /dev/null
+++ b/documentation20/Documentation-ch.md
@@ -0,0 +1,117 @@
+#TDengine文档
+
+TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是[数据模型](data-model-and-architecture)与数据建模一节。除本文档之外，欢迎[下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。
+
+##TDengine 介绍
+- TDengine 简介及特色
+- TDengine 适用场景
+- TDengine 性能指标介绍和验证方法
+
+##立即开始
+- 快捷安装：可通过源码、安装包或docker安装，三秒钟搞定
+- 轻松启动：使用systemctl 启停TDengine
+- 命令行程序TAOS：访问TDengine的简便方式 
+- [极速体验](https://www.taosdata.com/cn/getting-started/#TDengine-极速体验)：运行示例程序，快速体验高效的数据插入、查询
+
+##数据模型和整体架构
+- 数据模型：关系型数据库模型，但要求每个采集点单独建表  
+- 集群与基本逻辑单元：吸取NoSQL优点，支持水平扩展，支持高可靠
+- 存储模型与数据分区：标签数据与时序数据完全分离，按vnode和时间两个维度对数据切分
+- 数据写入与复制流程：先写入WAL、之后写入缓存，再给应用确认，支持多副本
+- 缓存与持久化：最新数据缓存在内存中，但落盘时采用列式存储、超高压缩比 
+- 高效查询：支持各种函数、时间轴聚合、插值、多表聚合
+
+##数据建模
+- 创建库：为具有相似数据特征的数据采集点创建一个库
+- 创建超级表：为同一类型的数据采集点创建一个超级表
+- 创建表：使用超级表做模板，为每一个具体的数据采集点单独建表
+
+##高效写入数据
+- SQL写入：使用SQL insert命令向一张或多张表写入单条或多条记录
+- Telegraf 写入：配置Telegraf, 不用任何代码，将采集数据直接写入
+- Prometheus写入：配置Prometheus, 不用任何代码，将数据直接写入
+- EMQ X Broker：配置EMQ X，不用任何代码，就可将MQTT数据直接写入
+
+##高效查询数据
+- 主要查询功能：支持各种标准函数，设置过滤条件，时间段查询
+- 多表聚合查询：使用超级表，设置标签过滤条件，进行高效聚合查询
+- 降采样查询：按时间段分段聚合，支持插值
+
+##高级功能
+- 连续查询(Continuous Query)：基于滑动窗口，定时自动的对数据流进行查询计算
+- 数据订阅(Publisher/Subscriber)：象典型的消息队列，应用可订阅接收到的最新数据
+- [缓存 (Cache)](https://www.taosdata.com/cn/documentation/advanced-features/#缓存-(Cache))：每个设备最新的数据都会缓存在内存中，可快速获取
+- [报警监测(Alarm monitoring)](https://www.taosdata.com/blog/2020/04/14/1438.html/)：根据配置规则，自动监测超限行为数据，并主动推送
+
+##连接器
+- C/C++ Connector：通过libtaos客户端的库，连接TDengine服务器的主要方法
+- Java Connector(JDBC)：通过标准的JDBC API，给Java应用提供到TDengine的连接
+- Python Connector：给Python应用提供一个连接TDengine服务器的驱动 
+- RESTful Connector：提供一最简单的连接TDengine服务器的方式
+- Go Connector：给Go应用提供一个连接TDengine服务器的驱动
+- Node.js Connector：给node应用提供一个链接TDengine服务器的驱动
+
+##与其他工具的连接
+- Grafana：获取并可视化保存在TDengine的数据
+- Matlab：通过配置Matlab的JDBC数据源访问保存在TDengine的数据
+- R：通过配置R的JDBC数据源访问保存在TDengine的数据 
+
+## TDengine集群的安装、管理
+
+- 安装：与单节点的安装一样，但要设好配置文件里的参数first
+- 节点管理：增加、删除、查看集群的节点
+- mnode的管理：系统自动创建、无需任何人工干预
+- 负载均衡：一旦节点个数或负载有变化，自动进行
+- 节点离线处理：节点离线超过一定时长，将从集群中剔除
+- Arbitrator：对于偶数个副本的情形，使用它可以防止split brain。
+
+##TDengine的运营和维护
+
+- 容量规划：根据场景，估算硬件资源
+- 容错和灾备：设置正确的WAL和数据副本数 
+- 系统配置：端口，缓存大小，文件块大小和其他系统配置
+- 用户管理：添加、删除TDengine用户，修改用户密码
+- 数据导入：可按脚本文件导入，也可按数据文件导入
+- 数据导出：从shell按表导出，也可用taosdump工具做各种导出
+- 系统监控：检查系统现有的连接、查询、流式计算，日志和事件等
+- 文件目录结构：TDengine数据文件、配置文件等所在目录 Hui Li
+
+##TAOS SQL 
+- 支持的数据类型：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 
+- 数据库管理：添加、删除、查看数据库
+- 表管理：添加、删除、查看、修改表
+- 超级表管理：添加、删除、查看、修改超级表
+- 标签管理：增加、删除、修改标签
+- 数据写入：支持单表单条、多条、多表多条写入，支持历史数据写入
+- 数据查询：支持时间段、值过滤、排序、查询结果手动分页等
+- SQL函数：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等
+- 时间维度聚合：将表中数据按照时间段进行切割后聚合，降维处理
+
+##TDengine的技术设计
+- 系统模块：taosd的功能和模块划分
+- 技术博客：更多的技术分析和架构设计文章
+
+## 常用工具
+
+- [TDengine样例数据导入工具](https://www.taosdata.com/cn/documentation/blog/2020/01/18/如何快速验证性能和主要功能？tdengine样例数据导入工/)
+- [TDengine性能对比测试工具](https://www.taosdata.com/cn/documentation/blog/2020/01/13/用influxdb开源的性能测试工具对比influxdb和tdengine/)
+
+##TDengine与其他数据库的对比测试
+
+- [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/cn/documentation/blog/2020/01/13/用influxdb开源的性能测试工具对比influxdb和tdengine/)
+- [TDengine与OpenTSDB对比测试](https://www.taosdata.com/cn/documentation/blog/2019/08/21/tdengine与opentsdb对比测试/)
+- [TDengine与Cassandra对比测试](https://www.taosdata.com/cn/documentation/blog/2019/08/14/tdengine与cassandra对比测试/)
+- [TDengine与InfluxDB对比测试](https://www.taosdata.com/cn/documentation/blog/2019/07/19/tdengine与influxdb对比测试/)
+- [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf)
+
+##物联网大数据
+- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/物联网、工业互联网大数据的特点/)
+- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/物联网大数据平台应具备的功能和特点/)
+- [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/通用互联网大数据处理架构为什么不适合处理物联/)
+- [物联网、车联网、工业互联网大数据平台，为什么推荐使用TDengine？](https://www.taosdata.com/blog/2019/07/09/物联网、车联网、工业互联网大数据平台，为什么/)
+
+##培训和FAQ
+- <a href='https://www.taosdata.com/en/faq'>FAQ</a>：常见问题与答案
+- <a href='https://www.taosdata.com/en/blog/?categories=4'>应用案列</a>：一些使用实例来解释如何使用TDengine
+
+ 
\ No newline at end of file
diff --git a/documentation20/Documentation.md b/documentation20/Documentation.md
new file mode 100644
index 0000000000000000000000000000000000000000..bdafd40f7c76425a4f9734a2561b2b9a945c757f
--- /dev/null
+++ b/documentation20/Documentation.md
@@ -0,0 +1,87 @@
+#Documentation 
+
+TDengine is a highly efficient platform to store, query, and analyze time-series data. It works like a relational database, but you are strongly suggested to read through the following documentation before you experience it.
+
+##Getting Started
+
+- Quick Start: download, install and experience TDengine in a few seconds
+- TDengine Shell: command-line interface to access TDengine server
+- Major Features: insert/query, aggregation, cache,  pub/sub, continuous query   
+
+## Data Model and Architecture
+
+- Data Model: relational database model, but one table for one device with static tags
+- Architecture: Management Module, Data Module, Client Module
+- Writing Process: records recieved are written to WAL, cache, then ack is sent back to client
+- Data Storage: records are sharded in the time range, and stored column by column 
+
+##TAOS SQL 
+
+- Data Types: support timestamp, int, float, double, binary, nchar, bool, and other types
+- Database Management: add, drop, check databases
+- Table Management: add, drop, check, alter tables
+- Inserting Records: insert one or more records into tables, historical records can be imported
+- Data Query: query data with time range and filter conditions, support limit/offset
+- SQL Functions: support aggregation, selector, transformation functions
+- Downsampling: aggregate data in successive time windows, support interpolation
+
+##STable: Super Table
+
+- What is a Super Table: an innovated way to aggregate tables
+- Create a STable: it is like creating a standard table, but with tags defined
+- Create a Table via STable: use STable as the template, with tags specified
+- Aggregate Tables via STable: group tables together by specifying the tags filter condition
+- Create Table Automatically: create tables automatically with a STable as a template
+- Management of STables: create/delete/alter super table just like standard tables
+- Management of Tags: add/delete/alter tags on super tables or tables  
+
+##Advanced Features
+
+- Continuous Query: query executed by TDengine periodically with a sliding window
+- Publisher/Subscriber: subscribe to the newly arrived data like a typical messaging system 
+- Caching: the newly arrived data of each device/table will always be cached
+
+##Connector
+
+- C/C++ Connector: primary method to connect to the server through libtaos client library
+- Java Connector: driver for connecting to the server from Java applications using the JDBC API
+- Python Connector: driver for connecting to the server from Python applications 
+- RESTful Connector: a simple way to interact with TDengine via HTTP
+- Go Connector: driver for connecting to the server from Go applications
+- Node.js Connector: driver for connecting to the server from node applications
+
+##Connections with Other Tools
+
+- Telegraf: pass the collected DevOps metrics to TDengine 
+- Grafana: query the data saved in TDengine and visualize them 
+- Matlab: access TDengine server from Matlab via JDBC
+- R: access TDengine server from R via JDBC 
+
+##Administrator
+
+- Directory and Files: files and directories related with TDengine
+- Configuration on Server: customize IP port, cache size, file block size and other settings
+- Configuration on Client: customize locale, default user and others   
+- User Management: add/delete users, change passwords
+- Import Data: import data into TDengine from either script or CSV file
+- Export Data: export data either from TDengine shell or from tool taosdump
+- Management of Connections, Streams, Queries: check or kill the connections, queries
+- System Monitor: collect the system metric, and log important operations
+
+##More on System Architecture
+
+- Storage Design: column-based storage with optimization on time-series data 
+- Query Design: an efficient way to query time-series data
+- Technical blogs to delve into the inside of TDengine
+
+## More on IoT Big Data
+
+- [Characteristics of IoT Big Data](https://www.taosdata.com/blog/2019/07/09/characteristics-of-iot-big-data/)
+- [Why don’t General Big Data Platforms Fit IoT Scenarios?](https://www.taosdata.com/blog/2019/07/09/why-does-the-general-big-data-platform-not-fit-iot-data-processing/)
+- [Why TDengine is the Best Choice for IoT Big Data Processing?](https://www.taosdata.com/blog/2019/07/09/why-tdengine-is-the-best-choice-for-iot-big-data-processing/)
+
+##Tutorials & FAQ
+
+- <a href='https://www.taosdata.com/en/faq'>FAQ</a>: a list of frequently asked questions and answers 
+- <a href='https://www.taosdata.com/en/blog/?categories=4'>Use cases</a>: a few typical cases to explain how to use TDengine in IoT platform
+
diff --git a/documentation20/Evaluation-ch.md b/documentation20/Evaluation-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa4ab14af8b4f03cb3af0187ad35ee3716d87f58
--- /dev/null
+++ b/documentation20/Evaluation-ch.md
@@ -0,0 +1,64 @@
+# TDengine 适用场景介绍（草案）
+
+## TDengine 简介
+
+<!-- 本节内容来源于白皮书 -->
+
+TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。
+
+* __10倍以上的性能提升__：定义了创新的数据存储结构，单核每秒就能处理至少2万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快了十倍以上。
+* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10
+* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融合一起，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 
+* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。临时查询可通过Shell, Python, R, Matlab随时进行。
+* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
+* __零运维成本、零学习成本__：安装、集群一秒搞定，无需分库分表，实时备份。标准SQL，支持JDBC, RESTful, 支持Python/Java/C/C++/Go, 与MySQL相似，零学习成本。
+
+
+
+<!--sdASF -->
+
+## TDengine 总体适用场景
+
+作为一个IOT大数据平台，TDengine的典型适用场景是在IOT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。
+
+
+## 数据源特点和需求
+从数据源角度，设计人员可以从已经角度分析TDengine在目标应用系统里面的适用性。
+
+|数据源特点和需求|不适用|可能适用|非常适用|简单说明|
+|---|---|---|---|---|
+|总体数据量巨大|  |  | ✅ |TDengine在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。|
+|数据输入速度偶尔或者持续巨大|   |   | ✅  | TDengine的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。|
+|数据源数目巨大|   |   | ✅ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。|
+
+
+
+## 系统架构要求
+|系统架构要求|不适用|可能适用|非常适用|简单说明|
+|---|---|---|---|---|
+|要求简单可靠的系统架构|  |  | ✅ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。|
+|要求容错和高可靠|  |  | ✅ |TDengine的集群功能，自动提供容错灾备等高可靠功能|
+|标准化规范|  |  | ✅ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范|
+
+## 系统功能需求
+|系统功能需求|不适用|可能适用|非常适用|简单说明|
+|---|---|---|---|---|
+|要求完整的内置数据处理算法|  | ✅ |  |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。|
+|需要大量的交叉查询处理|  | ✅ |  |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能|
+
+
+## 系统性能需求
+|系统性能需求|不适用|可能适用|非常适用|简单说明|
+|---|---|---|---|---|
+|要求较大的总体处理能力|  |  | ✅ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。|
+|要求高速处理数据  | | | ✅ |TDengine的专门为IOT优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。|
+|要求快速处理小粒度数据|  |  | ✅ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。|
+
+
+## 系统维护需求
+|系统维护需求|不适用|可能适用|非常适用|简单说明|
+|---|---|---|---|---|
+|要求系统可靠运行|  |  | ✅ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。|
+|要求运维学习成本可控|  |  | ✅ |同上|
+|要求市场有大量人才储备| ✅ |  |  |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务|
+
diff --git a/documentation20/Getting Started-ch.md b/documentation20/Getting Started-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..4b109aa8e6d03804ee95d0ab29b7628778187a56
--- /dev/null
+++ b/documentation20/Getting Started-ch.md	
@@ -0,0 +1,169 @@
+# 立即开始
+
+## 快捷安装
+
+TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版仅能在Linux系统上安装和运行，后续会支持Windows、MAC OS等系统。如果应用需要在Windows或Mac上运行，目前只能使用TDengine的RESTful接口连接服务器。硬件支持X64，后续会支持ARM、龙芯等CPU系统。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。
+
+### 通过源码安装
+
+请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装.
+
+### 通过Docker容器运行
+
+请参考[TDengine官方Docker镜像的发布、下载和使用](https://www.taosdata.com/blog/2020/05/13/1509.html)
+
+### 通过安装包安装
+
+服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。
+
+<ul id='packageList'>
+<li><a id='tdengine-rpm' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.rpm (5.3M)</a></li>
+<li><a id='tdengine-deb' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.deb (2.5M)</a></li>
+<li><a id='tdengine-tar' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.tar.gz (5.3M)</a></li>
+</ul>
+
+客户端部分，Linux安装包如下：
+
+- TDengine-client-2.0.0.0-Linux-x64.tar.gz (3.4M)
+
+报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）：
+
+- TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M)
+
+目前，TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which`命令来检测系统中是否存在`systemd`:
+
+```cmd
+which systemd
+```
+
+如果系统中不存在`systemd`命令，请考虑[通过源码安装](#通过源码安装)TDengine。
+
+具体的安装过程，请参见[`TDengine多种安装包的安装和卸载`](https://www.taosdata.com/blog/2019/08/09/566.html)
+
+## 轻松启动
+
+安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。
+
+```cmd
+systemctl start taosd
+```
+
+检查服务是否正常工作。
+```cmd
+systemctl status taosd
+```
+
+如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。
+
+**注：_systemctl_ 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 _sudo_**
+
+## TDengine命令行程序
+
+执行TDengine命令行程序，您只要在Linux终端执行`taos`即可
+
+```cmd
+taos
+```
+
+如果TDengine终端链接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/faq/)来解决终端链接服务端失败的问题）。TDengine终端的提示符号如下：
+
+```cmd
+taos>
+```
+
+在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例：
+
+```mysql
+create database db;
+use db;
+create table t (ts timestamp, cdata int);
+insert into t values ('2019-07-15 00:00:00', 10);
+insert into t values ('2019-07-15 01:00:00', 20);
+select * from t;
+          ts          |   speed   |
+===================================
+ 19-07-15 00:00:00.000|         10|
+ 19-07-15 01:00:00.000|         20|
+Query OK, 2 row(s) in set (0.001700s)
+```
+
+除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。
+
+### 命令行参数
+
+您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数：
+
+- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_
+- -h, --host: 指定服务的IP地址，默认为本地服务
+- -s, --commands: 在不进入终端的情况下运行TDengine命令
+- -u, -- user:  链接TDengine服务器的用户名，缺省为root
+- -p, --password: 链接TDengine服务器的密码，缺省为taosdata
+- -?, --help: 打印出所有命令行参数
+
+示例：
+
+```cmd
+taos -h 192.168.0.1 -s "use db; show tables;"
+```
+
+### 运行SQL命令脚本
+
+TDengine终端可以通过`source`命令来运行SQL命令脚本.
+
+```
+taos> source <filename>;
+```
+
+### Shell小技巧
+
+- 可以使用上下光标键查看已经历史输入的命令
+- 修改用户密码。在shell中使用alter user命令
+- ctrl+c 中止正在进行中的查询
+- 执行`RESET QUERY CACHE`清空本地缓存的表的schema
+
+## TDengine 极速体验
+
+启动TDengine的服务，在Linux终端执行taosdemo
+
+```
+> taosdemo
+```
+
+该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。
+
+执行这条命令大概需要10分钟，最后共插入10亿条记录。
+
+在TDengine客户端输入查询命令，体验查询速度。
+
+- 查询超级表下记录总条数：
+
+```
+taos>select count(*) from test.meters;
+```
+
+- 查询10亿条记录的平均值、最大值、最小值等：
+
+```
+taos>select avg(f1), max(f2), min(f3) from test.meters;
+```
+
+- 查询loc="beijing"的记录总条数：
+
+```
+taos>select count(*) from test.meters where loc="beijing";
+```
+
+- 查询areaid=10的所有记录的平均值、最大值、最小值等：
+
+```
+taos>select avg(f1), max(f2), min(f3) from test.meters where areaid=10;
+```
+
+- 对表t10按10s进行平均值、最大值和最小值聚合统计：
+
+```
+taos>select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
+```
+
+**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。
+
diff --git a/documentation20/Getting Started.md b/documentation20/Getting Started.md
new file mode 100644
index 0000000000000000000000000000000000000000..00d97d3d9cd3d2ce8317938eb9e46ae74b48dab1
--- /dev/null
+++ b/documentation20/Getting Started.md	
@@ -0,0 +1,151 @@
+#Getting Started
+
+## Quick Start
+
+At the moment, TDengine only runs on Linux. You can set up and install it either from the  <a href='#Install-from-Source'>source code</a> or the <a href='#Install-from-Package'>packages</a>. It takes only a few seconds from download to run it successfully. 
+
+### Install from Source
+
+Please visit our [github page](https://github.com/taosdata/TDengine) for instructions on installation from the source code.
+
+### Install from Package
+
+Three different packages are provided, please pick up the one you like. 
+<ul id='packageList'>
+<li><a id='tdengine-rpm' style='color:var(--b2)'>TDengine RPM package (1.5M)</a></li>
+<li><a id='tdengine-deb' style='color:var(--b2)'>TDengine DEB package (1.7M)</a></li>
+<li><a id='tdengine-tar' style='color:var(--b2)'>TDengine Tarball (3.0M)</a></li>
+</ul>
+For the time being, TDengine only supports installation on Linux systems using [`systemd`](https://en.wikipedia.org/wiki/Systemd) as the service manager. To check if your system has *systemd*, use the _which_ command.
+
+```cmd
+which systemd
+```
+
+If the `systemd` command is not found, please [install from source code](#Install-from-Source). 
+
+### Running TDengine
+
+After installation, start the TDengine service by the `systemctl` command.
+
+```cmd
+systemctl start taosd
+```
+
+Then check if the server is working now.
+```cmd
+systemctl status taosd
+```
+
+If the service is running successfully, you can play around through TDengine shell `taos`, the command line interface tool located in directory /usr/local/bin/taos 
+
+**Note: The _systemctl_ command needs the root privilege. Use _sudo_ if you are not the _root_ user.**
+
+##TDengine Shell
+To launch TDengine shell, the command line interface, in a Linux terminal, type:
+
+```cmd
+taos
+```
+
+The welcome message is printed if the shell connects to TDengine server successfully, otherwise, an error message will be printed (refer to our [FAQ](../faq) page for troubleshooting the connection error). The TDengine shell prompt is: 
+
+```cmd
+taos>
+```
+
+In the TDengine shell, you can create databases, create tables and insert/query data with SQL. Each query command ends with a semicolon. It works like MySQL, for example:
+
+```mysql
+create database db;
+use db;
+create table t (ts timestamp, cdata int);
+insert into t values ('2019-07-15 10:00:00', 10);
+insert into t values ('2019-07-15 10:01:05', 20);
+select * from t;
+          ts          |   speed   |
+===================================
+ 19-07-15 10:00:00.000|         10|
+ 19-07-15 10:01:05.000|         20|
+Query OK, 2 row(s) in set (0.001700s)
+```
+
+Besides the SQL commands, the system administrator can check system status, add or delete accounts, and manage the servers.
+
+###Shell Command Line Parameters
+
+You can run `taos` command with command line options to fit your needs. Some frequently used options are listed below:
+
+- -c, --config-dir: set the configuration directory. It is _/etc/taos_ by default
+- -h, --host: set the IP address of the server it will connect to, Default is localhost
+- -s, --commands: set the command to run without entering the shell
+- -u, -- user:  user name to connect to server. Default is root
+- -p, --password: password. Default is 'taosdata'
+- -?, --help: get a full list of supported options 
+
+Examples:
+
+```cmd
+taos -h 192.168.0.1 -s "use db; show tables;"
+```
+
+###Run Batch Commands
+
+Inside TDengine shell, you can run batch commands in a file with *source* command.
+
+```
+taos> source <filename>;
+```
+
+### Tips
+
+- Use up/down arrow key to check the command history
+- To change the default password, use "`alter user`" command 
+- ctrl+c to interrupt any queries 
+- To clean the cached schema of tables or STables, execute command `RESET QUERY CACHE` 
+
+## Major Features
+
+The core functionality of TDengine is the time-series database. To reduce the development and management complexity, and to improve the system efficiency further, TDengine also provides caching, pub/sub messaging system, and stream computing functionalities. It provides a full stack for IoT big data platform. The detailed features are listed below:
+
+- SQL like query language used to insert or explore data
+
+- C/C++, Java(JDBC), Python, Go, RESTful, and Node.JS interfaces for development
+
+- Ad hoc queries/analysis via Python/R/Matlab or TDengine shell
+
+- Continuous queries to support sliding-window based stream computing
+
+- Super table to aggregate multiple time-streams efficiently with flexibility  
+
+- Aggregation over a time window on one or multiple time-streams
+
+- Built-in messaging system to support publisher/subscriber model
+
+- Built-in cache for each time stream to make latest data available as fast as light speed
+
+- Transparent handling of historical data and real-time data 
+
+- Integrating with Telegraf, Grafana and other tools seamlessly  
+
+- A set of tools or configuration to manage TDengine 
+
+
+For enterprise edition, TDengine provides more advanced features below:
+
+- Linear scalability to deliver higher capacity/throughput 
+
+- High availability to guarantee the carrier-grade service  
+
+- Built-in replication between nodes which may span multiple geographical sites 
+
+- Multi-tier storage to make historical data management simpler and cost-effective
+
+- Web-based management tools and other tools to make maintenance simpler
+
+TDengine is specially designed and optimized for time-series data processing in IoT, connected cars, Industrial IoT, IT infrastructure and application monitoring, and other scenarios. Compared with other solutions, it is 10x faster on insert/query speed. With a single-core machine, over 20K requestes can be processed, millions data points can be ingested, and over 10 million data points can be retrieved in a second. Via column-based storage and tuned compression algorithm for different data types, less than 1/10 storage space is required. 
+
+## Explore More on TDengine
+
+Please read through the whole <a href='../documentation'>documentation</a> to learn more about TDengine.
+
diff --git a/documentation20/Model-ch.md b/documentation20/Model-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..dfb079d13d8ee7babd88b3a3e6905a4db0a6b447
--- /dev/null
+++ b/documentation20/Model-ch.md
@@ -0,0 +1,43 @@
+
+
+# 数据建模
+
+TDengine采用关系型数据模型，需要建库、建表。因此对于一个具体的应用场景，需要考虑库的设计，超级表和普通表的设计。本节不讨论细致的语法规则，只介绍概念。
+
+##创建库
+
+不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如：
+
+```cmd
+CREATE DATABASE power KEEP 365 DAYS 10 REPLICA 3 BLOCKS 4;
+```
+上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，副本数为3, 内存块数为4。详细的语法及参数请见TAOS SQL。
+
+注意：任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。
+
+## 创建超级表
+一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的设备创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表：
+```cmd
+CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);
+```
+与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义，为采集的物理量（示例中为ts, current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL一节。
+
+每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。一个系统可以有多个DB，一个DB里可以有一到多个超级表。
+
+## 创建表
+TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样，一张表有表名，Schema，但除此之外，还可以带有一到多个标签。创建时，需要使用超级表做模板，同时指定标签的具体值。以表一中的智能电表为例，可以使用如下的SQL命令建表：
+```cmd
+CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);
+```
+其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。
+
+TDengine建议将数据采集点的全局唯一ID作为表名。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。
+
+**自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如：
+
+```cmd
+INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);
+```
+上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。
+
+**多列模型**：TDengine支持多列模型，只要这些物理量是同时采集的，这些量就可以作为不同列放在同一张表里。有的数据采集点有多组采集量，每一组的数据采集时间是不一样的，这时需要对同一个采集点建多张表。但还有一种极限的设计，单列模型，无论是否同时采集，每个采集的物理量单独建表。TDengine建议，只要采集时间一致，就采用多列模型，因为插入效率以及存储效率更高。
\ No newline at end of file
diff --git a/documentation20/More on System Architecture-ch.md b/documentation20/More on System Architecture-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..8e5eeee1c5a6c96ddda1281110766a12a56b8d12
--- /dev/null
+++ b/documentation20/More on System Architecture-ch.md	
@@ -0,0 +1,248 @@
+# TDengine的技术设计
+
+## 存储设计
+
+TDengine的数据存储主要包含**元数据的存储**和**写入数据的存储**。以下章节详细介绍了TDengine各种数据的存储结构。
+
+### 元数据的存储
+
+TDengine中的元数据信息包括TDengine中的数据库，表，超级表等信息。元数据信息默认存放在 _/var/lib/taos/mgmt/_ 文件夹下。该文件夹的目录结构如下所示：
+```
+/var/lib/taos/
+      +--mgmt/
+          +--db.db
+          +--meters.db
+          +--user.db
+          +--vgroups.db
+```
+元数据在文件中按顺序排列。文件中的每条记录代表TDengine中的一个元数据机构（数据库、表等）。元数据文件只进行追加操作，即便是元数据的删除，也只是在数据文件中追加一条删除的记录。
+
+### 写入数据的存储
+
+TDengine中写入的数据在硬盘上是按时间维度进行分片的。同一个vnode中的表在同一时间范围内的数据都存放在同一文件组中，如下图中的v0f1804*文件。这一数据分片方式可以大大简化数据在时间维度的查询，提高查询速度。在默认配置下，硬盘上的每个文件存放10天数据。用户可根据需要调整数据库的 _daysPerFile_ 配置项进行配置。 数据在文件中是按块存储的。每个数据块只包含一张表的数据，且数据是按照时间主键递增排列的。数据在数据块中按列存储，这样使得同类型的数据存放在一起，可以大大提高压缩的比例，节省存储空间。TDengine对不同类型的数据采用了不同的压缩算法进行压缩，以达到最优的压缩结果。TDengine使用的压缩算法包括simple8B、delta-of-delta、RLE以及LZ4等。
+
+TDengine的数据文件默认存放在 */var/lib/taos/data/* 下。而 */var/lib/taos/tsdb/* 文件夹下存放了vnode的信息、vnode中表的信息以及数据文件的链接等。其完整目录结构如下所示：
+```
+/var/lib/taos/
+      +--tsdb/
+      |   +--vnode0
+      |        +--meterObj.v0
+      |        +--db/
+      |            +--v0f1804.head->/var/lib/taos/data/vnode0/v0f1804.head1
+      |            +--v0f1804.data->/var/lib/taos/data/vnode0/v0f1804.data
+      |            +--v0f1804.last->/var/lib/taos/data/vnode0/v0f1804.last1
+      |            +--v0f1805.head->/var/lib/taos/data/vnode0/v0f1805.head1
+      |            +--v0f1805.data->/var/lib/taos/data/vnode0/v0f1805.data
+      |            +--v0f1805.last->/var/lib/taos/data/vnode0/v0f1805.last1
+      |                   :
+      +--data/
+          +--vnode0/
+                +--v0f1804.head1
+                +--v0f1804.data
+                +--v0f1804.last1
+                +--v0f1805.head1
+                +--v0f1805.data
+                +--v0f1805.last1
+                        :
+```
+
+#### meterObj文件
+每个vnode中只存在一个 _meterObj_ 文件。该文件中存储了vnode的基本信息（创建时间，配置信息，vnode的统计信息等）以及该vnode中表的信息。其结构如下所示：
+```
+<文件开始>
+[文件头]
+[表记录1偏移量和长度]
+[表记录2偏移量和长度]
+...
+[表记录N偏移量和长度]
+[表记录1]
+[表记录2]
+...
+[表记录N]
+[表记录]
+<文件结尾>
+```
+其中，文件头大小为512字节，主要存放vnode的基本信息。每条表记录代表属于该vnode中的一张表在硬盘上的表示。
+
+#### head文件
+head文件中存放了其对应的data文件中数据块的索引信息。该文件组织形式如下：
+```
+<文件开始>
+[文件头]
+[表1偏移量]
+[表2偏移量]
+...
+[表N偏移量]
+[表1数据索引]
+[表2数据索引]
+...
+[表N数据索引]
+<文件结尾>
+```
+文件开头的偏移量列表表示对应表的数据索引块的开始位置在文件中的偏移量。每张表的数据索引信息在head文件中都是连续存放的。这也使得TDengine在读取单表数据时，可以将该表所有的数据块索引一次性读入内存，大大提高读取速度。表的数据索引块组织如下：
+```
+[索引块信息]
+[数据块1索引]
+[数据块2索引]
+...
+[数据块N索引]
+```
+其中，索引块信息中记录了数据块的个数等描述信息。每个数据块索引对应一个在data文件或last文件中的一个单独的数据块。索引信息中记录了数据块存放的文件、数据块起始位置的偏移量、数据块中数据时间主键的范围等。索引块中的数据块索引是按照时间范围顺序排放的，这也就是说，索引块M对应的数据块中的数据时间范围都大于索引块M-1的。这种预先排序的存储方式使得在TDengine在进行按照时间戳进行查询时可以使用折半查找算法，大大提高查询速度。
+
+#### data文件
+data文件中存放了真实的数据块。该文件只进行追加操作。其文件组织形式如下：
+```
+<文件开始>
+[文件头]
+[数据块1]
+[数据块2]
+...
+[数据块N]
+<文件结尾>
+```
+每个数据块只属于vnode中的一张表，且数据块中的数据按照时间主键排列。数据块中的数据按列组织排放，使得同一类型的数据排放在一起，方便压缩和读取。每个数据块的组织形式如下所示：
+```
+[列1信息]
+[列2信息]
+...
+[列N信息]
+[列1数据]
+[列2数据]
+...
+[列N数据]
+```
+列信息中包含该列的类型，列的压缩算法，列数据在文件中的偏移量以及长度等。除此之外，列信息中也包含该内存块中该列数据的预计算结果，从而在过滤查询时根据预计算结果判定是否读取数据块，大大提高读取速度。
+
+#### last文件
+为了防止数据块的碎片化，提高查询速度和压缩率，TDengine引入了last文件。当要落盘的数据块中的数据条数低于某个阈值时，TDengine会先将该数据块写入到last文件中进行暂时存储。当有新的数据需要落盘时，last文件中的数据会被读取出来与新数据组成新的数据块写入到data文件中。last文件的组织形式与data文件类似。
+
+### TDengine数据存储小结
+TDengine通过其创新的架构和存储结构设计，有效提高了计算机资源的使用率。一方面，TDengine的虚拟化使得TDengine的水平扩展及备份非常容易。另一方面，TDengine将表中数据按时间主键排序存储且其列式存储的组织形式都使TDengine在写入、查询以及压缩方面拥有非常大的优势。
+
+
+## 查询处理
+
+### 概述
+
+TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、管理节点、数据节点协同完成。 各组件包含的与查询处理相关的功能和模块如下：
+
+客户端（Client App）。客户端包含TAOS SQL的解析（SQL Parser）和查询请求执行器（Query Executor），第二阶段聚合器（Result Merger），连续查询管理器（Continuous Query Manager）等主要功能模块构成。SQL解析器负责对SQL语句进行解析校验，并转化为抽象语法树，查询执行器负责将抽象语法树转化查询执行逻辑，并根据SQL语句查询条件，将其转换为针对管理节点元数据查询和针对数据节点的数据查询两级查询处理。由于TAOS SQL当前不提供复杂的嵌套查询和pipeline查询处理机制，所以不再需要查询计划优化、逻辑查询计划到物理查询计划转换等过程。第二阶段聚合器负责将各数据节点查询返回的独立结果进行二阶段聚合生成最后的结果。连续查询管理器则负责针对用户建立的连续查询进行管理，负责定时拉起查询请求并按需将结果写回TDengine或返回给客户应用。此外，客户端还负责查询失败后重试、取消查询请求、以及维持连接心跳、向管理节点上报查询状态等工作。
+
+管理节点（Management Node）。管理节点保存了整个集群系统的全部数据的元数据信息，向客户端节点提供查询所需的数据的元数据，并根据集群的负载情况切分查询请求。通过超级表包含了通过该超级表创建的所有表的信息，因此查询处理器（Query Executor）负责针对标签（TAG）的查询处理，并将满足标签查询请求的表信息返回给客户端。此外，管理节点还维护集群的查询状态（Query Status Manager）维护，查询状态管理中在内存中临时保存有当前正在执行的全部查询，当客户端使用 *show queries* 命令的时候，将当前系统正在运行的查询信息返回客户端。
+
+数据节点（Data Node）。数据节点保存了数据库中全部数据内容，并通过查询执行器、查询处理调度器、查询任务队列（Query Task Queue）进行查询处理的调度执行，从客户端接收到的查询处理请求都统一放置到处理队列中，查询执行器从队列中获得查询请求，并负责执行。通过查询优化器（Query Optimizer）对于查询进行基本的优化处理，以及通过数据节点的查询执行器（Query Executor）扫描符合条件的数据单元并返回计算结果。等接收客户端发出的查询请求，执行查询处理，并将结果返回。同时数据节点还需要响应来自管理节点的管理信息和命令，例如 *kill query* 命令以后，需要即刻停止执行的查询任务。
+
+<center> <img src="../assets/fig1.png"> </center>
+<center>图 1. 系统查询处理架构图(只包含查询相关组件)</center>
+
+### 普通查询处理
+
+客户端、管理节点、数据节点协同完成TDengine的查询处理全流程。我们以一个具体的SQL查询为例，说明TDengine的查询处理流程。SQL语句向超级表*FOO_SUPER_TABLE*查询获取时间范围在2019年1月12日整天，标签TAG_LOC是'beijing'的表所包含的所有记录总数，SQL语句如下：
+
+```sql
+SELECT COUNT(*) 
+FROM FOO_SUPER_TABLE
+WHERE TAG_LOC = 'beijing' AND TS >= '2019-01-12 00:00:00' AND TS < '2019-01-13 00:00:00'
+```
+
+首先，客户端调用TAOS SQL解析器对SQL语句进行解析及合法性检查，然后生成语法树，并从中提取查询的对象 — 超级表 *FOO_SUPER_TABLE* ，然后解析器向管理节点（Management Node）请求其相应的元数据信息，并将过滤信息（TAG_LOC='beijing'）同时发送到管理节点。
+
+管理节点接收元数据获取的请求，首先找到超级表 *FOO_SUPER_TABLE* 基础信息，然后应用查询条件来过滤通过该超级表创建的全部表，最后满足查询条件（TAG_LOC='beijing'），即 *TAG_LOC* 标签列是 'beijing' 的的通过其查询执行器将满足查询要求的对象（表或超级表）的元数据信息返回给客户端。
+
+客户端获得了 *FOO_SUPER_TABLE* 的元数据信息后，查询执行器根据元数据中的数据分布，分别向保存有相应数据的节点发起查询请求，此时时间戳范围过滤条件（TS >= '2019-01-12 00:00:00' AND TS < '2019-01-13 00:00:00'）需要同时发送给全部的数据节点。
+
+数据节点接收到发自客户端的查询，转化为内部结构并进行优化以后将其放入任务执行队列，等待查询执行器执行。当查询结果获得以后，将查询结果返回客户端。数据节点执行查询的过程均相互独立，完全只依赖于自身的数据和内容进行计算。
+
+当所有查询涉及的数据节点返回结果后，客户端将每个数据节点查询的结果集再次进行聚合（针对本案例，即将所有结果再次进行累加），累加的结果即为最后的查询结果。第二阶段聚合并不是所有的查询都需要。例如，针对数据的列选取操作，实际上是不需要第二阶段聚合。
+
+### REST查询处理
+
+在 C/C++ 、Python接口、 JDBC 接口之外，TDengine 还提供基于 HTTP 协议的 REST 接口。不同于使用应用客户端开发程序进行的开发。当用户使用 REST 接口的时候，所有的查询处理过程都是在服务器端来完成，用户的应用服务不会参与数据库的计算过程，查询处理完成后结果通过 HTTP的 JSON 格式返回给用户。
+
+<center> <img src="../assets/fig2.png"> </center>
+<center>图 2.  REST查询架构</center>
+
+当用户使用基于HTTP的REST查询接口，HTTP的请求首先与位于数据节点的HTTP连接器（ Connector），建立连接，然后通过REST的签名机制，使用Token来确保请求的可靠性。对于数据节点，HTTP连接器接收到请求后，调用内嵌的客户端程序发起查询请求，内嵌客户端将解析通过HTTP连接器传递过来的SQL语句，解析该SQL语句并按需向管理节点请求元数据信息，然后向本机或集群中其他节点发送查询请求，最后按需聚合计算结果。HTTP连接器接收到请求SQL以后，后续的流程处理与采用应用客户端方式的查询处理完全一致。最后，还需要将查询的结果转换为JSON格式字符串，并通过HTTP 响应返回给客户端。
+
+可以看到，在处理HTTP流程的整个过程中，用户应用不再参与到查询处理的过程中，只负责通过HTTP协议发送SQL请求并接收JSON格式的结果。同时还需要注意的是，每个数据节点均内嵌了一个HTTP连接器和客户端程序，因此请求集群中任何一个数据节点，该数据节点均能够通过HTTP协议返回用户的查询结果。
+
+### 技术特征
+
+由于TDengine采用数据和标签分离存储的模式，能够极大地降低标签数据存储的冗余度。标签数据直接关联到每个表，并采用全内存的结构进行管理和维护标签数据,全内存的结构提供快速的查询处理，千万级别规模的标签数据查询可以在毫秒级别返回。首先针对标签数据的过滤可以有效地降低第二阶段的查询涉及的数据规模。为有效地提升查询处理的性能，针对物联网数据的不可更改的特点，TDengine采用在每个保存的数据块上，都记录下该数据块中数据的最大值、最小值、和等统计数据。如果查询处理涉及整个数据块的全部数据，则直接使用预计算结果，不再读取数据块的内容。由于预计算模块的大小远小于磁盘上存储的具体数据的大小，对于磁盘IO为瓶颈的查询处理，使用预计算结果可以极大地减小读取IO，并加速查询处理的流程。
+
+由于TDengine采用按列存储数据。当从磁盘中读取数据块进行计算的时候，按照查询列信息读取该列数据，并不需要读取其他不相关的数据，可以最小化读取数据。此外，由于采用列存储结构，数据节点针对数据的扫描采用该列数据块进行，可以充分利用CPU L2高速缓存，极大地加速数据扫描的速度。此外，对于某些查询，并不会等全部查询结果生成后再返回结果。例如，列选取查询，当第一批查询结果获得以后，数据节点直接将其返回客户端。同时，在查询处理过程中，系统在数据节点接收到查询请求以后马上返回客户端查询确认信息，并同时拉起查询处理过程，并等待查询执行完成后才返回给用户查询有响应。
+
+## TDengine集群设计
+
+### 1：集群与主要逻辑单元
+
+TDengine是基于硬件、软件系统不可靠、一定会有故障的假设进行设计的，是基于任何单台计算机都无足够能力处理海量数据的假设进行设计的。因此TDengine从研发的第一天起，就按照分布式高可靠架构进行设计，是完全去中心化的，是水平扩展的，这样任何单台或多台服务器宕机或软件错误都不影响系统的服务。通过节点虚拟化并辅以自动化负载均衡技术，TDengine能最大限度地利用异构集群中的计算和存储资源。而且只要数据副本数大于一，无论是硬软件的升级、还是IDC的迁移等都无需停止集群的服务，极大地保证系统的正常运行，并且降低了系统管理员和运维人员的工作量。
+
+下面的示例图上有八个物理节点，每个物理节点被逻辑的划分为多个虚拟节点。下面对系统的基本概念进行介绍。 
+
+
+
+![assets/nodes.png](../assets/nodes.png)
+
+**物理节点(dnode)**：集群中的一物理服务器或云平台上的一虚拟机。为安全以及通讯效率，一个物理节点可配置两张网卡，或两个IP地址。其中一张网卡用于集群内部通讯，其IP地址为**privateIp**, 另外一张网卡用于与集群外部应用的通讯，其IP地址为**publicIp**。在一些云平台（如阿里云），对外的IP地址是映射过来的，因此publicIp还有一个对应的内部IP地址**internalIp**(与privateIp不同)。对于只有一个IP地址的物理节点，publicIp, privateIp以及internalIp都是同一个地址，没有任何区别。一个dnode上有而且只有一个taosd实例运行。 
+
+**虚拟数据节点(vnode)**：在物理节点之上的可独立运行的基础逻辑单元，时序数据写入、存储、查询等操作逻辑都在虚拟节点中进行(图中V)，采集的时序数据就存储在vnode上。一个vnode包含固定数量的表。当创建一张新表时，系统会检查是否需要创建新的vnode。一个物理节点上能创建的vnode的数量取决于物理节点的硬件资源。一个vnode只属于一个DB，但一个DB可以有多个vnode。
+
+**虚拟数据节点组(vgroup)**: 位于不同物理节点的vnode可以组成一个虚拟数据节点组vnode group(如上图dnode0中的V0, dnode1中的V1, dnode6中的V2属于同一个虚拟节点组)。归属于同一个vgroup的虚拟节点采取master/slave的方式进行管理。写只能在master上进行，但采用asynchronous的方式将数据同步到slave，这样确保了一份数据在多个物理节点上有拷贝。如果master节点宕机，其他节点监测到后，将重新选举vgroup里的master, 新的master能继续处理数据请求，从而保证系统运行的可靠性。一个vgroup里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个物理节点。副本数在创建DB时通过参数replica可以指定，缺省为1。使用TDengine, 数据的安全依靠多副本解决，因此不再需要昂贵的磁盘阵列等存储设备。
+
+**虚拟管理节点(mnode)**：负责所有节点运行状态的监控和维护，以及节点之间的负载均衡（图中M）。同时，虚拟管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为Meta Node。TDengine集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个管理节点集群(图中M0, M1, M2)。mnode间采用master/slave的机制进行管理，而且采取强一致方式进行数据同步。mnode集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，而且每个dnode都知道整个集群中所有mnode的IP地址。 
+
+**taosc**：一个软件模块，是TDengine给应用提供的驱动程序（driver）,内嵌于JDBC、ODBC driver中，或者C语言连接库里。应用都是通过taosc而不是直接来与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的虚拟节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, ODBC, C/C++接口而言，这个模块是在应用所处的计算机上运行，但消耗的资源很小。为支持全分布式的REST接口，taosc在TDengine集群的每个dnode上都有一运行实例。 
+
+**对外服务地址**：TDengine集群可以容纳单台、多台甚至几千台物理节点。应用只需要向集群中任何一个物理节点的publicIp发起连接即可。启动CLI应用taos时，选项-h需要提供的就是publicIp。
+
+**master/secondIp**：每一个dnode都需要配置一个masterIp。dnode启动后，将对配置的masterIp发起加入集群的连接请求。masterIp是已经创建的集群中的任何一个节点的privateIp，对于集群中的第一个节点，就是它自己的privateIp。为保证连接成功，每个dnode还可配置secondIp, 该IP地址也是已创建的集群中的任何一个节点的privateIp。如果一个节点连接masterIp失败,它将试图链接secondIp。
+
+dnode启动后，会获知集群的mnode IP列表，并且定时向mnode发送状态信息。
+
+vnode与mnode只是逻辑上的划分，都是执行程序taosd里的不同线程而已，无需安装不同的软件，做任何特殊的配置。最小的系统配置就是一个物理节点,vnode,mnode和taosc都存在而且都正常运行，但单一节点无法保证系统的高可靠。 
+
+### 2：一典型的操作流程
+
+为解释vnode, mnode, taosc和应用之间的关系以及各自扮演的角色，下面对写入数据这个典型操作的流程进行剖析。
+
+ 
+
+![Picture1](../assets/Picture2.png)
+
+ 
+
+1. 应用通过JDBC、ODBC或其他API接口发起插入数据的请求。
+2. taosc会检查缓存，看是有保存有该表的meta data。如果有，直接到第4步。如果没有，taosc将向mnode发出get meta-data请求。
+3. mnode将该表的meta-data返回给taosc。Meta-data包含有该表的schema, 而且还有该表所属的vgroup信息（vnode ID以及所在的dnode的IP地址，如果副本数为N，就有N组vnodeID/IP)。如果taosc迟迟得不到mnode回应，而且存在多个mnode,taosc将向下一个mnode发出请求。
+4. taosc向master vnode发起插入请求。
+5. vnode插入数据后，给taosc一个应答，表示插入成功。如果taosc迟迟得不到vnode的回应，taosc会认为该节点已经离线。这种情况下，如果被插入的数据库有多个副本，taosc将向vgroup里下一个vnode发出插入请求。
+6. taosc通知APP，写入成功。 
+
+对于第二和第三步，taosc启动时，并不知道mnode的IP地址，因此会直接向配置的集群对外服务的IP地址发起请求。如果接收到该请求的dnode并没有配置mnode，该dnode会在回复的消息中告知mnode的IP地址列表（如果有多个dnodes，mnode的IP地址可以有多个），这样taosc会重新向新的mnode的IP地址发出获取meta-data的请求。
+
+对于第四和第五步，没有缓存的情况下，taosc无法知道虚拟节点组里谁是master，就假设第一个vnodeID/IP就是master,向它发出请求。如果接收到请求的vnode并不是master,它会在回复中告知谁是master，这样taosc就向建议的master vnode发出请求。一旦得到插入成功的回复，taosc会缓存住master节点的信息。
+
+上述是插入数据的流程，查询、计算的流程也完全一致。taosc把这些复杂的流程全部封装屏蔽了，因此应用无需处理重定向、获取meta data等细节，完全是透明的。
+
+通过taosc缓存机制，只有在第一次对一张表操作时，才需要访问mnode, 因此mnode不会成为系统瓶颈。但因为schema有可能变化，而且vgroup有可能发生改变（比如负载均衡发生），因此taosc需要定时自动刷新缓存。
+
+### 3：数据分区
+
+vnode(虚拟数据节点)保存采集的时序数据，而且查询、计算都在这些节点上进行。为便于负载均衡、数据恢复、支持异构环境，TDengine将一个物理节点根据其计算和存储资源切分为多个vnode。这些vnode的管理是TDengine自动完成的，对应用完全透明。 
+
+对于单独一个数据采集点，无论其数据量多大，一个vnode（或vnode group, 如果副本数大于1）有足够的计算资源和存储资源来处理（如果每秒生成一条16字节的记录，一年产生的原始数据不到0.5G），因此TDengine将一张表的所有数据都存放在一个vnode里，而不会让同一个采集点的数据分布到两个或多个dnode上。而且一个vnode可存储多张表的数据，一个vnode可容纳的表的数目由配置参数tables指定，缺省为2000。设计上，一个vnode里所有的表都属于同一个DB。因此一个数据库DB需要的vnode或vgroup的个数等于：数据库表的数目/tables。 
+
+创建DB时，系统并不会马上分配资源。但当创建一张表时，系统将看是否有已经分配的vnode, 而且是否有空位，如果有，立即在该有空位的vnode创建表。如果没有，系统将从集群中，根据当前的负载情况，在一个dnode上创建一新的vnode, 然后创建表。如果DB有多个副本，系统不是只创建一个vnode，而是一个vgroup(虚拟数据节点组)。系统对vnode的数目没有任何限制，仅仅受限于物理节点本身的计算和存储资源。
+
+参数tables的设置需要考虑具体场景，创建DB时，可以个性化指定该参数。该参数不宜过大，也不宜过小。过小，极端情况，就是每个数据采集点一个vnode, 这样导致系统数据文件过多。过大，虚拟化带来的优势就会丧失。给定集群计算资源的情况下，整个系统vnode的个数应该是CPU核的数目的两倍以上。
+
+### 4：负载均衡
+
+每个dnode(物理节点)都定时向 mnode(虚拟管理节点)报告其状态（包括硬盘空间、内存大小、CPU、网络、虚拟节点个数等），因此mnode了解整个集群的状态。基于整体状态，当mnode发现某个dnode负载过重，它会将dnode上的一个或多个vnode挪到其他dnode。在挪动过程中，对外服务继续进行，数据插入、查询和计算操作都不受影响。负载均衡操作结束后，应用也无需重启，将自动连接新的vnode。 
+
+如果mnode一段时间没有收到dnode的状态报告，mnode会认为这个dnode已经离线。如果离线时间超过一定时长（时长由配置参数offlineThreshold决定），该dnode将被mnode强制剔除出集群。该dnode上的vnodes如果副本数大于一，系统将自动在其他dnode上创建新的副本，以保证数据的副本数。
+
+
+
+**Note：**目前集群功能仅仅限于企业版
\ No newline at end of file
diff --git a/documentation20/More on System Architecture.md b/documentation20/More on System Architecture.md
new file mode 100644
index 0000000000000000000000000000000000000000..d7a38b99a3ae5a630509f3ef0f0ffdc97d3aaaf1
--- /dev/null
+++ b/documentation20/More on System Architecture.md	
@@ -0,0 +1,176 @@
+# TDengine System Architecture 
+
+## Storage Design
+
+TDengine data mainly include **metadata** and **data** that we will introduce in the following sections.
+
+### Metadata Storage
+
+Metadata include the information of databases, tables, etc. Metadata files are saved in _/var/lib/taos/mgmt/_ directory by default. The directory tree is as below:
+```
+/var/lib/taos/
+      +--mgmt/
+          +--db.db
+          +--meters.db
+          +--user.db
+          +--vgroups.db
+```
+
+A metadata structure (database, table, etc.) is saved as a record in a metadata file. All metadata files are appended only, and even a drop operation adds a deletion record at the end of the file.
+
+### Data storage
+
+Data in TDengine are sharded according to the time range. Data of tables in the same vnode in a certain time range are saved in the same filegroup, such as files v0f1804*. This sharding strategy can effectively improve data searching speed. By default, a group of files contains data in 10 days, which can be configured by *daysPerFile* in the configuration file or by *DAYS* keyword in *CREATE DATABASE* clause. Data in files are blockwised. A data block only contains one table's data. Records in the same data block are sorted according to the primary timestamp, which helps to improve the compression rate and save storage. The compression algorithms used in TDengine include simple8B, delta-of-delta, RLE, LZ4, etc.
+
+By default, TDengine data are saved in */var/lib/taos/data/* directory. _/var/lib/taos/tsdb/_ directory contains vnode informations and data file linkes.
+
+```
+/var/lib/taos/
+      +--tsdb/
+      |   +--vnode0
+      |        +--meterObj.v0
+      |        +--db/
+      |            +--v0f1804.head->/var/lib/taos/data/vnode0/v0f1804.head1
+      |            +--v0f1804.data->/var/lib/taos/data/vnode0/v0f1804.data
+      |            +--v0f1804.last->/var/lib/taos/data/vnode0/v0f1804.last1
+      |            +--v0f1805.head->/var/lib/taos/data/vnode0/v0f1805.head1
+      |            +--v0f1805.data->/var/lib/taos/data/vnode0/v0f1805.data
+      |            +--v0f1805.last->/var/lib/taos/data/vnode0/v0f1805.last1
+      |                   :
+      +--data/
+          +--vnode0/
+                +--v0f1804.head1
+                +--v0f1804.data
+                +--v0f1804.last1
+                +--v0f1805.head1
+                +--v0f1805.data
+                +--v0f1805.last1
+                        :
+```
+
+#### meterObj file
+There are only one meterObj file in a vnode. Informations bout the vnode, such as created time, configuration information, vnode statistic informations are saved in this file. It has the structure like below:
+
+```
+<start_of_file>
+[file_header]
+[table_record1_offset&length]
+[table_record2_offset&length]
+...
+[table_recordN_offset&length]
+[table_record1]
+[table_record2]
+...
+[table_recordN]
+<end_of_file>
+```
+The file header takes 512 bytes, which mainly contains informations about the vnode. Each table record is the representation of a table on disk.
+
+#### head file
+The _head_ files contain the index of data blocks in the _data_ file. The inner organization is as below:
+```
+<start_of_file>
+[file_header]
+[table1_offset]
+[table2_offset]
+...
+[tableN_offset]
+[table1_index_block]
+[table2_index_block]
+...
+[tableN_index_block]
+<end_of_file>
+```
+The table offset array in the _head_ file saves the information about the offsets of each table index block. Indices on data blocks in the same table are saved continuously. This also makes it efficient to load data indices on the same table. The data index block has a structure like:
+
+```
+[index_block_info]
+[block1_index]
+[block2_index]
+...
+[blockN_index]
+```
+The index block info part contains the information about the index block such as the number of index blocks, etc. Each block index corresponds to a real data block in the _data_ file or _last_ file. Information about the location of the real data block, the primary timestamp range of the data block, etc. are all saved in the block index part. The block indices are sorted in ascending order according to the primary timestamp. So we can apply algorithms such as the binary search on the data to efficiently search blocks according to time.
+
+#### data file
+The _data_ files store the real data block. They are append-only. The organization is as:
+```
+<start_of_file>
+[file_header]
+[block1]
+[block2]
+...
+[blockN]
+<end_of_file>
+```
+A data block in _data_ files only belongs to a table in the vnode and the records in a data block are sorted in ascending order according to the primary timestamp key. Data blocks are column-oriented. Data in the same column are stored contiguously, which improves reading speed and compression rate because of their similarity. A data block has the following organization:
+
+```
+[column1_info]
+[column2_info]
+...
+[columnN_info]
+[column1_data]
+[column2_data]
+...
+[columnN_data]
+```
+The column info part includes information about column types, column compression algorithm, column data offset and length in the _data_ file, etc. Besides, pre-calculated results of the column data in the block are also in the column info part, which helps to improve reading speed by avoiding loading data block necessarily.
+
+#### last file
+To avoid storage fragment and to import query speed and compression rate, TDengine introduces an extra file, the _last_ file. When the number of records in a data block is lower than a threshold, TDengine will flush the block to the _last_ file for temporary storage. When new data comes, the data in the _last_ file will be merged with the new data and form a larger data block and written to the _data_ file. The organization of the _last_ file is similar to the _data_ file.
+
+### Summary
+The innovation in architecture and storage design of TDengine improves resource usage. On the one hand, the virtualization makes it easy to distribute resources between different vnodes and for future scaling. On the other hand, sorted and column-oriented storage makes TDengine have a great advantage in writing, querying and compression.
+
+## Query Design
+
+#### Introduction
+
+TDengine provides a variety of query functions for both tables and super tables. In addition to regular aggregate queries, it also provides time window based query and statistical aggregation for time series data. TDengine's query processing requires the client app, management node, and data node to work together. The functions and modules involved in query processing included in each component are as follows:
+
+Client (Client App). The client development kit, embed in a client application, consists of TAOS SQL parser and query executor, the second-stage aggregator (Result Merger), continuous query manager and other major functional modules. The SQL parser is responsible for parsing and verifying the SQL statement and converting it into an abstract syntax tree. The query executor is responsible for transforming the abstract syntax tree into the query execution logic and creates the metadata query according to the query condition of the SQL statement. Since TAOS SQL does not currently include complex nested queries and pipeline query processing mechanism, there is no longer need for query plan optimization and physical query plan conversions. The second-stage aggregator is responsible for performing the aggregation of the independent results returned by query involved data nodes at the client side to generate final results. The continuous query manager is dedicated to managing the continuous queries created by users, including issuing fixed-interval query requests and writing the results back to TDengine or returning to the client application as needed. Also, the client is also responsible for retrying after the query fails, canceling the query request, and maintaining the connection heartbeat and reporting the query status to the management node.
+
+Management Node. The management node keeps the metadata of all the data of the entire cluster system, provides the metadata of the data required for the query from the client node, and divides the query request according to the load condition of the cluster. The super table contains information about all the tables created according to the super table, so the query processor (Query Executor) of the management node is responsible for the query processing of the tags of tables and returns the table information satisfying the tag query. Besides, the management node maintains the query status of the cluster in the Query Status Manager component, in which the metadata of all queries that are currently executing are temporarily stored in-memory buffer. When the client issues *show queries* command to management node, current running queries information is returned to the client.
+
+Data Node. The data node, responsible for storing all data of the database, consists of query executor, query processing scheduler, query task queue, and other related components. Once the query requests from the client received, they are put into query task queue and waiting to be processed by query executor. The query executor extracts the query request from the query task queue and invokes the query optimizer to perform the basic optimization for the query execution plan. And then query executor scans the qualified data blocks in both cache and disk to obtain qualified data and return the calculated results. Besides, the data node also needs to respond to management information and commands from the management node. For example, after the *kill query* received from the management node, the query task needs to be stopped immediately.
+
+<center> <img src="../assets/fig1.png"> </center>
+<center>Fig 1. System query processing architecture diagram (only query related components)</center>
+
+#### Query Process Design
+
+The client, the management node, and the data node cooperate to complete the entire query processing of TDengine. Let's take a concrete SQL query as an example to illustrate the whole query processing flow. The SQL statement is to query on super table *FOO_SUPER_TABLE* to get the total number of records generated on January 12, 2019, from the table, of which TAG_LOC equals to 'beijing'.  The SQL statement is as follows:
+
+```sql
+SELECT COUNT(*) 
+FROM FOO_SUPER_TABLE
+WHERE TAG_LOC = 'beijing' AND TS >= '2019-01-12 00:00:00' AND TS < '2019-01-13 00:00:00'
+```
+
+First, the client invokes the TAOS SQL parser to parse and validate the SQL statement, then generates a syntax tree, and extracts the object of the query - the super table *FOO_SUPER_TABLE*, and then the parser sends requests with filtering information (TAG_LOC='beijing') to management node to get the corresponding metadata about *FOO_SUPER_TABLE*.
+
+Once the management node receives the request for metadata acquisition, first finds the super table *FOO_SUPER_TABLE* basic information, and then applies the query condition (TAG_LOC='beijing') to filter all the related tables created according to it. And finally, the query executor returns the metadata information that satisfies the query request to the client.
+
+After the client obtains the metadata information of *FOO_SUPER_TABLE*, the query executor initiates a query request with timestamp range filtering condition (TS >= '2019- 01-12 00:00:00' AND TS < '2019-01-13 00:00:00') to all nodes that hold the corresponding data according to the information about data distribution in metadata.
+
+The data node receives the query sent from the client, converts it into an internal structure and puts it into the query task queue to be executed by query executor after optimizing the execution plan. When the query result is obtained, the query result is returned to the client. It should be noted that the data nodes perform the query process independently of each other, and rely solely on their data and content for processing.
+
+When all data nodes involved in the query return results, the client aggregates the result sets from each data node. In this case, all results are accumulated to generate the final query result. The second stage of aggregation is not always required for all queries. For example, a column selection query does not require a second-stage aggregation at all.
+
+#### REST Query Process
+
+In addition to C/C++, Python, and JDBC interface, TDengine also provides a REST interface based on the HTTP protocol, which is different from using the client application programming interface. When the user uses the REST interface, all the query processing is completed on the server-side, and the user's application is not involved in query processing anymore. After the query processing is completed, the result is returned to the client through the HTTP JSON string.
+
+<center> <img src="../assets/fig2.png"> </center>
+<center>Fig. 2 REST query architecture</center>
+
+When a client uses an HTTP-based REST query interface, the client first establishes a connection with the HTTP connector at the data node and then uses the token to ensure the reliability of the request through the REST signature mechanism. For the data node, after receiving the request, the HTTP connector invokes the embedded client program to initiate a query processing, and then the embedded client parses the SQL statement from the HTTP connector and requests the management node to get metadata as needed. After that, the embedded client sends query requests to the same data node or other nodes in the cluster and aggregates the calculation results on demand. Finally, you also need to convert the result of the query into a JSON format string and return it to the client via an HTTP response. After the HTTP connector receives the request SQL, the subsequent process processing is completely consistent with the query processing using the client application development kit.
+
+It should be noted that during the entire processing, the client application is no longer involved in, and is only responsible for sending SQL requests through the HTTP protocol and receiving the results in JSON format. Besides, each data node is embedded with an HTTP connector and a client, so any data node in the cluster received requests from a client, the data node can initiate the query and return the result to the client through the HTTP protocol, with transfer the request to other data nodes.
+
+#### Technology
+
+Because TDengine stores data and tags value separately, the tag value is kept in the management node and directly associated with each table instead of records, resulting in a great reduction of the data storage. Therefore, the tag value can be managed by a fully in-memory structure. First, the filtering of the tag data can drastically reduce the data size involved in the second phase of the query. The query processing for the data is performed at the data node. TDengine takes advantage of the immutable characteristics of IoT data by calculating the maximum, minimum, and other statistics of the data in one data block on each saved data block, to effectively improve the performance of query processing. If the query process involves all the data of the entire data block, the pre-computed result is used directly, and the content of the data block is no longer needed. Since the size of disk space required to store the pre-computation result is much smaller than the size of the specific data, the pre-computation result can greatly reduce the disk IO and speed up the query processing.
+
+TDengine employs column-oriented data storage techniques. When the data block is involved to be loaded from the disk for calculation, only the required column is read according to the query condition, and the read overhead can be minimized. The data of one column is stored in a contiguous memory block and therefore can make full use of the CPU L2 cache to greatly speed up the data scanning. Besides, TDengine utilizes the eagerly responding mechanism and returns a partial result before the complete result is acquired. For example, when the first batch of results is obtained, the data node immediately returns it directly to the client in case of a column select query. 
\ No newline at end of file
diff --git a/documentation20/Queries-ch.md b/documentation20/Queries-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..0f4124f1fca71a58c21b4c5f753d074a75d6e0c5
--- /dev/null
+++ b/documentation20/Queries-ch.md
@@ -0,0 +1,86 @@
+
+
+
+
+# 高效查询数据
+
+## 主要查询功能
+
+TDengine采用SQL作为查询语言，应用程序可以通过C/C++, JDBC, GO, Python连接器发送SQL查询语句，用户还可以通过TAOS Shell直接手动执行SQL即席查询，十分方便。支持如下查询功能：
+
+- 查询单列、或多列查询
+- 支持值过滤条件：\>, \<,  =, \<> 大于，小于，等于，不等于等等
+- 支持对标签的模糊匹配
+- 支持Group by, Order by, Limit, Offset
+- 支持列之间的四则运算
+- 支持时间戳对齐的JOIN操作
+- 支持多种函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff
+
+例如：在TAOS Shell中，从表d1001中查询出vlotage >215的记录，按时间降序排列，仅仅输出2条。
+```mysql
+taos> select * from d1001 where voltage > 215 order by ts desc limit 2;
+           ts            |       current        |   voltage   |        phase         |
+======================================================================================
+ 2018-10-03 14:38:16.800 |             12.30000 |         221 |              0.31000 |
+ 2018-10-03 14:38:15.000 |             12.60000 |         218 |              0.33000 |
+Query OK, 2 row(s) in set (0.001100s)
+```
+为满足物联网场景的需求，TDengine支持几个特殊的函数，比如twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。TDengine还支持连续查询。
+
+具体的查询语法请看TAOS SQL。
+
+## 多表聚合查询
+
+TDengine对每个数据采集点单独建表，但应用经常需要对数据点之间进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是表的集合，包含多张表。这集合里每张表的Schema是一样的，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。
+
+应用可通过指定标签的过滤条件，对一个STable下的全部或部分表进行聚合或统计操作，这样大大简化应用的开发。其具体流程如下图所示：
+
+<center> <img src="../assets/stable.png"> </center>
+
+<center> 多表聚合查询原理图  </center>
+
+1：应用将一个查询条件发往系统；2: taosc将超级表的名字发往Meta Node（管理节点)；3：管理节点将超级表所拥有的vnode列表发回taosc；4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点；5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc；6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。
+
+由于TDengine在vnode内将标签数据与时序数据分离存储，通过先在内存里过滤标签数据，将需要扫描的数据集大幅减少，大幅提升了聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。
+
+对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看TAOS SQL。
+
+比如：在TAOS Shell，查找所有智能电表采集的电压平均值，并按照location分组
+
+```mysql
+taos> select avg(voltage) from meters group by location;
+       avg(voltage)        |            location            |
+=============================================================
+             222.000000000 | Beijing.Haidian                |
+             219.200000000 | Beijing.Chaoyang               |
+Query OK, 2 row(s) in set (0.002136s)
+```
+
+## 降采样查询、插值
+
+物联网场景里，经常需要做down sampling，需要将采集的数据按时间段进行聚合。TDengine提供了一个简便的关键词interval让操作变得极为简单。比如：将智能电表d1001采集的电流值每10秒钟求和
+```mysql
+taos> SELECT sum(current) FROM d1001 interval(10s) ;
+           ts            |       sum(current)        |
+======================================================
+ 2018-10-03 14:38:00.000 |              10.300000191 |
+ 2018-10-03 14:38:10.000 |              24.900000572 |
+Query OK, 2 row(s) in set (0.000883s)
+```
+降采样操作还适用于超级表，比如：将所有智能电表采集的电流值每秒钟求和
+```mysql
+taos> SELECT sum(current) FROM meters interval(1s) ;
+           ts            |       sum(current)        |
+======================================================
+ 2018-10-03 14:38:04.000 |              10.199999809 |
+ 2018-10-03 14:38:05.000 |              32.900000572 |
+ 2018-10-03 14:38:06.000 |              11.500000000 |
+ 2018-10-03 14:38:15.000 |              12.600000381 |
+ 2018-10-03 14:38:16.000 |              36.000000000 |
+Query OK, 5 row(s) in set (0.001538s)
+```
+
+物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里，没有采集的数据，TDengine还提供插值计算的功能。
+
+语法规则细节请见TAOS SQL。
+
diff --git a/documentation20/Super Table-ch.md b/documentation20/Super Table-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..14145cbb70aa421b6c1d3340ce8139d8aa4b642c
--- /dev/null
+++ b/documentation20/Super Table-ch.md	
@@ -0,0 +1,224 @@
+# 超级表STable：多表聚合
+
+TDengine要求每个数据采集点单独建表，这样能极大提高数据的插入/查询性能，但是导致系统中表的数量猛增，让应用对表的维护以及聚合、统计操作难度加大。为降低应用的开发难度，TDengine引入了超级表STable (Super Table)的概念。
+
+## 什么是超级表
+
+STable是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。
+
+TDengine扩展标准SQL语法用于定义STable，使用关键词tags指定标签信息。语法如下：
+
+```mysql
+CREATE TABLE <stable_name> (<field_name> TIMESTAMP, field_name1 field_type,…)   TAGS(tag_name tag_type, …) 
+```
+
+其中tag_name是标签名，tag_type是标签的数据类型。标签可以使用时间戳之外的其他TDengine支持的数据类型，标签的个数最多为6个，名字不能与系统关键词相同，也不能与其他列名相同。如：
+
+```mysql
+create table thermometer (ts timestamp, degree float) 
+tags (location binary(20), type int)
+```
+
+上述SQL创建了一个名为thermometer的STable，带有标签location和标签type。
+
+为某个采集点创建表时，可以指定其所属的STable以及标签的值，语法如下：
+
+```mysql
+CREATE TABLE <tb_name> USING <stb_name> TAGS (tag_value1,...)
+```
+
+沿用上面温度计的例子，使用超级表thermometer建立单个温度计数据表的语句如下：
+
+```mysql
+create table t1 using thermometer tags (‘beijing’, 10)
+```
+
+上述SQL以thermometer为模板，创建了名为t1的表，这张表的Schema就是thermometer的Schema，但标签location值为‘beijing’，标签type值为10。
+
+用户可以使用一个STable创建数量无上限的具有不同标签的表，从这个意义上理解，STable就是若干具有相同数据模型，不同标签的表的集合。与普通表一样，用户可以创建、删除、查看超级表STable，大部分适用于普通表的查询操作都可运用到STable上，包括各种聚合和投影选择函数。除此之外，可以设置标签的过滤条件，仅对STbale中部分表进行聚合查询，大大简化应用的开发。
+
+TDengine对表的主键（时间戳）建立索引，暂时不提供针对数据模型中其他采集量（比如温度、压力值）的索引。每个数据采集点会采集若干数据记录，但每个采集点的标签仅仅是一条记录，因此数据标签在存储上没有冗余，且整体数据规模有限。TDengine将标签数据与采集的动态数据完全分离存储，而且针对STable的标签建立了高性能内存索引结构，为标签提供全方位的快速操作支持。用户可按照需求对其进行增删改查（Create，Retrieve，Update，Delete，CRUD）操作。
+
+STable从属于库，一个STable只属于一个库，但一个库可以有一到多个STable, 一个STable可有多个子表。
+
+## 超级表管理
+
+- 创建超级表
+
+    ```mysql
+    CREATE TABLE <stable_name> (<field_name> TIMESTAMP, field_name1 field_type,…) TAGS(tag_name tag_type, …)
+    ```
+
+    与创建表的SQL语法相似。但需指定TAGS字段的名称和类型。 
+
+    说明：
+
+    1. TAGS列总长度不能超过512 bytes；
+    2. TAGS列的数据类型不能是timestamp和nchar类型；
+    3. TAGS列名不能与其他列名相同;
+    4. TAGS列名不能为预留关键字. 
+
+- 显示已创建的超级表
+
+    ```mysql
+    show stables;
+    ```
+
+    查看数据库内全部STable，及其相关信息，包括STable的名称、创建时间、列数量、标签（TAG）数量、通过该STable建表的数量。 
+
+- 删除超级表
+
+    ```mysql
+    DROP TABLE <stable_name>
+    ```
+
+    Note: 删除STable不会级联删除通过STable创建的表；相反删除STable时要求通过该STable创建的表都已经被删除。
+
+- 查看属于某STable并满足查询条件的表
+
+    ```mysql
+    SELECT TBNAME,[TAG_NAME,…] FROM <stable_name> WHERE <tag_name> <[=|=<|>=|<>] values..> ([AND|OR] …)
+    ```
+
+    查看属于某STable并满足查询条件的表。说明：TBNAME为关键词，显示通过STable建立的子表表名，查询过程中可以使用针对标签的条件。
+
+    ```mysql
+    SELECT COUNT(TBNAME) FROM <stable_name> WHERE <tag_name> <[=|=<|>=|<>] values..> ([AND|OR] …)
+    ```
+
+    统计属于某个STable并满足查询条件的子表的数量
+
+## 写数据时自动建子表
+
+在某些特殊场景中，用户在写数据时并不确定某个设备的表是否存在，此时可使用自动建表语法来实现写入数据时里用超级表定义的表结构自动创建不存在的子表，若该表已存在则不会建立新表。注意：自动建表语句只能自动建立子表而不能建立超级表，这就要求超级表已经被事先定义好。自动建表语法跟insert/import语法非常相似，唯一区别是语句中增加了超级表和标签信息。具体语法如下：
+
+```mysql
+INSERT INTO <tb_name> USING <stb_name> TAGS (<tag1_value>, ...) VALUES (field_value, ...) (field_value, ...) ...;
+```
+
+向表tb_name中插入一条或多条记录，如果tb_name这张表不存在，则会用超级表stb_name定义的表结构以及用户指定的标签值(即tag1_value…)来创建名为tb_name新表，并将用户指定的值写入表中。如果tb_name已经存在，则建表过程会被忽略，系统也不会检查tb_name的标签是否与用户指定的标签值一致，也即不会更新已存在表的标签。
+
+```mysql
+INSERT INTO <tb1_name> USING <stb1_name> TAGS (<tag1_value1>, ...) VALUES (<field1_value1>, ...) (<field1_value2>, ...) ... <tb_name2> USING <stb_name2> TAGS(<tag1_value2>, ...) VALUES (<field1_value1>, ...) ...;
+```
+
+向多张表tb1_name，tb2_name等插入一条或多条记录，并分别指定各自的超级表进行自动建表。
+
+## STable中TAG管理
+
+除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于STable，不能对单个子表操作。对STable添加标签以后，依托于该STable建立的所有表将自动增加了一个标签，对于数值型的标签，新增加的标签的默认值是0.
+
+- 添加新的标签
+
+    ```mysql
+    ALTER TABLE <stable_name> ADD TAG <new_tag_name> <TYPE>
+    ```
+
+    为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过6个。
+
+- 删除标签
+
+    ```mysql
+    ALTER TABLE <stable_name> DROP TAG <tag_name>
+    ```
+
+    删除超级表的一个标签，从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。
+
+    	说明：第一列标签不能删除，至少需要为STable保留一个标签。
+
+- 修改标签名
+
+    ```mysql
+    ALTER TABLE <stable_name> CHANGE TAG <old_tag_name> <new_tag_name>
+    ```
+
+    	修改超级表的标签名，从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。
+
+- 修改子表的标签值
+
+    ```mysql
+    ALTER TABLE <table_name> SET TAG <tag_name>=<new_tag_value>
+    ```
+
+## STable多表聚合
+
+针对所有的通过STable创建的子表进行多表聚合查询，支持按照全部的TAG值进行条件过滤，并可将结果按照TAGS中的值进行聚合，暂不支持针对binary类型的模糊匹配过滤。语法如下：
+
+```mysql
+SELECT function<field_name>,… 
+ FROM <stable_name> 
+ WHERE <tag_name> <[=|<=|>=|<>] values..> ([AND|OR] …)
+ INTERVAL (<time range>)
+ GROUP BY <tag_name>, <tag_name>…
+ ORDER BY <tag_name> <asc|desc>
+ SLIMIT <group_limit>
+ SOFFSET <group_offset>
+ LIMIT <record_limit>
+ OFFSET <record_offset>
+```
+
+**说明**：
+
+超级表聚合查询，TDengine目前支持以下聚合\选择函数：sum、count、avg、first、last、min、max、top、bottom，以及针对全部或部分列的投影操作，使用方式与单表查询的计算过程相同。暂不支持其他类型的聚合计算和四则运算。当前所有的函数及计算过程均不支持嵌套的方式进行执行。
+
+ 不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY <tag_name> 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY <tag_name> 语句指定，每个分组内部，时间序列是单调递增的。 
+
+使用SLIMIT/SOFFSET语句指定组间分页，即指定结果集中输出的最大组数以及对组起始的位置。使用LIMIT/OFFSET语句指定组内分页，即指定结果集中每个组内最多输出多少条记录以及记录起始的位置。
+
+## STable使用示例
+
+以温度传感器采集时序数据作为例，示范STable的使用。 在这个例子中，对每个温度计都会建立一张表，表名为温度计的ID，温度计读数的时刻记为ts，采集的值记为degree。通过tags给每个采集器打上不同的标签，其中记录温度计的地区和类型，以方便我们后面的查询。所有温度计的采集量都一样，因此我们用STable来定义表结构。
+
+###定义STable表结构并使用它创建子表
+
+创建STable语句如下：
+
+```mysql
+CREATE TABLE thermometer (ts timestamp, degree double) 
+TAGS(location binary(20), type int)
+```
+
+假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下： 
+
+```mysql
+CREATE TABLE therm1 USING thermometer TAGS (’beijing’, 1);
+CREATE TABLE therm2 USING thermometer TAGS (’beijing’, 2);
+CREATE TABLE therm3 USING thermometer TAGS (’tianjin’, 1);
+CREATE TABLE therm4 USING thermometer TAGS (’shanghai’, 3);
+```
+
+其中therm1，therm2，therm3，therm4是超级表thermometer四个具体的子表，也即普通的Table。以therm1为例，它表示采集器therm1的数据，表结构完全由thermometer定义，标签location=”beijing”, type=1表示therm1的地区是北京，类型是第1类的温度计。
+
+###写入数据
+
+注意，写入数据时不能直接对STable操作，而是要对每张子表进行操作。我们分别向四张表therm1，therm2， therm3， therm4写入一条数据，写入语句如下：
+
+```mysql
+INSERT INTO therm1 VALUES (’2018-01-01 00:00:00.000’, 20);
+INSERT INTO therm2 VALUES (’2018-01-01 00:00:00.000’, 21);
+INSERT INTO therm3 VALUES (’2018-01-01 00:00:00.000’, 24);
+INSERT INTO therm4 VALUES (’2018-01-01 00:00:00.000’, 23);
+```
+
+###按标签聚合查询
+
+查询位于北京(beijing)和天津(tianjing)两个地区的温度传感器采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)、最低温度min(degree)，并将结果按所处地域(location)和传感器类型(type)进行聚合。
+
+```mysql
+SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)
+FROM thermometer
+WHERE location=’beijing’ or location=’tianjing’
+GROUP BY location, type 
+```
+
+###按时间周期聚合查询
+
+查询仅位于北京以外地区的温度传感器最近24小时(24h)采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)和最低温度min(degree)，将采集结果按照10分钟为周期进行聚合，并将结果按所处地域(location)和传感器类型(type)再次进行聚合。
+
+```mysql
+SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)
+FROM thermometer
+WHERE name<>’beijing’ and ts>=now-1d
+INTERVAL(10M)
+GROUP BY location, type
+```
\ No newline at end of file
diff --git a/documentation20/Super Table.md b/documentation20/Super Table.md
new file mode 100644
index 0000000000000000000000000000000000000000..a213567f6d67ed351fac67b821f4db1929fa3a22
--- /dev/null
+++ b/documentation20/Super Table.md	
@@ -0,0 +1,195 @@
+# STable: Super Table
+
+"One Table for One Device" design can improve the insert/query performance significantly for a single device. But it has a side effect, the aggregation of multiple tables becomes hard. To reduce the complexity and improve the efficiency, TDengine introduced a new concept: STable (Super Table).  
+
+## What is a Super Table
+
+STable is an abstract and a template for a type of device. A STable contains a set of devices (tables) that have the same schema or data structure. Besides the shared schema, a STable has a set of tags, like the model, serial number and so on. Tags are used to record the static attributes for the devices and are used to group a set of devices (tables) for aggregation. Tags are metadata of a table and can be added, deleted or changed.   
+
+TDengine does not save tags as a part of the data points collected. Instead, tags are saved as metadata. Each table has a set of tags. To improve query performance, tags are all cached and indexed. One table can only belong to one STable, but one STable may contain many tables. 
+
+Like a table, you can create, show, delete and describe STables. Most query operations on tables can be applied to STable too, including the aggregation and selector functions. For queries on a STable, if no tags filter, the operations are applied to all the tables created via this STable. If there is a tag filter, the operations are applied only to a subset of the tables which satisfy the tag filter conditions. It will be very convenient to use tags to put devices into different groups for aggregation.
+
+##Create a STable
+
+Similiar to creating a standard table, syntax is: 
+
+```mysql
+CREATE TABLE <stable_name> (<field_name> TIMESTAMP, field_name1 field_type,…) TAGS(tag_name tag_type, …)
+```
+
+New keyword "tags" is introduced, where tag_name is the tag name, and tag_type is the associated data type. 
+
+Note：
+
+1. The bytes of all tags together shall be less than 512 
+2. Tag's data type can not be time stamp or nchar
+3. Tag name shall be different from the field name
+4. Tag name shall not be the same as system keywords
+5. Maximum number of tags is 6 
+
+For example:
+
+```mysql
+create table thermometer (ts timestamp, degree float) 
+tags (location binary(20), type int)
+```
+
+The above statement creates a STable thermometer with two tag "location" and "type"
+
+##Create a Table via STable
+
+To create a table for a device, you can use a STable as its template and assign the tag values. The syntax is:
+
+```mysql
+CREATE TABLE <tb_name> USING <stb_name> TAGS (tag_value1,...)
+```
+
+You can create any number of tables via a STable, and each table may have different tag values. For example, you create five tables via STable thermometer below:
+
+```mysql
+ create table t1 using thermometer tags (‘beijing’, 10);
+ create table t2 using thermometer tags (‘beijing’, 20);
+ create table t3 using thermometer tags (‘shanghai’, 10);
+ create table t4 using thermometer tags (‘shanghai’, 20);
+ create table t5 using thermometer tags (‘new york’, 10);
+```
+
+## Aggregate Tables via STable
+
+You can group a set of tables together by specifying the tags filter condition, then apply the aggregation operations. The result set can be grouped and ordered based on tag value. Syntax is：
+
+```mysql
+SELECT function<field_name>,… 
+ FROM <stable_name> 
+ WHERE <tag_name> <[=|<=|>=|<>] values..> ([AND|OR] …)
+ INTERVAL (<time range>)
+ GROUP BY <tag_name>, <tag_name>…
+ ORDER BY <tag_name> <asc|desc>
+ SLIMIT <group_limit>
+ SOFFSET <group_offset>
+ LIMIT <record_limit>
+ OFFSET <record_offset>
+```
+
+For the time being, STable supports only the following aggregation/selection functions: *sum, count, avg, first, last, min, max, top, bottom*, and the projection operations, the same syntax as a standard table.  Arithmetic operations are not supported, embedded queries not either. 
+
+*INTERVAL* is used for the aggregation over a time range.
+
+If *GROUP BY* is not used, the aggregation is applied to all the selected tables, and the result set is output in ascending order of the timestamp, but you can use "*ORDER BY _c0 ASC|DESC*" to specify the order you like. 
+
+If *GROUP BY <tag_name>* is used, the aggregation is applied to groups based on tags. Each group is aggregated independently. Result set is a group of aggregation results. The group order is decided by *ORDER BY <tag_name>*. Inside each group, the result set is in the ascending order of the time stamp. 
+
+*SLIMIT/SOFFSET* are used to limit the number of groups and starting group number.
+
+*LIMIT/OFFSET* are used to limit the number of records in a group and the starting rows.
+
+###Example 1:
+
+Check the average, maximum, and minimum temperatures of Beijing and Shanghai, and group the result set by location and type. The SQL statement shall be:
+
+```mysql
+SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)
+FROM thermometer
+WHERE location=’beijing’ or location=’tianjing’
+GROUP BY location, type 
+```
+
+### Example 2:
+
+List the number of records, average, maximum, and minimum temperature every 10 minutes for the past 24 hours for all the thermometers located in Beijing with type 10. The SQL statement shall be:
+
+```mysql
+SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)
+FROM thermometer
+WHERE name=’beijing’ and type=10 and ts>=now-1d
+INTERVAL(10M)
+```
+
+## Create Table Automatically
+
+Insert operation will fail if the table is not created yet. But for STable, TDengine can create the table automatically if the application provides the STable name, table name and tags' value when inserting data points. The syntax is:
+
+```mysql
+INSERT INTO <tb_name> USING <stb_name> TAGS (<tag1_value>, ...) VALUES (field_value, ...) (field_value, ...) ... <tb_name2> USING <stb_name2> TAGS(<tag1_value2>, ...) VALUES (<field1_value1>, ...) ...;
+```
+
+When inserting data points into table tb_name, the system will check if table tb_name is created or not. If it is already created, the data points will be inserted as usual. But if the table is not created yet, the system will create the table tb_bame using STable stb_name as the template with the tags. Multiple tables can be specified in the SQL statement. 
+
+## Management of STables
+
+After you can create a STable, you can describe, delete, change STables. This section lists all the supported operations.
+
+### Show STables in current DB
+
+```mysql
+show stables;
+```
+
+It lists all STables in current DB, including the name, created time, number of fileds, number of tags, and number of tables which are created via this STable. 
+
+### Describe a STable
+
+```mysql
+DESCRIBE <stable_name>
+```
+
+It lists the STable's schema and tags
+
+### Drop a STable
+
+```mysql
+DROP TABLE <stable_name>
+```
+
+To delete a STable, all the tables created via this STable shall be deleted first, otherwise, it will fail.
+
+### List the Associated Tables of a STable
+
+```mysql
+SELECT TBNAME,[TAG_NAME,…] FROM <stable_name> WHERE <tag_name> <[=|=<|>=|<>] values..> ([AND|OR] …)
+```
+
+It will list all the tables which satisfy the tag filter conditions. The tables are all created from this specific STable. TBNAME is a new keyword introduced, it is the table name associated with the STable. 
+
+```mysql
+SELECT COUNT(TBNAME) FROM <stable_name> WHERE <tag_name> <[=|=<|>=|<>] values..> ([AND|OR] …)
+```
+
+The above SQL statement will list the number of tables in a STable, which satisfy the filter condition.
+
+## Management of Tags
+
+You can add, delete and change the tags for a STable, and you can change the tag value of a table. The SQL commands are listed below.  
+
+###Add a Tag
+
+```mysql
+ALTER TABLE <stable_name> ADD TAG <new_tag_name> <TYPE>
+```
+
+It adds a new tag to the STable with a data type. The maximum number of tags is 6. 
+
+###Drop a Tag
+
+```mysql
+ALTER TABLE <stable_name> DROP TAG <tag_name>
+```
+
+It drops a tag from a STable. The first tag could not be deleted, and there must be at least one tag.
+
+###Change a Tag's Name
+
+```mysql
+ALTER TABLE <stable_name> CHANGE TAG <old_tag_name> <new_tag_name>
+```
+
+It changes the name of a tag from old to new. 
+
+###Change the Tag's Value
+
+```mysql
+ALTER TABLE <table_name> SET TAG <tag_name>=<new_tag_value>
+```
+
+It changes a table's tag value to a new one. 
diff --git a/documentation20/TAOS SQL-ch.md b/documentation20/TAOS SQL-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..024a3c4c24f6fff143bccb3e491d82b134fa4605
--- /dev/null
+++ b/documentation20/TAOS SQL-ch.md	
@@ -0,0 +1,972 @@
+# TAOS SQL
+
+本文档说明TAOS SQL支持的语法规则、主要查询功能、支持的SQL查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的SQL语言的基础。
+
+TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手，在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上，TAOS SQL并不是也不试图提供SQL标准的语法。此外，由于TDengine针对的时序性结构化数据不提供修改和更新功能，因此在TAO SQL中不提供数据更新和数据删除的相关功能。
+
+本章节SQL语法遵循如下约定：
+
+- < > 里的内容是用户需要输入的，但不要输入<>本身
+- [ ]表示内容为可选项，但不能输入[]本身
+- | 表示多选一，选择其中一个即可，但不能输入|本身
+- … 表示前面的项可重复多个
+
+为更好地说明SQL语法的规则及其特点，本文假设存在一个数据集。以智能电表(meters)为例，假设每个智能电表采集电流、电压、相位三个量。其建模如下：
+```mysql
+taos> DESCRIBE meters;
+             Field              |        Type        |   Length    |    Note    |
+=================================================================================
+ ts                             | TIMESTAMP          |           8 |            |
+ current                        | FLOAT              |           4 |            |
+ voltage                        | INT                |           4 |            |
+ phase                          | FLOAT              |           4 |            |
+ location                       | BINARY             |          64 | TAG        |
+ groupid                        | INT                |           4 | TAG        |
+```
+数据集包含4个智能电表的数据，按照TDengine的建模规则，对应4个子表，其名称分别是 d1001, d1002, d1003, d1004。
+
+## 支持的数据类型
+
+使用TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则：
+
+- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128```
+- 内部函数now是服务器的当前时间
+- 插入记录时，如果时间戳为0，插入数据时使用服务器当前时间
+- Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数
+- 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据
+- TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下：interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d)
+
+TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。
+
+在TDengine中，普通表的数据模型中可使用以下10种数据类型。 
+
+|      |   类型    | Bytes  | 说明                                                         |
+| ---- | :-------: | ------ | ------------------------------------------------------------ |
+| 1    | TIMESTAMP | 8      | 时间戳。最小精度毫秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。 |
+| 2    |    INT    | 4      | 整型，范围 [-2^31+1,   2^31-1], -2^31用作Null           |
+| 3    |  BIGINT   | 8      | 长整型，范围 [-2^63+1,   2^63-1], -2^63用于NULL                                 |
+| 4    |   FLOAT   | 4      | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38]                  |
+| 5    |  DOUBLE   | 8      | 双精度浮点型，有效位数15-16，范围 [-1.7E308,   1.7E308]      |
+| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有16374字节，但由于每行数据最多16K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
+| 7    | SMALLINT  | 2      | 短整型， 范围 [-32767, 32767], -32768用于NULL                                |
+| 8    |  TINYINT  | 1      | 单字节整型，范围 [-127, 127], -128用于NULL                                |
+| 9    |   BOOL    | 1      | 布尔型，{true,   false}                                      |
+| 10   |   NCHAR   | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将会报错。 |
+
+**Tips**: TDengine对SQL语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。
+
+## 数据库管理
+
+- **创建数据库**  
+    ```mysql
+    CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep]
+    ```
+    说明：
+    
+    1) `KEEP`是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据；
+    2) 数据库名最大长度为33；
+    3) 一条SQL 语句的最大长度为65480个字符；
+    4) 数据库还有更多与存储相关的配置参数，请参见[系统管理](../administrator/#服务端配置)。
+
+
+- **使用数据库**
+    ```mysql
+    USE db_name
+    ```
+    使用/切换数据库
+
+
+- **删除数据库**
+    ```mysql
+    DROP DATABASE [IF EXISTS] db_name
+    ```
+    删除数据库。所包含的全部数据表将被删除，谨慎使用
+
+
+- **显示系统所有数据库**
+    ```mysql
+    SHOW DATABASES
+    ```
+
+
+## 表管理
+- **创建数据表**
+  
+    ```mysql
+    CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...])
+    ```
+    说明：
+    1) 表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；
+    2) 表名最大长度为193；
+    3) 表的每行长度不能超过16k个字符;     
+    5) 使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节；
+
+- **删除数据表**
+
+    ```mysql
+    DROP TABLE [IF EXISTS] tb_name
+    ```
+
+- **显示当前数据库下的所有数据表信息**
+
+    ```mysql
+    SHOW TABLES [LIKE tb_name_wildcar]
+    ```
+
+    显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。
+
+
+- **获取表的结构信息**
+
+    ```mysql
+    DESCRIBE tb_name
+    ```
+
+- **表增加列**
+
+    ```mysql
+    ALTER TABLE tb_name ADD COLUMN field_name data_type
+    ```
+    说明：
+    1) 列的最大个数为1024，最小个数为2；
+    2) 列名最大长度为65；
+
+- **表删除列**
+
+    ```mysql
+    ALTER TABLE tb_name DROP COLUMN field_name 
+    ```
+    如果表是通过[超级表](../super-table/)创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构
+
+## 超级表STable管理
+- **创建超级表**
+  
+    ```mysql
+    CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3])
+    ```
+    创建STable, 与创建表的SQL语法相似，但需指定TAGS字段的名称和类型
+    
+    说明：
+    1) TAGS 列的数据类型不能是timestamp类型；
+    2) TAGS 列名不能与其他列名相同；
+    3) TAGS 列名不能为预留关键字；
+    4) TAGS 最多允许128个，可以0个，总长度不超过16k个字符
+  
+- **删除超级表**
+
+    ```mysql
+    DROP TABLE [IF EXISTS] stb_name
+    ```
+    删除STable会自动删除通过STable创建的字表。
+
+- **显示当前数据库下的所有超级表信息**
+
+    ```mysql
+    SHOW STABLES [LIKE tb_name_wildcar]
+    ```
+    查看数据库内全部STable，及其相关信息，包括STable的名称、创建时间、列数量、标签（TAG）数量、通过该STable建表的数量。
+
+- **获取超级表的结构信息**
+
+    ```mysql
+    DESCRIBE stb_name
+    ```
+    
+- **超级表增加列**
+
+    ```mysql
+    ALTER TABLE stb_name ADD COLUMN field_name data_type
+    ```
+
+- **超级表删除列**
+
+    ```mysql
+    ALTER TABLE stb_name DROP COLUMN field_name 
+    ```
+
+## 超级表 STable 中 TAG 管理
+- **添加标签**
+  
+    ```mysql
+    ALTER TABLE stb_name ADD TAG new_tag_name tag_type
+    ```
+    为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符.
+
+- **删除标签**
+
+    ```mysql
+    ALTER TABLE stb_name DROP TAG tag_name
+    ```
+    删除超级表的一个标签，从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。
+
+- **修改标签名**
+
+    ```mysql
+    ALTER TABLE stb_name CHANGE TAG old_tag_name new_tag_name
+    ```
+    修改超级表的标签名，从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。
+
+- **修改字表标签值**
+
+    ```mysql
+    ALTER TABLE tb_name SET TAG tag_name=new_tag_value
+    ```
+    说明：除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于STable，不能对单个子表操作。对STable添加标签以后，依托于该STable建立的所有表将自动增加了一个标签，所有新增标签的默认值都是NULL。
+
+## 数据写入
+
+- **插入一条记录**
+    ```mysql
+    INSERT INTO tb_name VALUES (field_value, ...);
+    ```
+    向表tb_name中插入一条记录
+
+
+- **插入一条记录，数据对应到指定的列**
+    ```mysql
+    INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...)
+    ```
+    向表tb_name中插入一条记录，数据对应到指定的列。SQL语句中没有出现的列，数据库将自动填充为NULL。主键（时间戳）不能为NULL。
+
+
+- **插入多条记录**
+    ```mysql
+    INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...;
+    ```
+    向表tb_name中插入多条记录
+
+
+- **按指定的列插入多条记录**
+    ```mysql
+    INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
+    ```
+    向表tb_name中按指定的列插入多条记录
+
+
+- **向多个表插入多条记录**
+    ```mysql
+    INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)... 
+                tb2_name VALUES (field1_value1, ...)(field1_value2, ...)...;
+    ```
+    同时向表tb1_name和tb2_name中分别插入多条记录
+
+
+- **同时向多个表按列插入多条记录**
+    ```mysql
+    INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value1, ...)
+                tb2_name (tb2_field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
+    ```
+    同时向表tb1_name和tb2_name中按列分别插入多条记录 
+
+    注意：
+    1) 如果时间戳为0，系统将自动使用服务器当前时间作为该记录的时间戳；
+    2) 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。
+
+**历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。
+
+## 数据查询
+
+### 查询语法：
+
+```mysql
+SELECT select_expr [, select_expr ...]
+    FROM {tb_name_list}
+    [WHERE where_condition]
+    [INTERVAL [interval_offset,] interval_val]
+    [FILL fill_val]
+    [SLIDING fill_val]
+    [GROUP BY col_list]
+    [ORDER BY col_list { DESC | ASC }]    
+    [SLIMIT limit_val [, SOFFSET offset_val]]
+    [LIMIT limit_val [, OFFSET offset_val]]
+    [>> export_file]
+```
+
+#### SELECT子句
+一个选择子句可以是联合查询（UNION）和另一个查询的子查询（SUBQUERY）。
+
+##### 通配符
+通配符 * 可以用于代指全部列。对于普通表，结果中只有普通列。
+```mysql
+taos> SELECT * FROM d1001;
+           ts            |       current        |   voltage   |        phase         |
+======================================================================================
+ 2018-10-03 14:38:05.000 |             10.30000 |         219 |              0.31000 |
+ 2018-10-03 14:38:15.000 |             12.60000 |         218 |              0.33000 |
+ 2018-10-03 14:38:16.800 |             12.30000 |         221 |              0.31000 |
+Query OK, 3 row(s) in set (0.001165s)
+```
+
+在针对超级表，通配符包含 _标签列_ 。
+```mysql
+taos> SELECT * FROM meters;
+           ts            |       current        |   voltage   |        phase         |            location            |   groupid   |
+=====================================================================================================================================
+ 2018-10-03 14:38:05.500 |             11.80000 |         221 |              0.28000 | Beijing.Haidian                |           2 |
+ 2018-10-03 14:38:16.600 |             13.40000 |         223 |              0.29000 | Beijing.Haidian                |           2 |
+ 2018-10-03 14:38:05.000 |             10.80000 |         223 |              0.29000 | Beijing.Haidian                |           3 |
+ 2018-10-03 14:38:06.500 |             11.50000 |         221 |              0.35000 | Beijing.Haidian                |           3 |
+ 2018-10-03 14:38:04.000 |             10.20000 |         220 |              0.23000 | Beijing.Chaoyang               |           3 |
+ 2018-10-03 14:38:16.650 |             10.30000 |         218 |              0.25000 | Beijing.Chaoyang               |           3 |
+ 2018-10-03 14:38:05.000 |             10.30000 |         219 |              0.31000 | Beijing.Chaoyang               |           2 |
+ 2018-10-03 14:38:15.000 |             12.60000 |         218 |              0.33000 | Beijing.Chaoyang               |           2 |
+ 2018-10-03 14:38:16.800 |             12.30000 |         221 |              0.31000 | Beijing.Chaoyang               |           2 |
+Query OK, 9 row(s) in set (0.002022s)
+```
+
+通配符支持表名前缀，以下两个SQL语句均为返回全部的列：
+```mysql
+SELECT * FROM d1001;
+SELECT d1001.* FROM d1001;
+```
+在Join查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。
+```mysql
+taos> SELECT * FROM d1001, d1003 WHERE d1001.ts=d1003.ts;
+           ts            | current |   voltage   |    phase     |           ts            | current |   voltage   |    phase     |
+==================================================================================================================================
+ 2018-10-03 14:38:05.000 | 10.30000|         219 |      0.31000 | 2018-10-03 14:38:05.000 | 10.80000|         223 |      0.29000 |
+Query OK, 1 row(s) in set (0.017385s)
+```
+```mysql
+taos> SELECT d1001.* FROM d1001,d1003 WHERE d1001.ts = d1003.ts;
+           ts            |       current        |   voltage   |        phase         |
+======================================================================================
+ 2018-10-03 14:38:05.000 |             10.30000 |         219 |              0.31000 |
+Query OK, 1 row(s) in set (0.020443s)
+```
+
+在使用SQL函数来进行查询过程中，部分SQL函数支持通配符操作。其中的区别在于：
+```count(\*)```函数只返回一列。```first```、```last```、```last_row```函数则是返回全部列。
+
+```
+taos> select count(*) from d1001;
+       count(*)        |
+========================
+                     3 |
+Query OK, 1 row(s) in set (0.001035s)
+```
+
+```
+taos> select first(*) from d1001;
+        first(ts)        |    first(current)    | first(voltage) |     first(phase)     |
+=========================================================================================
+ 2018-10-03 14:38:05.000 |             10.30000 |            219 |              0.31000 |
+Query OK, 1 row(s) in set (0.000849s)
+```
+
+#### 结果集列名
+
+```SELECT```子句中，如果不指定返回结果集合的列名，结果集列名称默认使用```SELECT```子句中的表达式名称作为列名称。此外，用户可使用```AS```来重命名返回结果集合中列的名称。例如：
+```
+taos> select ts, ts as primary_key_ts from d1001;
+           ts            |     primary_key_ts      |
+====================================================
+ 2018-10-03 14:38:05.000 | 2018-10-03 14:38:05.000 |
+ 2018-10-03 14:38:15.000 | 2018-10-03 14:38:15.000 |
+ 2018-10-03 14:38:16.800 | 2018-10-03 14:38:16.800 |
+Query OK, 3 row(s) in set (0.001191s)
+```
+但是针对```first(*)```、```last(*)```、```last_row(*)```不支持针对单列的重命名。
+
+#### 隐式结果列
+```Select_exprs```可以是表所属列的列名，也可以是基于列的函数表达式或计算式，数量的上限256个。当用户使用了```interval```或```group by tags```的子句以后，在最后返回结果中会强制返回时间戳列（第一列）和group by子句中的标签列。后续的版本中可以支持关闭group by子句中隐式列的输出，列输出完全由select子句控制。
+
+#### 表（超级表）列表
+
+FROM关键字后面可以是若干个表（超级表）列表，也可以是子查询的结果。
+如果没有指定用户的当前数据库，可以在表名称之前使用数据库的名称来指定表所属的数据库。例如：```power.d1001``` 方式来跨库使用表。
+```
+SELECT * FROM power.d1001;
+------------------------------
+use power;
+SELECT * FROM d1001;
+```
+
+#### 特殊功能
+部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database() 
+```
+taos> SELECT database();
+           database()           |
+=================================
+ power                          |
+Query OK, 1 row(s) in set (0.000079s)
+```
+如果登录的时候没有指定默认数据库，且没有使用```use``命令切换数据，则返回NULL。
+```
+taos> SELECT database();
+           database()           |
+=================================
+ NULL                           |
+Query OK, 1 row(s) in set (0.000184s)
+```
+获取服务器和客户端版本号:
+```
+taos> SELECT client_version();
+ client_version() |
+===================
+ 2.0.0.0          |
+Query OK, 1 row(s) in set (0.000070s)
+
+taos> SELECT server_version();
+ server_version() |
+===================
+ 2.0.0.0          |
+Query OK, 1 row(s) in set (0.000077s)
+```
+服务器状态检测语句。如果服务器正常，返回一个数字（例如 1）。如果服务器异常，返回error code。该SQL语法能兼容连接池对于TDengine状态的检查及第三方工具对于数据库服务器状态的检查。并可以避免出现使用了错误的心跳检测SQL语句导致的连接池连接丢失的问题。
+```
+taos> SELECT server_status();
+ server_status() |
+==================
+               1 |
+Query OK, 1 row(s) in set (0.000074s)
+
+taos> SELECT server_status() as status;
+   status    |
+==============
+           1 |
+Query OK, 1 row(s) in set (0.000081s)
+```
+#### TAOS SQL中特殊关键词
+
+ >   TBNAME： 在超级表查询中可视为一个特殊的标签，代表查询涉及的子表名<br>
+    \_c0: 表示表（超级表）的第一列
+
+#### 小技巧
+获取一个超级表所有的子表名及相关的标签信息：
+```
+SELECT TBNAME, location FROM meters
+```
+统计超级表下辖子表数量：
+```
+SELECT COUNT(TBNAME) FROM meters
+```
+以上两个查询均只支持在Where条件子句中添加针对标签（TAGS）的过滤条件。例如：
+```
+taos> SELECT TBNAME, location FROM meters;
+             tbname             |            location            |
+==================================================================
+ d1004                          | Beijing.Haidian                |
+ d1003                          | Beijing.Haidian                |
+ d1002                          | Beijing.Chaoyang               |
+ d1001                          | Beijing.Chaoyang               |
+Query OK, 4 row(s) in set (0.000881s)
+
+taos> SELECT count(tbname) FROM meters WHERE groupId > 2;
+     count(tbname)     |
+========================
+                     2 |
+Query OK, 1 row(s) in set (0.001091s)
+```
+
+- 可以使用* 返回所有列，或指定列名。可以对数字列进行四则运算，可以给输出的列取列名
+- where语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串
+- 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序(_c0指首列时间戳)。使用ORDER BY对其他字段进行排序为非法操作。
+- 参数LIMIT控制输出条数，OFFSET指定从第几条开始输出。LIMIT/OFFSET对结果集的执行顺序在ORDER BY之后。
+- 通过”>>"输出结果可以导出到指定文件
+
+### 支持的条件过滤操作
+
+| Operation | Note                          | Applicable Data Types                 |
+| --------- | ----------------------------- | ------------------------------------- |
+| >         | larger than                   | **`timestamp`** and all numeric types |
+| <         | smaller than                  | **`timestamp`** and all numeric types |
+| >=        | larger than or equal to       | **`timestamp`** and all numeric types |
+| <=        | smaller than or equal to      | **`timestamp`** and all numeric types |
+| =         | equal to                      | all types                             |
+| <>        | not equal to                  | all types                             |
+| %         | match with any char sequences | **`binary`** **`nchar`**              |
+| _         | match with a single char      | **`binary`** **`nchar`**              |
+
+1. 同时进行多个字段的范围过滤需要使用关键词AND进行连接不同的查询条件，暂不支持OR连接的查询条件。
+2. 针对某一字段的过滤只支持单一区间的过滤条件。例如：value>20 and value<30是合法的过滤条件, 而Value<20 AND value<>5是非法的过滤条件。
+
+### Some Examples 
+
+- 对于下面的例子，表tb1用以下语句创建
+
+    ```mysql
+    CREATE TABLE tb1 (ts timestamp, col1 int, col2 float, col3 binary(50))
+    ```
+
+- 查询tb1刚过去的一个小时的所有记录
+
+    ```mysql
+    SELECT * FROM tb1 WHERE ts >= NOW - 1h
+    ```
+
+- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序
+
+    ```mysql
+    SELECT * FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND ts <= '2018-06-02 08:00:00.000' AND col3 LIKE '%nny' ORDER BY ts DESC
+    ```
+
+- 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始
+
+    ```mysql
+    SELECT (col1 + col2) AS 'complex' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' and col2 > 1.2 LIMIT 10 OFFSET 5
+    ```
+
+- 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`.
+
+    ```mysql
+    SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv
+    ```
+
+## SQL函数
+
+### 聚合函数
+
+TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数如下：
+
+- **COUNT**
+    ```mysql
+    SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表/超级表中记录行数或某列的非空值个数。  
+    返回结果数据类型：长整型INT64。  
+    应用字段：应用全部字段。  
+    适用于：表、超级表。  
+    说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。 
+
+    示例：
+    ```mysql
+    taos> SELECT COUNT(*), COUNT(VOLTAGE) FROM meters;
+        count(*)        |    count(voltage)     |
+    ================================================
+                        9 |                     9 |
+    Query OK, 1 row(s) in set (0.004475s)
+
+    taos> SELECT COUNT(*), COUNT(VOLTAGE) FROM d1001;
+        count(*)        |    count(voltage)     |
+    ================================================
+                        3 |                     3 |
+    Query OK, 1 row(s) in set (0.001075s)
+    ```
+
+
+- **AVG**
+    ```mysql
+    SELECT AVG(field_name) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的平均值。  
+    返回结果数据类型：双精度浮点数Double。  
+    应用字段：不能应用在timestamp、binary、nchar、bool字段。  
+    适用于：表、超级表。  
+    
+    示例：
+    ```mysql
+    taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters;
+        avg(current)        |       avg(voltage)        |        avg(phase)         |
+    ====================================================================================
+                11.466666751 |             220.444444444 |               0.293333333 |
+    Query OK, 1 row(s) in set (0.004135s)
+
+    taos> SELECT AVG(current), AVG(voltage), AVG(phase) from d1001;
+        avg(current)        |       avg(voltage)        |        avg(phase)         |
+    ====================================================================================
+                11.733333588 |             219.333333333 |               0.316666673 |
+    Query OK, 1 row(s) in set (0.000943s)
+    ```
+
+- **TWA**
+    ```mysql
+    SELECT TWA(field_name) FROM tb_name WHERE clause
+    ```
+    功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。  
+    返回结果数据类型：双精度浮点数Double。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
+    说明：时间加权平均(time weighted average, TWA）查询需要指定查询时间段的 _开始时间_ 和 _结束时间_ 。
+    适用于：表、超级表。
+
+- **SUM**
+    ```mysql
+    SELECT SUM(field_name) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的和。  
+    返回结果数据类型：双精度浮点数Double和长整型INT64。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    适用于：表、超级表。    
+
+    示例：
+    ```mysql
+    taos> SELECT SUM(current), SUM(voltage), SUM(phase) from meters;
+        sum(current)        |     sum(voltage)      |        sum(phase)         |
+    ================================================================================
+                103.200000763 |                  1984 |               2.640000001 |
+    Query OK, 1 row(s) in set (0.001702s)
+
+    taos> SELECT SUM(current), SUM(voltage), SUM(phase) from d1001;
+        sum(current)        |     sum(voltage)      |        sum(phase)         |
+    ================================================================================
+                35.200000763 |                   658 |               0.950000018 |
+    Query OK, 1 row(s) in set (0.000980s)
+    ```
+
+- **STDDEV**
+    ```mysql
+    SELECT STDDEV(field_name) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表中某列的均方差。  
+    返回结果数据类型：双精度浮点数Double。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    适用于：表。
+
+    示例：
+    ```mysql
+    taos> SELECT STDDEV(current) FROM d1001;
+        stddev(current)      |
+    ============================
+                1.020892909 |
+    Query OK, 1 row(s) in set (0.000915s)
+    ```
+
+- **LEASTSQUARES**
+    ```mysql
+    SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。  
+    返回结果数据类型：字符串表达式（斜率, 截距）。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：自变量是时间戳，因变量是该列的值。  
+    适用于：表。
+
+    示例：
+    ```mysql
+    taos> SELECT LEASTSQUARES(current, 1, 1) FROM d1001;
+                leastsquares(current, 1, 1)             |
+    =====================================================
+    {slop:1.000000, intercept:9.733334}                 |
+    Query OK, 1 row(s) in set (0.000921s)
+    ```
+
+### 选择函数
+
+- **MIN**
+    ```mysql
+    SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的值最小值。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
+
+    示例：
+    ```mysql
+    taos> SELECT MIN(current), MIN(voltage) FROM meters;
+        min(current)     | min(voltage) |
+    ======================================
+                10.20000 |          218 |
+    Query OK, 1 row(s) in set (0.001765s)
+
+    taos> SELECT MIN(current), MIN(voltage) FROM d1001;
+        min(current)     | min(voltage) |
+    ======================================
+                10.30000 |          218 |
+    Query OK, 1 row(s) in set (0.000950s)
+    ```
+
+- **MAX**
+    ```mysql
+    SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的值最大值。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
+
+    示例：
+    ```mysql
+    taos> SELECT MAX(current), MAX(voltage) FROM meters;
+        max(current)     | max(voltage) |
+    ======================================
+                13.40000 |          223 |
+    Query OK, 1 row(s) in set (0.001123s)
+
+    taos> SELECT MAX(current), MAX(voltage) FROM d1001;
+        max(current)     | max(voltage) |
+    ======================================
+                12.60000 |          221 |
+    Query OK, 1 row(s) in set (0.000987s)
+    ```
+
+
+- **FIRST**
+    ```mysql
+    SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的值最先写入的非NULL值。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：所有字段。  
+    说明：1）如果要返回各个列的首个（时间戳最小）非NULL值，可以使用FIRST(*)；2) 如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；3) 如果结果集中所有列全部为NULL值，则不返回结果。
+
+    示例：
+    ```mysql
+    taos> SELECT FIRST(*) FROM meters;
+            first(ts)        |    first(current)    | first(voltage) |     first(phase)     |
+    =========================================================================================
+    2018-10-03 14:38:04.000 |             10.20000 |            220 |              0.23000 |
+    Query OK, 1 row(s) in set (0.004767s)
+
+    taos> SELECT FIRST(current) FROM d1002;
+        first(current)    |
+    =======================
+                10.20000 |
+    Query OK, 1 row(s) in set (0.001023s)
+    ```
+
+- **LAST**
+    ```mysql
+    SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的值最后写入的非NULL值。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：所有字段。  
+    说明：1）如果要返回各个列的最后（时间戳最大）一个非NULL值，可以使用LAST(*)；2）如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；如果结果集中所有列全部为NULL值，则不返回结果。
+
+    示例：
+    ```mysql
+    taos> SELECT LAST(*) FROM meters;
+            last(ts)         |    last(current)     | last(voltage) |     last(phase)      |
+    ========================================================================================
+    2018-10-03 14:38:16.800 |             12.30000 |           221 |              0.31000 |
+    Query OK, 1 row(s) in set (0.001452s)
+
+    taos> SELECT LAST(current) FROM d1002;
+        last(current)     |
+    =======================
+                10.30000 |
+    Query OK, 1 row(s) in set (0.000843s)
+    ```
+
+- **TOP**
+    ```mysql
+    SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。     
+    返回结果数据类型：同应用的字段。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 
+
+    示例：
+    ```mysql
+    taos> SELECT TOP(current, 3) FROM meters;
+            ts            |   top(current, 3)    |
+    =================================================
+    2018-10-03 14:38:15.000 |             12.60000 |
+    2018-10-03 14:38:16.600 |             13.40000 |
+    2018-10-03 14:38:16.800 |             12.30000 |
+    Query OK, 3 row(s) in set (0.001548s)
+
+    taos> SELECT TOP(current, 2) FROM d1001;
+            ts            |   top(current, 2)    |
+    =================================================
+    2018-10-03 14:38:15.000 |             12.60000 |
+    2018-10-03 14:38:16.800 |             12.30000 |
+    Query OK, 2 row(s) in set (0.000810s)
+    ```
+
+- **BOTTOM**
+    ```mysql
+    SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。
+
+    示例：
+    ```mysql
+    taos> SELECT BOTTOM(voltage, 2) FROM meters;
+            ts            | bottom(voltage, 2) |
+    ===============================================
+    2018-10-03 14:38:15.000 |                218 |
+    2018-10-03 14:38:16.650 |                218 |
+    Query OK, 2 row(s) in set (0.001332s)
+
+    taos> SELECT BOTTOM(current, 2) FROM d1001;
+            ts            |  bottom(current, 2)  |
+    =================================================
+    2018-10-03 14:38:05.000 |             10.30000 |
+    2018-10-03 14:38:16.800 |             12.30000 |
+    Query OK, 2 row(s) in set (0.000793s)
+    ```
+
+- **PERCENTILE**
+    ```mysql
+    SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表中某列的值百分比分位数。  
+    返回结果数据类型： 双精度浮点数Double。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
+
+    示例：
+    ```mysql
+    taos> SELECT PERCENTILE(current, 20) FROM d1001;
+    percentile(current, 20)  |
+    ============================
+                11.100000191 |
+    Query OK, 1 row(s) in set (0.000787s)
+    ```
+
+- **APERCENTILE**
+    ```mysql
+    SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。  
+    返回结果数据类型： 双精度浮点数Double。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
+    ```mysql
+    taos> SELECT APERCENTILE(current, 20) FROM d1001;
+    apercentile(current, 20)  |
+    ============================
+                10.300000191 |
+    Query OK, 1 row(s) in set (0.000645s)
+    ```
+    
+- **LAST_ROW**
+    ```mysql
+    SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }
+    ```
+    功能说明：返回表（超级表）的最后一条记录。  
+    返回结果数据类型：同应用的字段。  
+    应用字段：所有字段。  
+    说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。
+
+    示例：
+    ```mysql
+    taos> SELECT LAST_ROW(current) FROM meters;
+    last_row(current)   |
+    =======================
+                12.30000 |
+    Query OK, 1 row(s) in set (0.001238s)
+
+    taos> SELECT LAST_ROW(current) FROM d1002;
+    last_row(current)   |
+    =======================
+                10.30000 |
+    Query OK, 1 row(s) in set (0.001042s)
+    ```
+
+### 计算函数
+- **DIFF**
+    ```mysql
+    SELECT DIFF(field_name) FROM tb_name [WHERE clause]
+    ```
+    功能说明：统计表中某列的值与前一行对应值的差。  
+    返回结果数据类型： 同应用字段。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：输出结果行数是范围内总行数减一，第一行没有结果输出。
+    
+    示例：
+    ```mysql
+    taos> SELECT DIFF(current) FROM d1001;
+            ts            |    diff(current)     |
+    =================================================
+    2018-10-03 14:38:15.000 |              2.30000 |
+    2018-10-03 14:38:16.800 |             -0.30000 |
+    Query OK, 2 row(s) in set (0.001162s)
+    ```
+
+
+- **SPREAD**
+    ```mysql
+    SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列的最大值和最小值之差。  
+    返回结果数据类型： 双精度浮点数。  
+    应用字段：不能应用在binary、nchar、bool类型字段。  
+    说明：可用于TIMESTAMP字段，此时表示记录的时间覆盖范围。
+
+    示例：
+    ```mysql
+    taos> SELECT SPREAD(voltage) FROM meters;
+        spread(voltage)      |
+    ============================
+                5.000000000 |
+    Query OK, 1 row(s) in set (0.001792s)
+
+    taos> SELECT SPREAD(voltage) FROM d1001;
+        spread(voltage)      |
+    ============================
+                3.000000000 |
+    Query OK, 1 row(s) in set (0.000836s)
+    ```
+
+
+- **四则运算**
+
+    ```mysql
+    SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name }  [WHERE clause]
+    ```
+    功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。  
+    返回结果数据类型：双精度浮点数。  
+    应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
+    说明：1）支持两列或多列之间进行计算，可使用括号控制计算优先级;2）NULL字段不参与计算，如果参与计算的某行中包含NULL，该行的计算结果为NULL。
+
+    ```mysql
+    taos> SELECT current + voltage * phase FROM d1001;
+    (current+(voltage*phase)) |
+    ============================
+                78.190000713 |
+                84.540003240 |
+                80.810000718 |
+    Query OK, 3 row(s) in set (0.001046s)
+    ```
+
+## 时间维度聚合
+TDengine支持按时间段进行聚合，可以将表中数据按照时间段进行切割后聚合生成结果，比如温度传感器每秒采集一次数据，但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下：
+
+```mysql
+SELECT function_list FROM tb_name 
+  [WHERE where_condition]
+  INTERVAL (interval)
+  [FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
+
+SELECT function_list FROM stb_name 
+  [WHERE where_condition]
+  INTERVAL (interval)
+  [FILL ({ VALUE | PREV | NULL | LINEAR})]
+  [GROUP BY tags]
+```
+
+- 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。
+- WHERE语句可以指定查询的起止时间和其他过滤条件 
+- FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种：
+  1. 不进行填充：NONE(默认填充模式)。
+  
+  2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。
+  
+  3. NULL填充：使用NULL填充数据。例如：fill(null)。
+  
+  4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 
+  
+
+说明：  
+  1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。
+  2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
+  3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。
+
+**示例：**智能电表的建表语句如下：
+
+```mysql
+CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);
+```
+
+针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。
+使用的查询语句如下：
+
+```mysql
+SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters
+  WHERE TS>=NOW-1d
+  INTERVAL(10m)
+  FILL(PREV);
+```
+
+##TAOS SQL 边界限制
+- 数据库名最大长度为33
+- 表名最大长度为193，每行数据最大长度16k个字符
+- 列名最大长度为65，最多允许1024列，最少需要2列，第一列必须是时间戳
+- 标签最多允许128个，可以0个，标签总长度不超过16k个字符
+- SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改
+- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制
\ No newline at end of file
diff --git a/documentation20/TAOS SQL.md b/documentation20/TAOS SQL.md
new file mode 100644
index 0000000000000000000000000000000000000000..ae2d9d34a203de41d0deed523d16095c8830b452
--- /dev/null
+++ b/documentation20/TAOS SQL.md	
@@ -0,0 +1,636 @@
+# TAOS SQL
+
+TDengine provides a SQL like query language to insert or query data. You can execute the SQL statements through the TDengine Shell, or through C/C++, Java(JDBC), Python, Restful, Go, and Node.js APIs to interact with the `taosd` service.
+
+Before reading through, please have a look at the conventions used for syntax descriptions here in this documentation.
+
+* Squared brackets ("[]") indicate optional arguments or clauses
+* Curly braces ("{}") indicate that one member from a set of choices in the braces must be chosen
+* A single verticle line ("|") works a separator for multiple optional args or clauses
+* Dots ("…") means repeating for as many times
+
+## Data Types
+
+### Timestamp
+
+The timestamp is the most important data type in TDengine. The first column of each table must be  **`TIMESTAMP`** type, but other columns can also be  **`TIMESTAMP`** type. The following rules for timestamp: 
+
+* String Format: `'YYYY-MM-DD HH:mm:ss.MS'`, which represents the year, month, day, hour, minute and second and milliseconds. For example,`'2017-08-12 18:52:58.128'` is a valid timestamp string. Note: timestamp string must be quoted by either single quote or double quote. 
+
+* Epoch Time:  a timestamp value can also be a long integer representing milliseconds since the epoch. For example, the values in the above example can be represented as an epoch `1502535178128` in milliseconds. Please note the epoch time doesn't need any quotes.
+
+* Internal Function **`NOW`** : this is the current time of the server
+* If timestamp is 0 when inserting a record, timestamp will be set to the current time of the server
+* Arithmetic operations can be applied to timestamp. For example: `now-2h` represents a timestamp which is 2 hours ago from the current server time. Units include `a` (milliseconds),  `s` (seconds),  `m` (minutes),  `h` (hours),  `d` (days),  `w` (weeks), `n` (months), `y` (years). **`NOW`** can be used in either insertions or queries. 
+
+Default time precision is millisecond, you can change it to microseocnd by setting parameter enableMicrosecond in [system configuration](../administrator/#Configuration-on-Server). For epoch time, the long integer shall be microseconds since the epoch. For the above string format, MS shall be six digits. 
+
+### Data Types
+
+The full list of data types is listed below.  For string types of data, we will use ***M*** to indicate the maximum length of that type.
+
+|      |  Data Type  |  Bytes  | Note                                                         |
+| ---- | :---------: | :-----: | ------------------------------------------------------------ |
+| 1    |   TINYINT   |    1    | A nullable integer type with a range of [-127, 127]​          |
+| 2    |  SMALLINT   |    2    | A nullable integer type with a range of [-32767, 32767]​      |
+| 3    |     INT     |    4    | A nullable integer type with a range of [-2^31+1, 2^31-1 ]   |
+| 4    |   BIGINT    |    8    | A nullable integer type with a range of [-2^59, 2^59 ]​       |
+| 5    |    FLOAT    |    4    | A standard nullable float type with 6 -7 significant digits and a range of [-3.4E38, 3.4E38] |
+| 6    |   DOUBLE    |    8    | A standard nullable double float type with 15-16 significant digits and a range of [-1.7E308, 1.7E308]​ |
+| 7    |    BOOL     |    1    | A nullable boolean type, [**`true`**, **`false`**]           |
+| 8    |  TIMESTAMP  |    8    | A nullable timestamp type with the same usage as the primary column timestamp |
+| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 16374, but as maximum row size is 16K bytes, the actual upper limit will generally less than 16374. This type of string only supports ASCii encoded chars. |
+| 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars. The **`NCHAR`** type supports Unicode encoded chars. |
+
+All the keywords in a SQL statement are case-insensitive, but strings values are case-sensitive and must be quoted by a pair of `'` or `"`. To quote a `'` or a `"` , you can use the escape character `\`.
+
+## Database Management
+
+- **Create a Database**
+    ```mysql
+    CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep]
+    ```
+    Option: `KEEP` is used for data retention policy. The data records will be removed once keep-days are passed. There are more parameters related to DB storage, please check [system configuration](../administrator/#Configuration-on-Server).
+
+
+- **Use a Database**
+  
+    ```mysql
+    USE db_name
+    ```
+    Use or switch the current database.
+
+
+- **Drop a Database**
+    ```mysql
+    DROP DATABASE [IF EXISTS] db_name
+    ```
+    Remove a database, all the tables inside the DB will be removed too, be careful.
+
+
+- **List all Databases**
+  
+    ```mysql
+    SHOW DATABASES
+    ```
+
+
+## Table Management
+
+- **Create a Table**
+  
+    ```mysql
+    CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...])
+    ```
+    Note: 
+    
+    1) The first column must be a `timestamp`, and the system will set it as the primary key. 
+    2) The maximum number of columns is 1024, the minimum number of columns is 2.
+    3) The maximum length: database name is 33, table nume is 193, column name is 65.    
+    4) The record size is limited to 16k bytes.
+    5) The total tag length is limited 16k bytes, the maximum number of tags is 128.
+    6) For `binary` or `nchar` data types, the length must be specified. For example, binary(20) means a binary data type with 20 bytes.
+
+
+- **Drop a Table**
+
+    ```mysql
+    DROP TABLE [IF EXISTS] tb_name
+    ```
+
+
+- **List all Tables **
+    ```mysql
+    SHOW TABLES [LIKE tb_name_wildcar]
+    ```
+    It shows all tables in the current DB. 
+    
+    Note: Wildcard characters can be used in the table name to filter tables. 
+    Wildcard characters: 
+    1) ’%’ means 0 to any number of characters. 
+    2）’_’ underscore means exactly one character.
+
+
+- **Print Table Schema**
+
+    ```mysql
+    DESCRIBE tb_name
+    ```
+
+
+- **Add a Column**
+
+    ```mysql
+    ALTER TABLE tb_name ADD COLUMN field_name data_type
+    ```
+
+
+- **Drop a Column**
+
+    ```mysql
+    ALTER TABLE tb_name DROP COLUMN field_name 
+    ```
+    If the table is created via [Super Table](), the schema can only be changed via STable. But for tables not created from STable, you can change their schema directly.
+
+**Tips**: You can apply an operation on a table not in the current DB by concatenating DB name with the character '.', then with the table name. For example, 'demo.tb1' means the operation is applied to table `tb1` in DB `demo` even though `demo` is not the currently selected DB.   
+
+## STable Management
+
+- **Create a STable**
+  
+    ```mysql
+    CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3])
+    ```
+    Create a STable is the same as you create a table, but the tag name and type shoudl be specified
+    Note: 
+    
+    1) The total tag length should not be exceeded 512 bytes
+    2) The type of tag cannot be timestamp
+    3) Tag name should not be the same as other column name or tag name
+    4) Tag name should not be any of Taos key words.
+
+
+- **Drop a STable**
+
+    ```mysql
+    DROP TABLE [IF EXISTS] tb_name
+    ```
+    Delete a STable will also delete all the tables created using the STable.
+
+
+- **List all STables**
+  
+    ```mysql
+    SHOW STABLES [LIKE tb_name_wildcar]
+    ```
+    It shows all STables in the current DB. 
+    
+    Note: Wildcard characters can be used in the table name to filter tables. 
+    Wildcard characters: 
+    1) ’%’ means 0 to any number of characters. 
+    2）’_’ underscore means exactly one character.
+
+
+- **Print STable Schema**
+
+    ```mysql
+    DESCRIBE stb_name
+    ```
+
+- **Add a Column**
+
+    ```mysql
+    ALTER TABLE stb_name ADD COLUMN field_name data_type
+    ```
+
+- **Drop a Column**
+
+    ```mysql
+    ALTER TABLE stb_name DROP COLUMN field_name 
+    ```
+    If the table is created via [Super Table](), the schema can only be changed via STable. But for tables not created from STable, you can change their schema directly.
+
+**Tips**: You can apply an operation on a table not in the current DB by concatenating DB name with the character '.', then with the table name. For example, 'demo.tb1' means the operation is applied to table `tb1` in DB `demo` even though `demo` is not the currently selected DB.
+
+## STable Tag Management
+
+- **Add a tag**
+  
+    ```mysql
+    ALTER TABLE stb_name ADD TAG new_tag_name tag_type
+    ```
+    Add a tag and specify tag type for STable. The total number of tags should be no more than 128.
+
+- **Drop a tag**
+
+    ```mysql
+    ALTER TABLE stb_name DROP TAG tag_name
+    ```
+    Delete a tag from STable, it will also delete the tag for the tables created using STable.
+
+
+- **Modify tag name**
+  
+    ```mysql
+    ALTER TABLE stb_name CHANGE TAG old_tag_name new_tag_name
+    ```
+    Modify a tag name for a STable, it will also modify the tag name for the tables created using STable.
+
+
+- **Change tag value**
+
+    ```mysql
+    ALTER TABLE tb_name SET TAG tag_name=new_tag_value
+    ```
+**Note**: 'Add a tag', 'Drop a tag', 'Modify tag name' used for STable, 'Change tag value' used for table. 
+
+
+## Inserting Records
+
+- **Insert a Record**
+    ```mysql
+    INSERT INTO tb_name VALUES (field_value, ...);
+    ```
+    Insert a data record into table tb_name
+
+
+- **Insert a Record with Selected Columns**
+  
+    ```mysql
+    INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...)
+    ```
+    Insert a data record into table tb_name, with data in selected columns. If a column is not selected, the system will put NULL there. First column (time stamp ) cant not be null, it must be inserted.
+
+
+- **Insert a Batch of Records**
+  
+    ```mysql
+    INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...;
+    ```
+    Insert multiple data records into the table
+
+
+- **Insert a Batch of Records with Selected Columns**
+  
+    ```mysql
+    INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
+    ```
+
+
+- **Insert Records into Multiple Tables**
+  
+    ```mysql
+    INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)... 
+                tb2_name VALUES (field1_value1, ...)(field1_value2, ...)...;
+    ```
+    Insert data records into table tb1_name and tb2_name
+
+
+- **Insert Records into Multiple Tables with Selected Columns**
+  
+    ```mysql
+    INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value1, ...)
+                tb2_name (tb2_field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
+    ```
+
+Note: 1. If the timestamp is 0, the time stamp will be set to the system time on the server.
+      2. The timestamp of the oldest record allowed to be inserted is relative to the current server time, minus the configured keep value (the number of days the data is retained), and the timestamp of the latest record allowed to be inserted is relative to the current server time, plus the configured days value (the time span in which the data file stores data, in days). Both keep and days can be specified when creating the database. The default values are 3650 days and 10 days, respectively.
+ 
+**IMPORT**: If you do want to insert a historical data record into a table, use IMPORT command instead of INSERT. IMPORT has the same syntax as INSERT. 
+
+## Data Query
+
+###Query Syntax:
+
+```mysql
+SELECT select_expr [, select_expr ...]
+    FROM {tb_name_list}
+    [WHERE where_condition]
+    [INTERVAL [interval_offset,] interval_val]
+    [FILL fill_val]
+    [SLIDING fill_val]
+    [GROUP BY col_list]
+    [ORDER BY col_list { DESC | ASC }]
+    [HAVING expr_list]
+    [SLIMIT limit_val [, SOFFSET offset_val]]
+    [LIMIT limit_val [, OFFSET offset_val]]
+    [>> export_file]
+    
+SELECT function_list FROM tb_name
+    [WHERE where_condition]
+    [LIMIT limit [, OFFSET offset]]
+    [>> export_file]
+```
+
+- To query a table, use `*` to select all data from a table; or a specified list of expressions `expr_list` of columns. The SQL expression can contain alias and arithmetic operations between numeric typed columns.
+- For the `WHERE` conditions, use logical operations to filter the timestamp column and all numeric columns, and wild cards to filter the two string typed columns. 
+- Sort the result set by the first column timestamp `_c0` (or directly use the timestamp column name) in either descending or ascending order (by default). "Order by" could not be applied to other columns.
+- Use `LIMIT` and `OFFSET` to control the number of rows returned and the starting position of the retrieved rows. LIMIT/OFFSET is applied after "order by" operations.
+- Export the retrieved result set into a CSV file using `>>`. The target file's full path should be explicitly specified in the statement.
+
+###Supported Operations of Data Filtering:
+
+| Operation | Note                          | Applicable Data Types                 |
+| --------- | ----------------------------- | ------------------------------------- |
+| >         | larger than                   | **`timestamp`** and all numeric types |
+| <         | smaller than                  | **`timestamp`** and all numeric types |
+| >=        | larger than or equal to       | **`timestamp`** and all numeric types |
+| <=        | smaller than or equal to      | **`timestamp`** and all numeric types |
+| =         | equal to                      | all types                             |
+| <>        | not equal to                  | all types                             |
+| %         | match with any char sequences | **`binary`** **`nchar`**              |
+| _         | match with a single char      | **`binary`** **`nchar`**              |
+
+1. For two or more conditions, only AND is supported, OR is not supported yet.
+2. For filtering, only a single range is supported. For example, `value>20 and value<30` is a valid condition, but `value<20 AND value<>5` is an invalid condition
+
+### Some Examples 
+
+- For the examples below, table tb1 is created via the following statements
+
+    ```mysql
+    CREATE TABLE tb1 (ts timestamp, col1 int, col2 float, col3 binary(50))
+    ```
+
+- Query all the records in tb1 in the last hour:
+    ```mysql
+    SELECT * FROM tb1 WHERE ts >= NOW - 1h
+    ```
+
+
+- Query all the records in tb1 between 2018-06-01 08:00:00.000 and 2018-06-02 08:00:00.000, and filter out only the records whose col3 value ends with 'nny', and sort the records by their timestamp in a descending order:
+    ```mysql
+    SELECT * FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND ts <= '2018-06-02 08:00:00.000' AND col3 LIKE '%nny' ORDER BY ts DESC
+    ```
+
+
+- Query the sum of col1 and col2 as alias 'complex_metric', and filter on the timestamp and col2 values. Limit the number of returned rows to 10, and offset the result by 5.
+    ```mysql
+    SELECT (col1 + col2) AS 'complex_metric' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' and col2 > 1.2 LIMIT 10 OFFSET 5
+    ```
+
+
+- Query the number of records in tb1 in the last 10 minutes, whose col2 value is larger than 3.14, and export the result to file `/home/testoutpu.csv`.
+    ```mysql
+    SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv
+    ```
+
+## SQL Functions
+
+### Aggregation Functions
+
+TDengine supports aggregations over numerical values, they are listed below:
+
+- **COUNT**
+
+    ```mysql
+    SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]
+    ```
+    Function: return the number of rows.  
+    Return Data Type: `integer`.  
+    Applicable Data Types: all.  
+    Applied to: table/STable.  
+    Note: 
+    
+    1) `*` can be used for all columns, as long as a column has non-NULL values, it will be counted.
+    
+    2) If it is on a specific column, only rows with non-NULL values will be counted 
+
+
+- **AVG**
+  
+    ```mysql
+    SELECT AVG(field_name) FROM tb_name [WHERE clause]
+    ```
+    Function: return the average value of a specific column.  
+    Return Data Type: `double`.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table/STable. 
+
+
+- **TWA**
+  
+    ```mysql
+    SELECT TWA(field_name) FROM tb_name WHERE clause
+    ```
+    Function: return the time-weighted average value of a specific column  
+    Return Data Type: `double`  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`  
+    Applied to: table/STable
+
+
+- **SUM**
+  
+    ```mysql
+    SELECT SUM(field_name) FROM tb_name [WHERE clause]
+    ```
+    Function: return the sum of a specific column.  
+    Return Data Type: `long integer` or `double`.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table/STable. 
+
+
+- **STDDEV**
+  
+    ```mysql
+    SELECT STDDEV(field_name) FROM tb_name [WHERE clause]
+    ```
+    Function: returns the standard deviation of a specific column.  
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table. 
+
+
+- **LEASTSQUARES**
+    ```mysql
+    SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
+    ```
+    Function: performs a linear fit to the primary timestamp and the specified column. 
+    Return Data Type: return a string of the coefficient and the interception of the fitted line.  
+    Applicable Data Types: all types except timestamp, binary, nchar, bool.  
+    Applied to: table.  
+    Note: The timestmap is taken as the independent variable while the specified column value is taken as the dependent variables.
+
+
+### Selector Functions
+
+- **MIN**
+  
+    ```mysql
+    SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]
+    ```
+    Function: return the minimum value of a specific column.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types except timestamp, binary, nchar, bool.  
+    Applied to: table/STable.  
+
+
+- **MAX**
+    ```mysql
+    SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the maximum value of a specific column.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types except timestamp, binary, nchar, bool.  
+    Applied to: table/STable.    
+
+
+- **FIRST**
+  
+    ```mysql
+    SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the first non-NULL value.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types.  
+    Applied to: table/STable.  
+    Note: To return all columns, use first(*). 
+
+
+- **LAST**
+    ```mysql
+    SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the last non-NULL value.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types.  
+    Applied to: table/STable.  
+    Note: To return all columns, use last(*). 
+
+
+- **TOP**
+  
+    ```mysql
+    SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the `k` largest values.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table/STable.  
+    Note: 
+    1) Valid range of `k`: 1≤*k*≤100
+    2) The associated `timestamp` will be returned too.  
+
+
+- **BOTTOM**
+  
+    ```mysql
+    SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the `k` smallest values.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table/STable.  
+    Note: 
+    1) valid range of `k`: 1≤*k*≤100; 
+    2) The associated `timestamp` will be returned too. 
+
+
+- **PERCENTILE**
+    ```mysql
+    SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: the value of the specified column below which `P` percent of the data points fall.  
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. 
+    Applied to: table/STable.  
+    Note: The range of `P` is `[0, 100]`. When `P=0` , `PERCENTILE` returns the equal value as `MIN`; when `P=100`, `PERCENTILE` returns the equal value as `MAX`. 
+
+- **APERCENTILE**
+    ```mysql
+    SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: the value of the specified column below which `P` percent of the data points fall, it returns approximate value of percentile.
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. 
+    Applied to: table/STable.  
+    Note: The range of `P` is `[0, 100]`. When `P=0` , `APERCENTILE` returns the equal value as `MIN`; when `P=100`, `APERCENTILE` returns the equal value as `MAX`. `APERCENTILE` has a much better performance than `PERCENTILE`.
+
+- **LAST_ROW**
+    ```mysql
+    SELECT LAST_ROW(field_name) FROM { tb_name | stb_name } 
+    ```
+    Function: return the last row.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types.  
+    Applied to: table/STable.  
+    Note: different from last, last_row returns the last row even if it has NULL values. 
+
+
+### Transformation Functions
+
+- **DIFF**
+    ```mysql
+    SELECT DIFF(field_name) FROM tb_name [WHERE clause]
+    ```
+    Function: return the difference between successive values of the specified column.  
+    Return Data Type: the same data type.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table. 
+
+
+- **SPREAD**
+    ```mysql
+    SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: return the difference between the maximum and the mimimum value.  
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.  
+    Applied to: table/STable.  
+    Note: spread gives the range of data variation in a table/supertable; it is equivalent to `MAX()` - `MIN()`
+
+
+- **Arithmetic Operations**
+    ```mysql
+    SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name }  [WHERE clause]
+    ```
+    Function: arithmetic operations on the selected columns.  
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`.
+    Applied to: table/STable.  
+    Note: 1) bracket can be used for operation priority; 2) If a column has NULL value, the result is NULL. 
+
+
+## Downsampling
+
+Time-series data are usually sampled by sensors at a very high frequency, but more often we are only interested in the downsampled, aggregated data of each timeline. TDengine provides a convenient way to downsample the highly frequently sampled data points as well as filling the missing data with a variety of interpolation choices.
+
+```mysql
+SELECT function_list FROM tb_name 
+  [WHERE where_condition]
+  INTERVAL (interval)
+  [FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
+
+SELECT function_list FROM stb_name 
+  [WHERE where_condition]
+  INTERVAL (interval)
+  [FILL ({ VALUE | PREV | NULL | LINEAR})]
+  [GROUP BY tags]
+```
+
+The downsampling time window is defined by `interval`, which is at least 10 milliseconds. The query returns a new series of downsampled data that has a series of fixed timestamps with an increment of `interval`. 
+
+For the time being, only function count, avg, sum, stddev, leastsquares, percentile, min, max, first, last are supported. Functions that may return multiple rows are not supported. 
+
+You can also use `FILL` to interpolate the intervals that don't contain any data.`FILL` currently supports four different interpolation strategies which are listed below:
+
+| Interpolation                     | Usage                                                        |
+| --------------------------------- | ------------------------------------------------------------ |
+| `FILL(VALUE, val1 [, val2, ...])` | Interpolate with specified constants                         |
+| `FILL(PREV)`                      | Interpolate with the value at the previous timestamp         |
+| `FILL(LINEAR)`                    | Linear interpolation with the non-null values at the previous timestamp and at the next timestamp |
+| `FILL(NULL)`                      | Interpolate with **`NULL`** value                            |
+
+ A few downsampling examples:
+
+- Find the number of data points, the maximum value of `col1` and minimum value of `col2` in a tb1 for every 10 minutes in the last 5 hours:
+    ```mysql
+    SELECT COUNT(*), MAX(col1), MIN(col2) FROM tb1 WHERE ts > NOW - 5h INTERVAL (10m)
+    ```
+
+
+- Fill the above downsampling results using constant-value interpolation:
+    ```mysql
+    SELECT COUNT(*), MAX(col1), MIN(col2) FROM tb1 WHERE ts > NOW - 5h INTERVAL(10m) FILL(VALUE, 0, 1, -1)
+    ```
+    Note that the number of constant values in `FILL()` should be equal or fewer than the number of functions in the `SELECT` clause. Exceeding fill constants will be ignored.
+
+
+- Fill the above downsampling results using `PREV` interpolation:
+    ```mysql
+    SELECT COUNT(*), MAX(col1), MIN(col2) FROM tb1 WHERE ts > NOW - 5h INTERVAL(10m) FILL(PREV)
+    ```
+    This will interpolate missing data points with the value at the previous timestamp.
+
+
+- Fill the above downsampling results using `NULL` interpolation:
+    ```mysql
+    SELECT COUNT(*), MAX(col1), MIN(col2) FROM tb1 WHERE ts > NOW - 5h INTERVAL(10m) FILL(NULL)
+    ```
+    Fill **`NULL`** to the interpolated data points.
+
+Notes:
+1. `FILL` can generate tons of interpolated data points if the interval is small and the queried time range is large. So always remember to specify a time range when using interpolation. For each query with interpolation, the result set can not exceed 10,000,000 records.
+2. The result set will always be sorted by time in ascending order.
+3. If the query object is a supertable, then all the functions will be applied to all the tables that qualify the `WHERE` conditions. If the `GROUP BY` clause is also applied, the result set will be sorted ascendingly by time in each single group, otherwise, the result set will be sorted ascendingly by time as a whole.
+
diff --git a/documentation20/administrator-ch.md b/documentation20/administrator-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..f8bb21ddcd1849d85595d7e75924305c77ecf6da
--- /dev/null
+++ b/documentation20/administrator-ch.md
@@ -0,0 +1,301 @@
+#系统管理
+
+## 容量规划
+
+系统的处理能力是有限的，但通过对TDengine配置参数的调整，可以在性能与容量上进行平衡。
+
+**内存需求**
+
+每个Database可以创建固定数目的Vnode，默认与CPU核数相同，可通过maxVgroupsPerDb配置；每个vnode会占用固定大小的内存，与参数blocks和cache有关；每个Table会占用与Tag总大小有关的内存；此外，系统会有一些固定的内存开销。因此，每个Database需要的系统内存可通过如下公式计算：
+
+```
+Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb)
+```
+
+**CPU需求**
+
+每个Vnode可以容纳的最大数据表数，可以通过参数maxTablesPerVnode设置，但每个Vnode的数据表数在1万以下性能最佳。系统总的Vnode数目最好不要超过CPU核数的两倍。
+
+**存储需求**
+
+在绝大多数场景下，TDengine的压缩比不会低于5倍。压缩前的原始数据大小可通过如下方式计算：
+
+```
+Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
+```
+
+用户可以通过参数keep，设置数据在磁盘中的最大保存时长。
+
+## 容错和灾备
+
+**容错**
+
+TDengine支持**WAL**（Write Ahead Log）机制，实现数据的容错能力，保证数据的高可用。
+
+TDengine接收到应用的请求数据包时，先将请求的原始数据包写入数据库日志文件，等数据成功写入数据库数据文件后，再删除相应的WAL。这样保证了TDengine能够在断电等因素导致的服务重启时从数据库日志文件中恢复数据，避免数据的丢失。
+
+涉及的系统配置参数有两个.
+
+walLevel：WAL级别，1：写wal, 但不执行fsync; 2：写wal, 而且执行fsync。
+
+fsync：当walLevel设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。
+
+**灾备**
+
+TDengine的集群通过多个副本的机制，来提供系统的高可靠性，实现灾备能力。
+
+TDengine集群是由mnode负责管理的，为保证mnode的高可靠，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，为了支持高可靠，需要设置大于1。为保证元数据的强一致性，mnode副本之间通过同步方式进行数据复制，保证了元数据的强一致性。
+
+TDengine集群中的时序数据的副本数是与数据库关联的，一个集群里可以有多个数据库，每个数据库可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数。为了支持高可靠，需要设置副本数大于1。
+
+TDengine集群的节点数必须大于等于副本数，否则创建表时将报错。
+
+当TDengine集群中的节点部署在不同的物理机上（比如不同的机架、或不同的IDC），并设置多个副本数时，就实现了异地容灾，从而提供系统的高可靠性，无需再使用其他软件或工具。
+
+## 文件目录结构
+
+安装TDengine后，默认会在操作系统中生成下列目录或文件：
+
+| 目录/文件               | 说明                                              |
+| ---------------------- | :------------------------------------------------|
+| /usr/local/taos/bin | TDengine可执行文件目录。其中的执行文件都会软链接到/usr/bin目录下。 |
+| /usr/local/taos/connector | TDengine各种连接器目录。 |
+| /usr/local/taos/driver | TDengine动态链接库目录。会软链接到/usr/lib目录下。 |
+| /usr/local/taos/examples | TDengine各种语言应用示例目录。 |
+| /usr/local/taos/include | TDengine对外提供的C语言接口的头文件。 |
+| /etc/taos/taos.cfg     | TDengine默认[配置文件]                            |
+| /var/lib/taos          | TDengine默认数据文件目录,可通过[配置文件]修改位置.    |
+| /var/log/taos          | TDengine默认日志文件目录,可通过[配置文件]修改位置     |
+
+**可执行文件**
+
+TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括：
+
+- _taosd_：TDengine服务端可执行文件
+- _taos_： TDengine Shell可执行文件
+- _taosdump_：数据导入导出工具
+- remove.sh：卸载TDengine的脚本, 请谨慎执行，链接到/usr/bin目录下的rmtaos命令。会删除TDengine的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos。
+
+您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
+
+## 服务端配置
+
+TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录，可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。
+
+下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节。**注意：配置修改后，需要重启*taosd*服务才能生效。**
+
+- first: taosd启动时，主动连接的集群中第一个dnode的end point, 缺省值为 localhost:6030。
+- second: taosd启动时，如果first连接不上，尝试连接集群中第二个dnode的end point, 缺省值为空。
+- fqdn：数据节点的FQDN。如果为空，将自动获取操作系统配置的第一个, 缺省值为空。
+- serverPort：taosd启动后，对外服务的端口号，默认值为6030。
+- httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。
+- dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。
+- logDir：日志文件目录，客户端和服务器的运行日志文件将写入该目录。默认值：/var/log/taos。
+- arbitrator：系统中裁决器的end point, 缺省值为空。
+- role：dnode的可选角色。0-any; 既可作为mnode，也可分配vnode；1-mgmt;只能作为mnode，不能分配vnode；2-dnode;不能作为mnode，只能分配vnode
+- debugFlag：运行日志开关。131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志）。默认值：131或135（不同模块有不同的默认值）。
+- numOfLogLines：单个日志文件允许的最大行数。默认值：10,000,000行。
+- maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
+- maxBinaryDisplayWidth：Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn动态修改此选项。
+
+不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数：
+
+- days：一个数据文件存储数据的时间跨度，单位为天，默认值：10。
+- keep：数据库中数据保留的天数，单位为天，默认值：3650。
+- minRows: 文件块中记录的最小条数，单位为条，默认值：100。
+- maxRows: 文件块中记录的最大条数，单位为条，默认值：4096。
+- comp: 文件压缩标志位，0：关闭，1:一阶段压缩，2:两阶段压缩。默认值：2。
+- walLevel：WAL级别。1：写wal, 但不执行fsync; 2：写wal, 而且执行fsync。默认值：1。
+- fsync：当wal设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。单位为毫秒，默认值：3000。
+- cache: 内存块的大小，单位为兆字节（MB），默认值：16。
+- blocks: 每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。
+- replica：副本个数，取值范围：1-3。单位为个，默认值：1
+- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms
+
+对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： 
+
+```
+ create database demo days 10 cache 32 blocks 8 replica 3
+```
+
+该SQL创建了一个库demo, 每个数据文件存储10天数据，内存块为32兆字节，每个VNODE占用8个内存块，副本数为3，而其他参数与系统配置完全一致。
+
+TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数必须与已有集群的配置相同，否则不能成功加入到集群中。会进行校验的参数如下：
+
+- numOfMnodes：系统中管理节点个数。默认值：3。
+- balance：是否启动负载均衡。0：否，1：是。默认值：1。
+- mnodeEqualVnodeNum: 一个mnode等同于vnode消耗的个数。默认值：4。
+- offlineThreshold: dnode离线阈值，超过该时间将导致该dnode从集群中删除。单位为秒，默认值：86400*10（即10天）。
+- statusInterval: dnode向mnode报告状态时长。单位为秒，默认值：1。
+- maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。
+- maxVgroupsPerDb: 每个数据库中能够使用的最大vnode个数。
+- arbitrator: 系统中裁决器的end point。
+- timezone：时区。从系统中动态获取当前的时区设置。
+- locale：系统区位信息及编码格式。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。
+- charset：字符集编码。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。
+
+## 客户端配置 
+
+TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
+
+客户端配置参数列表及解释
+
+- first: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。
+- second: taos启动时，如果first连接不上，尝试连接集群中第二个taosd实例的end point, 缺省值为空。
+- charset：字符集编码。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。
+- locale：系统区位信息及编码格式。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。
+
+日志的配置参数，与server的配置参数完全一样。
+
+启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。
+
+## 用户管理
+
+系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下：
+
+```
+CREATE USER user_name PASS ‘password’
+```
+
+创建用户，并指定用户名和密码，密码需要用单引号引起来
+
+```
+DROP USER user_name
+```
+
+删除用户，限root用户使用
+
+```
+ALTER USER user_name PASS ‘password’  
+```
+
+修改用户密码, 为避免被转换为小写，密码需要用单引号引用
+
+```
+SHOW USERS
+```
+
+显示所有用户
+
+## 数据导入
+
+TDengine提供多种方便的数据导入功能，一种按脚本文件导入，一种按数据文件导入，一种是taosdump工具导入本身导出的文件。
+
+**按脚本文件导入**
+
+TDengine的shell支持source filename命令，用于批量运行文件中的SQL语句。用户可将建库、建表、写数据等SQL命令写在同一个文件中，每条命令单独一行，在shell中运行source命令，即可按顺序批量运行文件中的SQL语句。以‘#’开头的SQL语句被认为是注释，shell将自动忽略。
+
+**按数据文件导入**
+
+TDengine也支持在shell对已存在的表从CSV文件中进行数据导入。CSV文件只属于一张表且CSV文件中的数据格式需与要导入表的结构相同, 在导入的时候，其语法如下
+
+```mysql
+insert into tb1 file 'path/data.csv'
+```
+注意：如果CSV文件首行存在描述信息，请手动删除后再导入
+
+例如，现在存在一个子表d1001, 其表结构如下：
+
+```mysql
+taos> DESCRIBE d1001
+             Field              |        Type        |   Length    |    Note    |
+=================================================================================
+ ts                             | TIMESTAMP          |           8 |            |
+ current                        | FLOAT              |           4 |            |
+ voltage                        | INT                |           4 |            |
+ phase                          | FLOAT              |           4 |            |
+ location                       | BINARY             |          64 | TAG        |
+ groupid                        | INT                |           4 | TAG        |
+```
+要导入的data.csv的格式如下：
+
+```csv
+'2018-10-04 06:38:05.000',10.30000,219,0.31000
+'2018-10-05 06:38:15.000',12.60000,218,0.33000
+'2018-10-06 06:38:16.800',13.30000,221,0.32000
+'2018-10-07 06:38:05.000',13.30000,219,0.33000
+'2018-10-08 06:38:05.000',14.30000,219,0.34000
+'2018-10-09 06:38:05.000',15.30000,219,0.35000
+'2018-10-10 06:38:05.000',16.30000,219,0.31000
+'2018-10-11 06:38:05.000',17.30000,219,0.32000
+'2018-10-12 06:38:05.000',18.30000,219,0.31000
+```
+那么可以用如下命令导入数据
+
+```
+taos> insert into d1001 file '~/data.csv';
+Query OK, 9 row(s) affected (0.004763s)
+```
+
+
+**taosdump工具导入**
+
+TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：
+
+[TDengine DUMP工具使用指南]: https://www.taosdata.com/blog/2020/03/09/1334.html
+
+## 数据导出
+
+为方便数据导出，TDengine提供了两种导出方式，分别是按表导出和用taosdump导出。
+
+**按表导出CSV文件**
+
+如果用户需要导出一个表或一个STable中的数据，可在shell中运行
+
+```
+select * from <tb_name> >> data.csv
+```
+
+这样，表tb_name中的数据就会按照CSV格式导出到文件data.csv中。
+
+**用taosdump导出数据**
+
+TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见博客：
+
+[TDengine DUMP工具使用指南]: https://www.taosdata.com/blog/2020/03/09/1334.html
+
+## 系统连接、任务查询管理
+
+系统管理员可以从CLI查询系统的连接、正在进行的查询、流式计算，并且可以关闭连接、停止正在进行的查询和流式计算。CLI里SQL语法如下：
+
+```
+SHOW CONNECTIONS
+```
+
+显示数据库的连接，其中一列显示ip:port, 为连接的IP地址和端口号。
+
+```
+KILL CONNECTION <connection-id>
+```
+
+强制关闭数据库连接，其中的connection-id是SHOW CONNECTIONS中显示的第一列的数字。
+
+```
+SHOW QUERIES
+```
+
+显示数据查询，其中第一列显示的以冒号隔开的两个数字为query-id，为发起该query应用连接的connection-id和查询次数。
+
+```
+KILL QUERY <query-id>
+```
+
+强制关闭数据查询，其中query-id是SHOW QUERIES中显示的 connection-id:query-no字串，如“105:2”，拷贝粘贴即可。
+
+```
+SHOW STREAMS
+```
+
+显示流式计算，其中第一列显示的以冒号隔开的两个数字为stream-id, 为启动该stream应用连接的connection-id和发起stream的次数。
+
+```
+KILL STREAM <stream-id>
+```
+
+强制关闭流式计算，其中的中stream-id是SHOW STREAMS中显示的connection-id:stream-no字串，如103:2，拷贝粘贴即可。
+
+## 系统监控
+
+TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
+
+这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
\ No newline at end of file
diff --git a/documentation20/administrator.md b/documentation20/administrator.md
new file mode 100644
index 0000000000000000000000000000000000000000..e118be525d117f0dc85f91c9fc3b66bf676ef30a
--- /dev/null
+++ b/documentation20/administrator.md
@@ -0,0 +1,231 @@
+#Administrator
+
+## Directory and Files
+
+After TDengine is installed, by default, the following directories will be created:
+
+| Directory/File         | Description                     |
+| ---------------------- | :------------------------------ |
+| /etc/taos/taos.cfg     | TDengine configuration file     |
+| /usr/local/taos/driver | TDengine dynamic link library   |
+| /var/lib/taos          | TDengine default data directory |
+| /var/log/taos          | TDengine default log directory  |
+| /usr/local/taos/bin.   | TDengine executables            |
+
+### Executables
+
+All TDengine executables are located at _/usr/local/taos/bin_ , including:
+
+- `taosd`：TDengine server 
+- `taos`： TDengine Shell, the command line interface.
+- `taosdump`：TDengine data export tool
+- `rmtaos`： a script to uninstall TDengine
+
+You can change the data directory and log directory setting through the system configuration file
+
+## Configuration on Server
+
+`taosd` is running on the server side, you can change the system configuration file taos.cfg to customize its behavior. By default, taos.cfg is located at /etc/taos, but you can specify the path to configuration file via the command line parameter -c. For example: `taosd -c /home/user` means the configuration file will be read from directory /home/user.
+
+This section lists only the most important configuration parameters. Please check taos.cfg to find all the configurable parameters. **Note: to make your new configurations work, you have to restart taosd after you change taos.cfg**.
+
+- mgmtShellPort: TCP and UDP port between client and TDengine mgmt (default: 6030). Note: 5 successive UDP ports (6030-6034) starting from this number will be used.
+- vnodeShellPort: TCP and UDP port between client and TDengine vnode (default: 6035). Note: 5 successive UDP ports (6035-6039) starting from this number will be used.
+- httpPort: TCP port for RESTful service (default: 6020)
+- dataDir: data directory, default is /var/lib/taos
+- maxUsers: maximum number of users allowed
+- maxDbs: maximum number of databases allowed
+- maxTables: maximum number of tables allowed
+- enableMonitor: turn on/off system monitoring, 0: off, 1: on
+- logDir: log directory, default is /var/log/taos
+- numOfLogLines: maximum number of lines in the log file
+- debugFlag: log level, 131: only error and warnings, 135: all
+
+In different scenarios, data characteristics are different. For example, the retention policy, data sampling period, record size, the number of devices, and data compression may be different. To gain the best performance, you can change the following configurations related to storage:  
+
+- days: number of days to cover for a data file
+- keep: number of days to keep the data
+- rows: number of rows of records in a block in data file.
+- comp: compression algorithm, 0: off, 1: standard; 2: maximum compression
+- ctime: period (seconds) to flush data to disk
+- clog: flag to turn on/off Write Ahead Log, 0: off, 1: on 
+- tables: maximum number of tables allowed in a vnode
+- cache: cache block size (bytes)
+- tblocks: maximum number of cache blocks for a table
+- abloks: average number of cache blocks for a table 
+- precision: timestamp precision, us: microsecond ms: millisecond, default is ms
+
+For an application, there may be multiple data scenarios. The best design is to put all data with the same characteristics into one database. One application may have multiple databases, and every database has its own configuration to maximize the system performance. You can specify the above configurations related to storage when you create a database. For example:  
+
+```mysql
+CREATE DATABASE demo DAYS 10 CACHE 16000 ROWS 2000 
+```
+
+The above SQL statement will create a database demo, with 10 days for each data file, 16000 bytes for a cache block, and 2000 rows in a file block.
+
+The configuration provided when creating a database will overwrite the configuration in taos.cfg. 
+
+## Configuration on Client 
+
+*taos* is the TDengine shell and is a client that connects to taosd. TDengine uses the same configuration file taos.cfg for the client, with default location at /etc/taos. You can change it by specifying command line parameter -c when you run taos. For example, *taos -c /home/user*, it will read the configuration file taos.cfg from directory /home/user.
+
+The parameters related to client configuration are listed below: 
+
+- masterIP: IP address of TDengine server
+- charset: character set, default is the system . For data type nchar, TDengine uses unicode to store the data. Thus, the client needs to tell its character set.
+- locale: system language setting
+- defaultUser: default login user, default is root
+- defaultPass: default password, default is taosdata
+
+For TCP/UDP port, and system debug/log configuration, it is the same as the server side.
+
+For server IP, user name, password, you can always specify them in the command line when you run taos. If they are not specified, they will be read from the taos.cfg
+
+## User Management
+
+System administrator (user root) can add, remove a user, or change the password from the TDengine shell. Commands are listed below:
+
+Create a user, password shall be quoted with the single quote.
+
+```mysql
+CREATE USER user_name PASS ‘password’
+```
+
+Remove a user
+
+```mysql
+DROP USER user_name
+```
+
+Change the password for a user
+
+```mysql
+ALTER USER user_name PASS ‘password’  
+```
+
+List all users
+
+```mysql
+SHOW USERS
+```
+
+## Import Data
+
+Inside the TDengine shell, you can import data into TDengine from either a script or CSV file
+
+**Import from Script**
+
+```
+source <filename>
+```
+
+Inside the file, you can put all SQL statements there. Each SQL statement has a line. If a line starts with "#", it means comments, it will be skipped. The system will execute the SQL statements line by line automatically until the ends 
+
+**Import from CSV**
+
+```mysql
+insert into tb1 file 'path/data.csv'
+```
+
+CSV file contains records for only one table, and the data structure shall be the same as the defined schema for the table. The header of CSV file shall be removed. 
+
+For example, the following is a sub-table d1001:
+
+```mysql
+taos> DESCRIBE d1001
+             Field              |        Type        |   Length    |    Note    |
+=================================================================================
+ ts                             | TIMESTAMP          |           8 |            |
+ current                        | FLOAT              |           4 |            |
+ voltage                        | INT                |           4 |            |
+ phase                          | FLOAT              |           4 |            |
+ location                       | BINARY             |          64 | TAG        |
+ groupid                        | INT                |           4 | TAG        |
+```
+The data format in data.csv like this：
+
+```csv
+'2018-10-04 06:38:05.000',10.30000,219,0.31000
+'2018-10-05 06:38:15.000',12.60000,218,0.33000
+'2018-10-06 06:38:16.800',13.30000,221,0.32000
+'2018-10-07 06:38:05.000',13.30000,219,0.33000
+'2018-10-08 06:38:05.000',14.30000,219,0.34000
+'2018-10-09 06:38:05.000',15.30000,219,0.35000
+'2018-10-10 06:38:05.000',16.30000,219,0.31000
+'2018-10-11 06:38:05.000',17.30000,219,0.32000
+'2018-10-12 06:38:05.000',18.30000,219,0.31000
+```
+then data can be imported into database by this cmd:
+
+```
+taos> insert into d1001 file '~/data.csv';
+Query OK, 9 row(s) affected (0.004763s)
+```
+## Export Data
+
+You can export data either from TDengine shell or from tool taosdump.
+
+**Export from TDengine Shell**
+
+```mysql
+select * from <tb_name> >> data.csv
+```
+
+The above SQL statement will dump the query result set into data.csv file.  
+
+**Export Using taosdump**
+
+TDengine provides a data dumping tool taosdump. You can choose to dump a database, a table, all data or data only a time range, even only the metadata. For example:
+
+- Export one or more tables in a DB: taosdump [OPTION…] dbname tbname …
+- Export one or more DBs: taosdump [OPTION…] --databases dbname…
+- Export all DBs (excluding system DB): taosdump [OPTION…] --all-databases
+
+run *taosdump —help* to get a full list of the options
+
+## Management of Connections, Streams, Queries 
+
+The system administrator can check, kill the ongoing connections, streams, or queries. 
+
+```
+SHOW CONNECTIONS
+```
+
+It lists all connections. The first column shows connection-id from the client. 
+
+```
+KILL CONNECTION <connection-id>
+```
+
+It kills the connection, where connection-id is the number of the first column showed by "SHOW CONNECTIONS".
+
+```
+SHOW QUERIES
+```
+
+It shows the ongoing queries. The first column shows the connection-id:query-no, where connection-id is the connection from the client, and id assigned by the system
+
+```
+KILL QUERY <query-id>
+```
+
+It kills the query, where query-id is the connection-id:query-no showed by "SHOW QUERIES". You can copy and paste it.
+
+```
+SHOW STREAMS
+```
+
+It shows the continuous queries. The first column shows the connection-id:stream-no, where connection-id is the connection from the client, and id assigned by the system.
+
+```
+KILL STREAM <stream-id>
+```
+
+It kills the continuous query, where stream-id is the connection-id:stream-no showed by "SHOW STREAMS". You can copy and paste it.
+
+## System Monitor
+
+TDengine runs a system monitor in the background. Once it is started, it will create a database sys automatically. System monitor collects the metric like CPU, memory, network, disk, number of requests periodically, and writes them into database sys. Also, TDengine will log all important actions, like login, logout, create database, drop database and so on, and write them into database sys. 
+
+You can check all the saved monitor information from database sys. By default, system monitor is turned on. But you can turn it off by changing the parameter in the configuration file.
+
diff --git a/documentation20/advanced features-ch.md b/documentation20/advanced features-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..ccde21bfffa129e6e15a668ca557c46fd7625c9a
--- /dev/null
+++ b/documentation20/advanced features-ch.md	
@@ -0,0 +1,314 @@
+# 高级功能
+
+## 连续查询(Continuous Query)
+
+连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。
+针对库中的表或超级表，TDengine可提供定期自动执行的连续查询，
+用户可让TDengine推送查询的结果，也可以将结果再写回到TDengine中。
+每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。
+在定义连续查询的时候需要指定时间窗口（time window, 参数interval）大小和每次前向增量时间（forward sliding times, 参数sliding）。
+
+TDengine的连续查询采用时间驱动模式，可以直接使用TAOS SQL进行定义，不需要额外的操作。
+使用连续查询，可以方便快捷地按照时间窗口生成结果，从而对原始采集数据进行降采样（down sampling）。
+用户通过TAOS SQL定义连续查询以后，TDengine自动在最后的一个完整的时间周期末端拉起查询，
+并将计算获得的结果推送给用户或者写回TDengine。
+
+TDengine提供的连续查询与普通流计算中的时间窗口计算具有以下区别：
+
+- 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。
+例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。
+
+- 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算，
+也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。
+
+- 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。
+如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。
+如果使用写回模式，TDengine可确保数据写回的有效性和连续性。
+
+### 使用连续查询
+
+下面以智能电表场景为例介绍连续查询的具体使用方法。假设我们通过下列SQL语句创建了超级表和子表：
+
+```sql
+create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupdId int);
+create table D1001 using meters tags ("Beijing.Chaoyang", 2);
+create table D1002 using meters tags ("Beijing.Haidian", 2);
+...
+```
+
+我们已经知道，可以通过下面这条SQL语句以一分钟为时间窗口、30秒为前向增量统计这些电表的平均电压。
+
+```sql
+select avg(voltage) from meters interval(1m) sliding(30s)
+```
+
+每次执行这条语句，都会重新计算所有数据。 
+如果需要每隔30秒执行一次来增量计算最近一分钟的数据，
+可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行：
+
+```sql
+select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s)
+```
+
+这样做没有问题，但TDengine提供了更简单的方法，
+只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如：
+
+```sql
+create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s)
+```
+
+会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句，
+并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如：
+
+```shell
+taos> select * from avg_vol;
+            ts           |        avg_voltage_    |
+===================================================
+ 2020-07-29 13:37:30.000 |            222.0000000 |
+ 2020-07-29 13:38:00.000 |            221.3500000 |
+ 2020-07-29 13:38:30.000 |            220.1700000 |
+ 2020-07-29 13:39:00.000 |            223.0800000 |
+```
+
+需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。
+
+
+此外，TDengine还支持用户指定连续查询的起止时间。
+如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始；
+如果没有输入结束时间，连续查询将永久运行；
+如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。
+比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。
+
+```sql
+create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s)
+```
+
+需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。 
+另外，为了尽量避免原始数据延迟写入导致的问题，TDengine中连续查询的计算有一定的延迟。
+也就是说，一个时间窗口过去后，TDengine并不会立即计算这个窗口的数据，
+所以要稍等一会（一般不会超过1分钟）才能查到计算结果。
+
+
+### 管理连续查询
+
+用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询，
+并可以通过 `kill stream` 命令杀掉对应的连续查询。
+后续版本会提供更细粒度和便捷的连续查询管理命令。
+
+
+## 数据订阅(Publisher/Subscriber)
+
+基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致，
+均可视为系统中插入一条带时间戳的新记录。
+同时，TDengine在内部严格按照数据时间序列单调递增的方式保存数据。
+本质上来说，TDengine中里每一张表均可视为一个标准的消息队列。
+
+TDengine内嵌支持轻量级的消息订阅与推送服务。
+使用系统提供的API，用户可使用普通查询语句订阅数据库中的一张或多张表。
+订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达，
+有新的记录到达就会将结果反馈到客户。
+
+TDengine的订阅与推送服务的状态是客户端维持，TDengine服务器并不维持。
+因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。
+
+TDengine的API中，与订阅相关的主要有以下三个：
+
+```c
+taos_subscribe
+taos_consume
+taos_unsubscribe
+```
+
+这些API的文档请见 [C/C++ 数据订阅接口](TODO: update link)，
+下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”），
+完整的示例代码可以在[这里](TODO: update link)找到。
+
+如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法：
+一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据：
+
+```sql
+select * from D1001 where ts > {last_timestamp1} and current > 10;
+select * from D1002 where ts > {last_timestamp2} and current > 10;
+...
+```
+
+这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响，
+当电表数增长到一定的程度，系统就无法承受了。
+
+另一种方法是对超级表进行查询。这样，无论有多少电表，都只需一次查询：
+
+```sql
+select * from meters where ts > {last_timestamp} and current > 10;
+```
+
+但是，如何选择 `last_timestamp` 就成了一个新的问题。
+因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大；
+另一方面，不同电表的数据到达TDengine的时间也会有差异。
+所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`，
+就可能重复读入其它电表的数据；
+如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。
+
+TDengine的订阅功能为上面这个问题提供了一个彻底的解决方案。
+
+首先是使用`taos_subscribe`创建订阅：
+
+```c
+TAOS_SUB* tsub = NULL;
+if (async) {
+　　// create an asynchronized subscription, the callback function will be called every 1s
+　　tsub = taos_subscribe(taos, restart, topic, sql, subscribe_callback, &blockFetch, 1000);
+} else {
+　　// create an synchronized subscription, need to call 'taos_consume' manually
+　　tsub = taos_subscribe(taos, restart, topic, sql, NULL, NULL, 0);
+}
+```
+
+TDengine中的订阅既可以是同步的，也可以是异步的，
+上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。
+这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据，
+而异步则由API在内部的另一个线程中调用`taos_consume`，
+然后把拉取到的数据交给回调函数`subscribe_callback`去处理。
+
+参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。
+但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误，
+因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。
+
+参数`sql`是查询语句，可以在其中使用where子句指定过滤条件。
+在我们的例子中，如果只想订阅电流超过10A时的数据，可以这样写：
+
+```sql
+select * from meters where current > 10;
+```
+
+注意，这里没有指定起始时间，所以会读到所有时间的数据。
+如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件：
+
+```sql
+select * from meters where ts > now - 1d and current > 10;
+```
+
+订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的，
+所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。
+
+如果名`topic`的订阅不存在，参数`restart`没有意义；
+但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时，
+`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。
+本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。
+但如果这个订阅之前就存在了，并且已经读取了一部分数据，
+且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。
+
+`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。
+在同步模式下，如过前后两次调用`taos_consume`的时间间隔小于此时间，
+`taos_consume`会阻塞，直到间隔超过此时间。
+异步模式下，这个时间是两次调用回调函数的最小时间间隔。
+
+`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数，
+订阅API不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。
+
+订阅创建以后，就可以消费其数据了，同步模式下，示例代码是下面的 else 部分：
+
+```c
+if (async) {
+　　getchar();
+} else while(1) {
+　　TAOS_RES* res = taos_consume(tsub);
+　　if (res == NULL) {
+　　　　printf("failed to consume data.");
+　　　　break;
+　　} else {
+　　　　print_result(res, blockFetch);
+　　　　getchar();
+　　}
+}
+```
+
+这里是一个 **while** 循环，用户每按一次回车键就调用一次`taos_consume`，
+而`taos_consume`的返回值是查询到的结果集，与`taos_use_result`完全相同，
+例子中使用这个结果集的代码是函数`print_result`：
+
+```c
+void print_result(TAOS_RES* res, int blockFetch) {
+　　TAOS_ROW row = NULL;
+　　int num_fields = taos_num_fields(res);
+　　TAOS_FIELD* fields = taos_fetch_fields(res);
+　　int nRows = 0;
+　　if (blockFetch) {
+　　　　nRows = taos_fetch_block(res, &row);
+　　　　for (int i = 0; i < nRows; i++) {
+　　　　　　char temp[256];
+　　　　　　taos_print_row(temp, row + i, fields, num_fields);
+　　　　　　puts(temp);
+　　　　}
+　　} else {
+　　　　while ((row = taos_fetch_row(res))) {
+　　　　　　char temp[256];
+　　　　　　taos_print_row(temp, row, fields, num_fields);puts(temp);
+　　　　　　nRows++;
+　　　　}
+　　}
+　　printf("%d rows consumed.\n", nRows);
+}
+```
+
+其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。
+而异步模式下，消费订阅到的数据则显得更为简单：
+
+```c
+void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
+　　print_result(res, *(int*)param);
+}
+```
+
+当要结束一次数据订阅时，需要调用`taos_unsubscribe`：
+
+```c
+taos_unsubscribe(tsub, keep);
+```
+
+其第二个参数，用于决定是否在客户端保留订阅的进度信息。
+如果这个参数是**false**（**0**），那无论下次调用`taos_subscribe`的时的`restart`参数是什么，
+订阅都只能重新开始。
+另外，进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下，
+每个订阅有一个与其`topic`同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。
+
+代码介绍完毕，我们来看一下实际的运行效果。假设：
+
+* 示例代码已经下载到本地
+* TDengine 也已经在同一台机器上安装好
+* 示例所需的数据库、超级表、子表已经全部创建好
+
+则可以在示例代码所在目录执行以下命令来编译并启动示例程序：
+
+```shell
+$ make
+$ ./subscribe -sql='select * from meters where current > 10;'
+```
+
+示例程序启动后，打开另一个终端窗口，启动 TDengine 的 shell 向 **D1001** 插入一条电流为 12A 的数据：
+
+```shell
+$ taos
+> use test;
+> insert into D1001 values(now, 12, 220, 1);
+```
+
+这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。
+您可以继续插入一些数据观察示例程序的输出。
+
+
+## 缓存 (Cache)
+TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Use，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。
+
+TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署额外的缓存系统，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。
+
+TDengine分配固定大小的内存空间作为缓存空间，缓存空间可根据应用的需求和硬件资源配置。通过适当的设置缓存空间，TDengine可以提供极高性能的写入和查询的支持。TDengine中每个虚拟节点（virtual node）创建时分配独立的缓存池。每个虚拟节点管理自己的缓存池，不同虚拟节点间不共享缓存池。每个虚拟节点内部所属的全部表共享该虚拟节点的缓存池。
+
+TDengine将内存池按块划分进行管理，数据在内存块里按照列式存储。一个vnode的内存池是在vnode创建时按块分配好的，而且每个内存块按照先进先出的原则进行管理。一张表所需要的内存块是从vnode的内存池中进行分配的，块的大小由系统配置参数cache决定。每张表最大内存块的数目由配置参数tblocks决定，每张表平均的内存块的个数由配置参数ablocks决定。因此对于一个vnode, 总的内存大小为: `cache * ablocks * tables`。内存块参数cache不宜过小，一个cache block需要能存储至少几十条以上记录，才会有效率。参数ablocks最小为2，保证每张表平均至少能分配两个内存块。
+
+你可以通过函数last_row快速获取一张表或一张超级表的最后一条记录，这样很便于在大屏显示各设备的实时状态或采集值。例如：
+
+```mysql
+select last_row(degree) from thermometer where location='beijing';
+```
+
+该SQL语句将获取所有位于北京的传感器最后记录的温度值。
diff --git a/documentation20/advanced features.md b/documentation20/advanced features.md
new file mode 100644
index 0000000000000000000000000000000000000000..05f6da5ed9e94684e7d39f086d850805eb966b85
--- /dev/null
+++ b/documentation20/advanced features.md	
@@ -0,0 +1,80 @@
+#Advanced Features
+
+##Continuous Query
+Continuous Query is a query executed by TDengine periodically with a sliding window, it is a simplified stream computing driven by timers, not by events. Continuous query can be applied to a table or a STable, and the result set can be passed to the application directly via call back function, or written into a new table in TDengine. The query is always executed on a specified time window (window size is specified by parameter interval), and this window slides forward while time flows (the sliding period is specified by parameter sliding).    
+
+Continuous query is defined by TAOS SQL, there is nothing special. One of the best applications is downsampling. Once it is defined, at the end of each cycle, the system will execute the query, pass the result to the application or write it to a database. 
+
+If historical data pints are inserted into the stream, the query won't be re-executed, and the result set won't be updated. If the result set is passed to the application, the application needs to keep the status of continuous query, the server won't maintain it. If application re-starts, it needs to decide the time where the stream computing shall be started.
+
+####How to use continuous query
+
+ 
+
+  - Pass result set to application
+
+    Application shall use API taos_stream (details in connector section) to start the stream computing. Inside the API, the SQL syntax is:
+  
+    ```sql
+    SELECT aggregation FROM [table_name | stable_name] 
+    INTERVAL(window_size) SLIDING(period)
+    ```
+  
+    where the new keyword INTERVAL specifies the window size, and SLIDING specifies the sliding period. If parameter sliding is not specified, the sliding period will be the same as window size. The minimum window size is 10ms. The sliding period shall not be larger than the window size. If you set a value larger than the window size, the system will adjust it to window size automatically.
+  
+    For example:
+  
+    ```sql
+    SELECT COUNT(*) FROM FOO_TABLE 
+    INTERVAL(1M) SLIDING(30S)
+    ```
+  
+    The above SQL statement will count the number of records for the past 1-minute window every 30 seconds.
+  
+  - Save the result into a database
+  
+    If you want to save the result set of stream computing into a new table, the SQL shall be: 
+  
+    ```sql
+    CREATE TABLE table_name AS 
+    SELECT aggregation from [table_name | stable_name]  
+    INTERVAL(window_size) SLIDING(period)
+    ```
+  
+    Also, you can set the time range to execute the continuous query. If no range is specified, the continuous query will be executed forever. For example, the following continuous query will be executed from now and will stop in one hour.
+  
+    ```sql
+    CREATE TABLE QUERY_RES AS 
+    SELECT COUNT(*) FROM FOO_TABLE 
+    WHERE TS > NOW AND TS <= NOW + 1H 
+    INTERVAL(1M) SLIDING(30S) 
+    ```
+
+###Manage the Continuous Query
+
+Inside TDengine shell, you can use the command "show streams" to list the ongoing continuous queries, the command "kill stream" to kill a specific continuous query. 
+
+If you drop a table generated by the continuous query, the query will be removed too.
+
+##Publisher/Subscriber
+
+Time series data is a sequence of data points over time. Inside a table, the data points are stored in order of timestamp. Also, there is a data retention policy, the data points will be removed once their lifetime is passed. From another view, a table in DTengine is just a standard message queue. 
+
+To reduce the development complexity and improve data consistency, TDengine provides the pub/sub functionality. To publish a message, you simply insert a record into a table. Compared with popular messaging tool Kafka, you subscribe to a table or a SQL query statement, instead of a topic. Once new data points arrive, TDengine will notify the application. The process is just like Kafka.  
+
+The detailed API will be introduced in the [connectors](https://www.taosdata.com/en/documentation/advanced-features/) section. 
+
+##Caching
+TDengine allocates a fixed-size buffer in memory, the newly arrived data will be written into the buffer first. Every device or table gets one or more memory blocks. For typical IoT scenarios, the hot data shall always be newly arrived data, they are more important for timely analysis. Based on this observation, TDengine manages the cache blocks in First-In-First-Out strategy. If no enough space in the buffer, the oldest data will be saved into hard disk first, then be overwritten by newly arrived data. TDengine also guarantees every device can keep at least one block of data in the buffer. 
+
+By this design, the application can retrieve the latest data from each device super-fast, since they are all available in memory. You can use last or last_row function to return the last data record. If the super table is used, it can be used to return the last data records of all or a subset of devices. For example, to retrieve the latest temperature from thermometers in located Beijing, execute the following SQL  
+
+```mysql
+select last(*) from thermometers where location=’beijing’
+```
+
+By this design, caching tool, like Redis, is not needed in the system. It will reduce the complexity of the system. 
+
+TDengine creates one or more virtual nodes(vnode) in each data node. Each vnode contains data for multiple tables and has its own buffer. The buffer of a vnode is fully separated from the buffer of another vnode, not shared. But the tables in a vnode share the same buffer.  
+
+System configuration parameter cacheBlockSize configures the cache block size in bytes, and another parameter cacheNumOfBlocks configures the number of cache blocks. The total memory for the buffer of a vnode is $cacheBlockSize \times cacheNumOfBlocks$. Another system parameter numOfBlocksPerMeter configures the maximum number of cache blocks a table can use. When you create a database, you can specify these parameters. 
\ No newline at end of file
diff --git a/documentation20/architecture-ch.md b/documentation20/architecture-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..c4e607076153097347d9e09f3cafc00c6a132be6
--- /dev/null
+++ b/documentation20/architecture-ch.md
@@ -0,0 +1,275 @@
+#数据模型和整体架构 
+
+## 数据模型
+### 物联网典型场景
+在典型的物联网、车联网、运维监测场景中，往往有多种不同类型的数据采集设备，采集一个到多个不同的物理量。而同一种采集设备类型，往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总，然后进行计算和分析。对于同一类设备，其采集的数据都是很规则的。以智能电表为例，假设每个智能电表采集电流、电压、相位三个量，其采集的数据类似如下的表格：
+
+| Device ID |  Time Stamp   | current | voltage | phase |      |     location     | groupId |
+| :-------: | :-----------: | :-----: | :-----: | :---: | :--- | :--------------: | :-----: |
+|   D1001   | 1538548685000 |  10.3   |   219   | 0.31  |      | Beijing.Chaoyang |    2    |
+|   D1002   | 1538548684000 |  10.2   |   220   | 0.23  |      | Beijing.Chaoyang |    3    |
+|   D1003   | 1538548686500 |  11.5   |   221   | 0.35  |      | Beijing.Haidian  |    3    |
+|   D1004   | 1538548685500 |  13.4   |   223   | 0.29  |      | Beijing.Haidian  |    2    |
+|   D1001   | 1538548695000 |  12.6   |   218   | 0.33  |      | Beijing.Chaoyang |    2    |
+|   D1004   | 1538548696600 |  11.8   |   221   | 0.28  |      | Beijing.Haidian  |    2    |
+|   D1002   | 1538548696650 |  10.3   |   218   | 0.25  |      | Beijing.Chaoyang |    3    |
+|   D1001   | 1538548696800 |  12.3   |   221   | 0.31  |      | Beijing.Chaoyang |    2    |
+<center> 表1：智能电表数据示例</center>
+每一条记录都有设备ID，时间戳，采集的物理量(如上图中的电流、电压、相位），还有与每个设备相关的静态标签（如上述表一中的位置Location和分组groupId）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
+
+### 数据特征 
+除时序特征外，仔细研究发现，物联网、车联网、运维监测类数据还具有很多其他明显的特征。
+
+1. 数据是结构化的；
+2. 数据极少有更新或删除操作；
+3. 无需传统数据库的事务处理；
+4. 相对互联网应用，写多读少；
+5. 流量平稳，根据设备数量和采集频次，可以预测出来；
+6. 用户关注的是一段时间的趋势，而不是某一特点时间点的值；
+7. 数据是有保留期限的；
+8. 数据的查询分析一定是基于时间段和地理区域的；
+9. 除存储查询外，还往往需要各种统计和实时计算操作；
+10. 数据量巨大，一天采集的数据就可以超过100亿条。
+
+充分利用上述特征，TDengine采取了一特殊的优化的存储和计算设计来处理时序数据，能将系统处理能力显著提高。
+
+### 关系型数据库模型
+因为采集的数据一般是结构化数据，而且为降低学习门槛，TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库，然后创建表，之后才能插入或查询数据。TDengine采用的是结构化存储，而不是NoSQL的key-value存储。
+
+### 一个数据采集点一张表 
+为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的D1001, D1002, D1003, D1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点：
+
+1. 能保证一个采集点的数据在存储介质上是一块一块连续的。如果读取一个时间段的数据，它能大幅减少随机读取操作，成数量级的提升读取和查询速度。
+2. 由于不同采集设备产生数据的过程完全独立，每个设备的数据源是唯一的，一张表也就只有一个写入者，这样就可采用无锁方式来写，写入速度就能大幅提升。
+3. 对于一个数据采集点而言，其产生的数据是时序的，因此写的操作可用追加的方式实现，进一步大幅提高数据写入速度。
+
+如果采用传统的方式，将多个设备的数据写入一张表，由于网络延时不可控，不同设备的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个设备的数据是难以保证连续存储在一起的。**采用一个采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。**
+
+TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。每个数据采集点可能同时采集多个物理量(如上表中的curent, voltage, phase)，每个物理量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集的数据，TDengine将自动按照时间戳建立索引，但对采集的物理量不建任何索引。数据是用列式存储方式保存。
+
+### 超级表：同一类型数据采集点的集合
+由于一个数据采集点一张表，导致表的数量巨大，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine引入超级表(Super Table，简称为STable)的概念。
+
+超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（一特定类型的数据采集点），除需要定义采集量的表结构之外，还需要定义其标签的schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。 如果整个系统有N个不同类型的数据采集点，就需要建立N个超级表。
+
+在TDengine的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点**。当为某个具体数据采集点创建表时，用户使用超级表的定义做模板，同时指定该具体采集点（表）的标签值。与传统的关系型数据库相比，表（一个数据采集点）是带有静态标签的，而且这些标签可以事后增加、删除、修改。**一张超级表包含有多张表，这些表具有相同的时序数据schema，但带有不同的标签值**。
+
+当对多个具有相同数据类型的数据采集点进行聚合操作时，TDengine将先把满足标签过滤条件的表从超级表的中查找出来，然后再扫描这些表的时序数据，进行聚合操作，这样能将需要扫描的数据集大幅减少，从而大幅提高聚合计算的性能。
+
+##集群与基本逻辑单元
+TDengine 的设计是基于单个硬件、软件系统不可靠，基于任何单台计算机都无法提供足够计算能力和存储能力处理海量数据的假设进行设计的。因此 TDengine 从研发的第一天起，就按照分布式高可靠架构进行设计，是支持水平扩展的，这样任何单台或多台服务器发生硬件故障或软件错误都不影响系统的可用性和可靠性。同时，通过节点虚拟化并辅以自动化负载均衡技术，TDengine 能最高效率地利用异构集群中的计算和存储资源降低硬件投资。
+
+###主要逻辑单元
+TDengine 分布式架构的逻辑结构图如下：
+<center> <img src="../assets/structure.png"> </center>
+
+<center> 图 1 TDengine架构示意图  </center>
+一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine客户端(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
+
+**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。
+
+**数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode). dnode在系统中的唯一标识由实例的End Point(EP)决定。EP是dnode所在物理节点的FQDN(Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。
+
+**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的SCHEMA、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的Vgroup ID在系统内唯一标识，是由管理节点创建并管理的。
+
+**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个管理节点集群(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
+
+**虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性,可以不再需要昂贵的磁盘阵列等存储设备，获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一系统唯一的ID，vnode group ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。Vnode group ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。
+
+**TAOSC:** taosc是TDengine给应用提供的驱动程序(driver)，负责处理应用与集群的接口交互，内嵌于JDBC、ODBC driver中，或者C、Python、Go语言连接库里。应用都是通过taosc，而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的数据节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, ODBC, C/C++接口而言，这个模块是在应用所处的物理节点上运行，但消耗的资源很小。同时，为支持全分布式的RESTful接口，taosc在TDengine集群的每个dnode上都有一运行实例。
+
+### 节点之间的通讯
+**通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受链接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。
+
+**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过选项“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接使用IP地址作为FQDN，但不建议使用，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。
+
+**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP链接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。
+
+**集群对外链接:** TDengine集群可以容纳单个、多个甚至几千个数据节点。应用只需要向集群中任何一个数据节点发起连接即可，链接需要提供的网络参数是一数据节点的End Point(FQDN加配置的端口号）。通过命令行CLI启动应用taos时，可以通过选项-h来指定数据节点的FQDN, -P来指定其配置的端口号，如果端口不配置，将采用TDengine的系统配置参数serverPort。
+
+**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行链接。一个数据节点启动时，将获取mnode所在的dnode的EP信息，然后与系统中的mnode建立起链接，交换信息。获取mnode的EP信息有三步，1：检查mnodeEpList文件是否存在，如果不存在或不能正常打开获得mnode EP信息，进入第二步；2：检查系统配置文件taos.cfg, 获取mnode EP配置参数first, second，如果不存在或者taos.cfg里没有这两个配置参数，或无效，进入第三步；3：将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后，数据节点发起链接，如果链接成功，则成功加入进工作的集群，如果不成功，则尝试mnode EP列表中的下一个。如果都尝试了，但链接都仍然失败，则休眠几秒后，再进行尝试。
+
+**MNODE的选择:** TDengine逻辑上有管理节点，但没有单独的执行代码，服务器侧只有一套执行代码taosd。那么哪个数据节点会是管理节点呢？这是系统自动决定的，无需任何人工干预。原则如下：一个数据节点启动时，会检查自己的End Point, 并与获取的mnode EP List进行比对，如果在其中，该数据节点认为自己应该启动mnode模块，成为mnode。如果自己的EP不在mnode EP List里，则不启动mnode模块。在系统的运行过程中，由于负载均衡、宕机等原因，mnode有可能迁移至新的dnode，但一切都是透明的，无需人工干预，配置参数的修改，是mnode自己根据资源做出的决定。
+
+**新数据节点的加入**：系统有了一个数据节点后，就已经成为一个工作的系统。添加新的节点进集群时，有两个步骤，第一步：使用TDengine CLI链接到现有工作的数据节点，然后用命令”create dnode"将新的数据节点的End Point添加进去; 第二步：在新的数据节点的系统配置参数文件taos.cfg里，将first, second参数设置为现有集群中任意两个数据节点的EP即可。具体添加的详细步骤请见详细的用户手册。这样就把集群一步一步的建立起来。
+
+**重定向**：无论是dnode还是taosc，最先都是要发起与mnode的链接，但mnode是系统自动创建并维护的，因此对于用户来说，并不知道哪个dnode在运行mnode。TDengine只要求向系统中任何一个工作的dnode发起链接即可。因为任何一个正在运行的dnode，都维护有目前运行的mnode EP List。当收到一个来自新启动的dnode或taosc的链接请求，如果自己不是mnode，则将mnode EP List回复给对方，taosc或新启动的dnode收到这个list, 就重新尝试建立链接。当mnode EP List发生改变，通过节点之间的消息交互，各个数据节点就很快获取最新列表，并通知taosc。
+
+###一典型的操作流程
+为解释vnode, mnode, taosc和应用之间的关系以及各自扮演的角色，下面对写入数据这个典型操作的流程进行剖析。
+<center> <img src="../assets/message.png"> </center>
+
+<center> 图 2 TDengine典型的操作流程 </center>
+
+1. 应用通过JDBC、ODBC或其他API接口发起插入数据的请求。
+2. taosc会检查缓存，看是有保存有该表的meta data。如果有，直接到第4步。如果没有，taosc将向mnode发出get meta-data请求。
+3. mnode将该表的meta-data返回给taosc。Meta-data包含有该表的schema, 而且还有该表所属的vgroup信息（vnode ID以及所在的dnode的End Point，如果副本数为N，就有N组End Point)。如果taosc迟迟得不到mnode回应，而且存在多个mnode, taosc将向下一个mnode发出请求。
+4. taosc向master vnode发起插入请求。
+5. vnode插入数据后，给taosc一个应答，表示插入成功。如果taosc迟迟得不到vnode的回应，taosc会认为该节点已经离线。这种情况下，如果被插入的数据库有多个副本，taosc将向vgroup里下一个vnode发出插入请求。
+6. taosc通知APP，写入成功。
+
+对于第二和第三步，taosc启动时，并不知道mnode的End Point，因此会直接向配置的集群对外服务的End Point发起请求。如果接收到该请求的dnode并没有配置mnode，该dnode会在回复的消息中告知mnode EP列表，这样taosc会重新向新的mnode的EP发出获取meta-data的请求。
+
+对于第四和第五步，没有缓存的情况下，taosc无法知道虚拟节点组里谁是master，就假设第一个vnodeID就是master,向它发出请求。如果接收到请求的vnode并不是master,它会在回复中告知谁是master，这样taosc就向建议的master vnode发出请求。一旦得到插入成功的回复，taosc会缓存住master节点的信息。
+
+上述是插入数据的流程，查询、计算的流程也完全一致。taosc把这些复杂的流程全部封装屏蔽了，对于应用来说无感知也无需任何特别处理。
+
+通过taosc缓存机制，只有在第一次对一张表操作时，才需要访问mnode,因此mnode不会成为系统瓶颈。但因为schema有可能变化，而且vgroup有可能发生改变（比如负载均衡发生），因此taosc会定时和mnode交互，自动更新缓存。
+
+## 存储模型与数据分区、分片
+
+###存储模型
+TDengine存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分：
+
+- 时序数据：存放于vnode里，由data、head和last三个文件组成，数据量大，查询量取决于应用场景。容许乱序写入，但暂时不支持删除和更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。
+- 标签数据：存放于vnode里的meta文件，支持增删改查四个标准操作。数据量不大，有N张表，就有N条记录，因此可以全内存存储。如果标签过滤操作很多，查询将十分频繁，因此TDengine支持多核多线程并发查询。只要计算资源足够，即使有数千万张表，过滤结果能毫秒级返回。
+- 其他元数据：存放于mnode里，包含系统节点、用户、DB、Table Schema等等，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。
+
+与典型的NoSQL存储模型相比，TDengine将标签数据与时序数据完全分离存储，它具有两大优势：
+
+- 能够极大地降低标签数据存储的冗余度：一般的NoSQL数据库或时序数据库，采用的K-V存储，其中的Key包含时间戳、设备ID、各种标签。每条记录都带有这些重复的内容，浪费存储空间。而且如果应用要在历史数据上增加、修改或删除标签，需要遍历数据，重写一遍，操作成本极其昂贵。
+- 能够实现极为高效的多表之间的聚合查询：做多表之间聚合查询时，先把符合标签过滤条件的表查找出来，然后再查找这些表相应的数据块，这样大幅减少要扫描的数据集，从而大幅提高查询效率。而且标签数据采用全内存的结构进行管理和维护，千万级别规模的标签数据查询可以在毫秒级别返回。
+
+###数据分片
+对于海量的数据管理，为实现水平扩展，一般都需要采取分片(Sharding)分区(Partitioning)策略。TDengine是通过vnode来实现数据分片的，通过一个时间段一个数据文件来实现时序数据分区的。
+
+vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和计算功能。为便于负载均衡、数据恢复、支持异构环境，TDengine将一个数据节点根据其计算和存储资源切分为多个vnode。这些vnode的管理是TDengine自动完成的，对应用完全透明。
+
+对于单独一个数据采集点，无论其数据量多大，一个vnode（或vnode group, 如果副本数大于1）有足够的计算资源和存储资源来处理（如果每秒生成一条16字节的记录，一年产生的原始数据不到0.5G），因此TDengine将一张表（一个数据采集点）的所有数据都存放在一个vnode里，而不会让同一个采集点的数据分布到两个或多个dnode上。而且一个vnode可存储多个数据采集点(表）的数据，一个vnode可容纳的表的数目的上限为一百万。设计上，一个vnode里所有的表都属于同一个DB。一个数据节点上，一个DB拥有的vnode数目不会超过系统核的数目。
+
+创建DB时，系统并不会马上分配资源。但当创建一张表时，系统将看是否有已经分配的vnode, 且该vnode是否有空余的表空间，如果有，立即在该有空位的vnode创建表。如果没有，系统将从集群中，根据当前的负载情况，在一个dnode上创建一新的vnode, 然后创建表。如果DB有多个副本，系统不是只创建一个vnode，而是一个vgroup(虚拟数据节点组)。系统对vnode的数目没有任何限制，仅仅受限于物理节点本身的计算和存储资源。
+
+每张表的meda data（包含schema, 标签等）也存放于vnode里，而不是集中存放于mnode，实际上这是对Meta数据的分片，这样便于高效并行的进行标签过滤操作。
+
+###数据分区
+TDengine除vnode分片之外，还按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由DB的配置参数days决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数keep)，将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。
+
+总的来说，**TDengine是通过vnode以及时间两个维度，对大数据进行切分**，便于并行高效的管理，实现水平扩展。
+
+###负载均衡
+每个dnode都定时向 mnode(虚拟管理节点)报告其状态（包括硬盘空间、内存大小、CPU、网络、虚拟节点个数等），因此mnode了解整个集群的状态。基于整体状态，当mnode发现某个dnode负载过重，它会将dnode上的一个或多个vnode挪到其他dnode。在挪动过程中，对外服务继续进行，数据插入、查询和计算操作都不受影响。
+
+如果mnode一段时间没有收到dnode的状态报告，mnode会认为这个dnode已经离线。如果离线时间超过一定时长（时长由配置参数offlineThreshold决定），该dnode将被mnode强制剔除出集群。该dnode上的vnodes如果副本数大于一，系统将自动在其他dnode上创建新的副本，以保证数据的副本数。如果该dnode上还有mnode, 而且mnode的副本数大于一，系统也将自动在其他dnode上创建新的mnode, 以保证mnode的副本数。
+
+当新的数据节点被添加进集群，因为新的计算和存储被添加进来，系统也将自动启动负载均衡流程。
+
+负载均衡过程无需任何人工干预，应用也无需重启，将自动连接新的节点，完全透明。
+
+##数据写入与复制流程
+如果一个数据库有N个副本，那一个虚拟节点组就有N个虚拟节点，但是只有一个是Master，其他都是slave。当应用将新的记录写入系统时，只有Master vnode能接受写的请求。如果slave vnode收到写的请求，系统将通知taosc需要重新定向。
+###Master vnode写入流程
+Master Vnode遵循下面的写入流程：
+<center> <img src="../assets/write_master.png">  </center>
+
+<center> 图 3 TDengine Master写入流程  </center>
+
+1. Master vnode收到应用的数据插入请求，验证OK，进入下一步；
+2. 如果系统配置参数walLevel打开（设置为2），vnode将把该请求的原始数据包写入数据库日志文件WAL，以保证TDengine能够在断电等因素导致的服务重启时从数据库日志文件中恢复数据，避免数据的丢失；
+3. 如果有多个副本，vnode将把数据包转发给同一虚拟节点组内slave vnodes, 该转发包带有数据的版本号(version)
+4. 写入内存，并加记录加入到skip list；
+5. Master vnode返回确认信息给应用，表示写入成功。
+6. 如果第2，3，4步中任何一步失败，将直接返回错误给应用。
+
+### Slave vnode写入流程
+对于slave vnode, 写入流程是：
+<center> <img src="../assets/write_slave.png">  </center>
+
+<center> 图 4 TDengine Slave写入流程  </center>
+
+1. Slave vnode收到Master vnode转发了的数据插入请求。
+2. 如果系统配置参数walLevl设置为2，vnode将把该请求的原始数据包写入日志(WAL)；
+3. 写入内存，更新内存中的skip list。
+
+与Master vnode相比，slave vnode不存在转发环节，也不存在回复确认环节，少了两步。但写内存与WAL是完全一样的。
+
+### 异地容灾、IDC迁移
+从上述Master和Slave流程可以看出，TDengine采用的是异步复制的方式进行数据同步。这种方式能够大幅提高写入性能，网络延时对写入速度不会有大的影响。通过配置每个物理节点的IDC和机架号，可以让一个虚拟节点组内，虚拟节点由来自不同IDC、不同机架的物理节点组成，从而实现异地容灾。因此TDengine原生支持异地容灾，无需再使用其他工具。
+
+另外一方面，TDengine支持动态修改副本数，一旦副本数增加，新加入的虚拟节点将立即进入数据同步流程，同步结束后，新加入的虚拟节点即可提供服务。而在同步过程中，master以及其他已经同步的虚拟节点都可以对外提供服务。利用这一特性，TDengine可以实现无服务中断的IDC机房迁移。只需要将新IDC的物理节点加入现有集群，等数据同步完成后，再将老的IDC的物理节点从集群中剔除即可。
+
+但是，这种异步复制的方式，存在极小的时间窗口，丢失写入的数据。具体场景如下：
+
+1. Master vnode完成了它的5步操作，已经给APP确认写入成功，然后宕机；
+2. Slave vnode收到写入请求后，在第2步写入日志之前，处理失败
+3. Slave vnode将成为新的master, 从而丢失了一条记录
+
+理论上，只要是异步复制，就无法保证100%不丢失。但是这个窗口极小，mater与slave要同时发生故障，而且发生在刚给应用确认写入成功之后。
+
+注：异地容灾、IDC无中断迁移，仅仅企业版支持
+
+### 主从选择
+Vnode会保持一个数据版本号(Version)，对内存数据进行持久化存储时，对该版本号也进行持久化存储。每个数据更新操作，无论是采集的时序数据还是元数据，这个版本号将增一。
+
+一个vnode启动时，角色(master、slave) 是不定的，数据是处于未同步状态，它需要与虚拟节点组内其他节点建立TCP链接，并互相交换status，其中包括version和自己的角色。通过status的交换，系统进入选主流程，规则如下：
+
+1. 如果只有一个副本，该副本永远就是master
+2. 所有副本都在线时，版本最高的被选为master
+3. 在线的虚拟节点数过半，而且有虚拟节点是slave的话，该虚拟节点自动成为master
+4. 对于2和3，如果多个虚拟节点满足成为master的要求，那么虚拟节点组的节点列表里，最前面的选为master
+
+更多的关于数据复制的流程，请见[TDengine 2.0 数据复制模块设计](https://jira.taosdata.com:18090/pages/viewpage.action?pageId=6266055)。
+
+###同步复制
+对于数据一致性要求更高的场景，异步数据复制无法满足要求，因为有极小的概率丢失数据，因此TDengine提供同步复制的机制供用户选择。在创建数据库时，除指定副本数replica之外，用户还需要指定新的参数quorum。如果quorum大于一，它表示每次Master转发给副本时，需要等待quorum-1个回复确认，才能通知应用，数据在slave已经写入成功。如果在一定的时间内，得不到quorum-1个回复确认，master vnode将返回错误给应用。
+
+采用同步复制，系统的性能会有所下降，而且latency会增加。因为元数据要强一致，Mnode之间的数据同步就是采用的同步复制。
+
+注：vnode之间的同步复制仅仅企业版支持
+
+##缓存与持久化
+###缓存
+TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心的是刚产生的数据，即当前状态。TDengine充分利用这一特性，将最近到达的（当前状态）数据保存在缓存中。
+
+TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。
+
+每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当一半以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有一半内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。
+
+###持久化存储
+TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。
+
+为充分利用时序数据特点，TDengine将一个vnode保存在持久化存储的数据切分成多个文件，每个文件只保存固定天数的数据，这个天数由系统配置参数days决定。切分成多个文件后，给定查询的起止日期，无需任何索引，就可以立即定位需要打开哪些数据文件，大大加快读取速度。
+
+对于采集的数据，一般有保留时长，这个时长由系统配置参数keep决定。超过这个设置天数的数据文件，将被系统将自动删除，释放存储空间。
+
+给定days与keep两个参数，一个vnode总的数据文件数为：keep/days。总的数据文件个数不宜过大，也不宜过小。10到100以内合适。基于这个原则，可以设置合理的days。 目前的版本，参数keep可以修改，但对于参数days，一但设置后，不可修改。
+
+在每个数据文件里，一张表的数据是一块一块存储的。一张表可以有一到多个数据文件块。在一个文件块里，数据是列式存储的，占用的是一片连续的存储空间，这样大大提高读取速度。文件块的大小由系统参数maxRows（每块最大记录条数）决定，缺省值为4096。这个值不宜过大，也不宜过小。过大，定位具体时间段的数据的搜索时间会变长，影响读取速度；过小，数据块的索引太大，压缩效率偏低，也影响读取速度。
+
+每个数据文件(.data结尾)都有一个对应的索引文件（.head结尾），该索引文件对每张表都有一数据块的摘要信息，记录了每个数据块在数据文件中的偏移量，数据的起止时间等信息，以帮助系统迅速定位需要查找的数据。每个数据文件还有一对应的last文件(.last结尾)，该文件是为防止落盘时数据块碎片化而设计的。如果一张表落盘的记录条数没有达到系统配置参数minRows（每块最小记录条数），将被先存储到last文件，等下次落盘时，新落盘的记录将与last文件的记录进行合并，再写入数据文件。
+
+数据写入磁盘时，根据系统配置参数comp决定是否压缩数据。TDengine提供了三种压缩选项：无压缩、一阶段压缩和两阶段压缩，分别对应comp值为0、1和2的情况。一阶段压缩根据数据的类型进行了相应的压缩，压缩算法包括delta-delta编码、simple 8B方法、zig-zag编码、LZ4等算法。二阶段压缩在一阶段压缩的基础上又用通用压缩算法进行了压缩，压缩率更高。
+
+###多级存储
+在默认配置下，TDengine会将所有数据保存在/var/lib/taos目录下，而且每个vnode的数据文件保存在该目录下的不同目录。为扩大存储空间，尽量减少文件读取的瓶颈，提高数据吞吐率 TDengine可通过配置系统参数dataDir让多个挂载的硬盘被系统同时使用。除此之外，TDengine也提供了数据分级存储的功能，即根据数据文件的新老程度存储在不同的存储介质上。比如最新的数据存储在SSD上，超过一周的数据存储在本地硬盘上，超过4周的数据存储在网络存储设备上，这样来降低存储成本，而又保证高效的访问数据。数据在不同存储介质上的移动是由系统自动完成的，对应用是完全透明的。数据的分级存储也是通过系统参数dataDir来配置。
+
+dataDir的配置格式如下：
+```
+dataDir data_path [tier_level]
+```
+其中data_path为挂载点的文件夹路径，tier_level为介质存储等级。介质存储等级越高，盛放数据文件越老。同一存储等级可挂载多个硬盘，同一存储等级上的数据文件分布在该存储等级的所有硬盘上。TDengine最多支持3级存储，所以tier_level的取值为0、1和2。在配置dataDir时，必须存在且只有一个挂载路径不指定tier_level，称之为特殊挂载盘（路径）。该挂载路径默认为0级存储介质，且包含特殊文件链接，不可被移除，否则会对写入的数据产生毁灭性影响。
+
+假设一物理节点有六个可挂载的硬盘/mnt/disk1、/mnt/disk2、…、/mnt/disk6，其中disk1和disk2需要被指定为0级存储介质，disk3和disk4为1级存储介质， disk5和disk6为2级存储介质。disk1为特殊挂载盘，则可在/etc/taos/taos.cfg中做如下配置：
+
+```
+dataDir /mnt/disk1/taos
+dataDir /mnt/disk2/taos 0
+dataDir /mnt/disk3/taos 1
+dataDir /mnt/disk4/taos 1
+dataDir /mnt/disk5/taos 2
+dataDir /mnt/disk6/taos 2
+```
+
+挂载的盘也可以是非本地的网络盘，只要系统能访问即可。
+
+注：多级存储功能仅企业版支持
+
+##数据查询
+TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。
+
+5.1 单表查询
+
+5.4 时间轴聚合、插值 
+
+5.2 多表聚合查询
+
+5.3 预计算
\ No newline at end of file
diff --git a/documentation20/architecture.md b/documentation20/architecture.md
new file mode 100644
index 0000000000000000000000000000000000000000..3a91f1e8dc24314f66acf906e69d3dcd0df8e370
--- /dev/null
+++ b/documentation20/architecture.md
@@ -0,0 +1,101 @@
+# Data Model and Architecture
+## Data Model
+
+### A Typical IoT Scenario 
+
+In a typical IoT scenario, there are many types of devices. Each device is collecting one or multiple metrics. For a specific type of device, the collected data looks like the table below: 
+
+| Device ID |  Time Stamp   | Value 1 | Value 2 | Value 3 | Tag 1 | Tag 2 |
+| :-------: | :-----------: | :-----: | :-----: | :-----: | :---: | :---: |
+|   D1001   | 1538548685000 |  10.3   |   219   |  0.31   |  Red  | Tesla |
+|   D1002   | 1538548684000 |  10.2   |   220   |  0.23   | Blue  |  BMW  |
+|   D1003   | 1538548686500 |  11.5   |   221   |  0.35   | Black | Honda |
+|   D1004   | 1538548685500 |  13.4   |   223   |  0.29   |  Red  | Volvo |
+|   D1001   | 1538548695000 |  12.6   |   218   |  0.33   |  Red  | Tesla |
+|   D1004   | 1538548696600 |  11.8   |   221   |  0.28   | Black | Honda |
+
+Each data record has device ID, timestamp, the collected metrics, and static tags associated with the device. Each device generates a data record in a pre-defined timer or triggered by an event. It is a sequence of data points, like a stream. 
+
+### Data Characteristics
+
+Being a series of data points over time, data points generated by devices, sensors, servers, or applications have strong common characteristics. 
+
+1. metric is always structured data; 
+2. there are rarely delete/update operations on collected data; 
+3. there is only one single data source for one device or sensor; 
+4. ratio of read/write is much lower than typical Internet application; 
+5. the user pays attention to the trend of data, not the specific value at a specific time; 
+6. there is always a data retention policy; 
+7. the data query is always executed in a given time range and a subset of devices; 
+8. real-time aggregation or analytics is mandatory; 
+9. traffic is predictable based on the number of devices and sampling frequency; 
+10. data volume is huge, a system may generate 10 billion data points in a day.    
+
+By utilizing the above characteristics, TDengine designs the storage and computing engine in a special and optimized way for time-series data. The system efficiency is improved significantly.    
+
+### Relational Database Model 
+
+Since time-series data is more likely to be structured data, TDengine adopts the traditional relational database model to process them. You need to create a database, create tables with schema definition, then insert data points and execute queries to explore the data. Standard SQL is used, there is no learning curve. 
+
+### One Table for One Device
+
+Due to different network latency, the data points from different devices may arrive at the server out of order. But for the same device, data points will arrive at the server in order if system is designed well. To utilize this special feature, TDengine requires the user to create a table for each device (time-stream). For example, if there are over 10,000 smart meters, 10,000 tables shall be created. For the table above, 4 tables shall be created for device D1001, D1002, D1003 and D1004, to store the data collected. 
+
+This strong requirement can guarantee the data points from a device can be saved in a continuous memory/hard disk space block by block. If queries are applied only on one device in a time range, this design will reduce the read latency significantly since a whole block is owned by one single device. Also, write latency can be significantly reduced too, since the data points generated by the same device will arrive in order, the new data point will be simply appended to a block. Cache block size and the rows of records in a file block can be configured to fit the scenarios.
+
+### Best Practices
+
+**Table**: TDengine suggests to use device ID as the table name (like D1001 in the above diagram). Each device may collect one or more metrics (like value1, valu2, valu3 in the diagram). Each metric has a column in the table, the metric name can be used as the column name. The data type for a column can be int, float, double, tinyint, bigint, bool or binary. Sometimes, a device may have multiple metric group, each group have different sampling period, you shall create a table for each group for each device. The first column in the table must be time stamp. TDengine uses time stamp as the index, and won’t build the index on any metrics stored.
+
+**Tags:** to support aggregation over multiple tables efficiently, [STable(Super Table)](../super-table) concept is introduced by TDengine. A STable is used to represent the same type of device. The schema is used to define the collected metrics(like value1, value2, value3 in the diagram), and tags are used to define the static attributes for each table or device(like tag1, tag2 in the diagram). A table is created via STable with a specific tag value. All or a subset of tables in a STable can be aggregated by filtering tag values.  
+
+**Database:** different types of devices may generate data points in different patterns and shall be processed differently. For example, sampling frequency, data retention policy, replication number, cache size, record size, the compression algorithm may be different. To make the system more efficient, TDengine suggests creating a different database with unique configurations for different scenarios 
+
+**Schemaless vs Schema:** compared with NoSQL database, since a table with schema definition shall be created before the data points can be inserted, flexibilities are not that good, especially when the schema is changed. But in most IoT scenarios, the schema is well defined and is rarely changed, the loss of flexibilities won’t be a big pain to developers or the administrator. TDengine allows the application to change the schema in a second even there is a huge amount of historical data when schema has to be changed. 
+
+TDengine does not impose a limitation on the number of tables, [STables](../super-table), or databases. You can create any number of STable or databases to fit the scenarios.   
+
+## Architecture
+
+There are two main modules in TDengine server as shown in Picture 1: **Management Module (MGMT)** and **Data Module(DNODE)**. The whole TDengine architecture also includes a **TDengine Client Module**.
+
+<center> <img src="../assets/structure.png"> </center>
+<center> Picture 1 TDengine Architecture  </center>
+### MGMT Module
+The MGMT module deals with the storage and querying on metadata, which includes information about users, databases, and tables. Applications will connect to the MGMT module at first when connecting the TDengine server. When creating/dropping databases/tables, The request is sent to the MGMT module at first to create/delete metadata. Then the MGMT module will send requests to the data module to allocate/free resources required. In the case of writing or querying, applications still need to visit MGMT module to get meta data, according to which, then access the DNODE module.
+
+### DNODE Module
+The DNODE module is responsible for storing and querying data. For the sake of future scaling and high-efficient resource usage, TDengine applies virtualization on resources it uses. TDengine introduces the concept of virtual node (vnode), which is the unit of storage, resource allocation and data replication (enterprise edition). As is shown in Picture 2, TDengine treats each data node as an aggregation of vnodes. 
+
+When a DB is created, the system will allocate a vnode. Each vnode contains multiple tables, but a table belongs to only one vnode. Each DB has one or mode vnodes, but one vnode belongs to only one DB. Each vnode contains all the data in a set of tables. Vnodes have their own cache, directory to store data. Resources between different vnodes are exclusive with each other, no matter cache or file directory. However, resources in the same vnode are shared between all the tables in it. By virtualization, TDengine can distribute resources reasonably to each vnode and improve resource usage and concurrency. The number of vnodes on a dnode is configurable according to its hardware resources.
+
+<center> <img src="../assets/vnode.png">  </center>
+<center> Picture 2 TDengine Virtualization  </center> 
+
+### Client Module
+TDengine client module accepts requests (mainly in SQL form) from applications and converts the requests to internal representations and sends to the server side. TDengine supports multiple interfaces, which are all built on top of TDengine client module. 
+
+For the communication between client and MGMT module, TCP/UDP is used, the port is set by the parameter mgmtShellPort in system configuration file taos.cfg, default is 6030. For the communication between client and DNODE module, TCP/UDP is used, the port is set by the parameter vnodeShellPort in the system configuration file, default is 6035. 
+
+## Writing Process
+Picture 3 shows the full writing process of TDengine. TDengine uses [Writing Ahead Log] (WAL) strategy to assure data security and integrity. Data received from the client is written to the commit log at first. When TDengine recovers from crashes caused by power lose or other situations, the commit log is used to recover data. After writting to commit log, data will be wrtten to the corresponding vnode cache, then an acknowledgment is sent to the application. There are two mechanisms that can flush data in cache to disk for persistent storage:
+
+1. **Flush driven by timer**: There is a backend timer which flushes data in cache periodically to disks. The period is configurable via parameter commitTime in system configuration file taos.cfg.
+2. **Flush driven by data**: Data in the cache is also flushed to disks when the left buffer size is below a threshold. Flush driven by data can reset the timer of flush driven by the timer.
+
+<center> <img src="../assets/write_process.png">  </center>
+<center> Picture 3 TDengine Writting Process  </center>
+
+New commit log file will be opened when the committing process begins. When the committing process finishes, the old commit file will be removed.
+
+## Data Storage
+
+TDengine data are saved in _/var/lib/taos_ directory by default. It can be changed to other directories by setting the parameter dataDir in system configuration file taos.cfg.
+
+TDengine's metadata includes the database, table, user, super table and tag information. To reduce the latency, metadata are all buffered in the cache.
+
+Data records saved in tables are sharded according to the time range. Data of tables in the same vnode in a certain time range are saved in the same file group. This sharding strategy can effectively improve data searching speed. By default, one group of files contain data in 10 days, which can be configured by *daysPerFile* in the configuration file or by *DAYS* keyword in *CREATE DATABASE* clause. 
+
+Data records are removed automatically once their lifetime is passed. The lifetime is configurable via parameter daysToKeep in the system configuration file. The default value is 3650 days. 
+
+Data in files are blockwise. A data block only contains one table's data. Records in the same data block are sorted according to the primary timestamp. To improve the compression ratio, records are stored column by column, and the different compression algorithm is applied based on each column's data type. 
\ No newline at end of file
diff --git a/documentation20/cluster.md b/documentation20/cluster.md
new file mode 100644
index 0000000000000000000000000000000000000000..dbef70343efc29406fd90eb717f0cabf9a3568d7
--- /dev/null
+++ b/documentation20/cluster.md
@@ -0,0 +1,107 @@
+#集群管理
+
+多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。
+
+TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。
+
+##创建第一个节点
+集群是由一个一个dnode组成的，是从一个dnode的创建开始的。创建第一个节点很简单，确保系统配置文件taos.cfg里的参数first与second没有进行设置后，简单的运行taosd即可。配置文件taos.cfg里参数first与second缺省都是设置为空，因此无需特殊操作。如果为了统一配置，可以将参数first配置为第一个dnode的End Point。
+
+这个节点创建后，它是集群中的第一个节点。如果该节点收到一个新节点加入集群的请求，它将检查这个新节点的End Point是否在集群的EP列表中，如果在，就容许其加入。如果不在列表中，就拒绝其加入。
+
+##节点管理
+###添加节点
+具体有以下几个步骤：
+1. 修改新节点配置文件taos.cfg, 将其中参数first与second设置为现有集群中运行节点的End Point, 然后启动taosd。second可以不用设置，目的是为集群规模较大时，first无法访问时，该节点将尝试访问second节点。
+2. 使用CLI程序taos, 登录进系统, 使用命令:
+   ```
+   CREATE DNODE "fqdn:port"； 
+   ```
+   将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置。
+3. 使用命令
+   ```
+   SHOW DNODES
+   ```
+   查看新节点是否被成功加入。`
+
+**示例：**假设有两个节点，其EP分别为 host1:6030, host2:6050. 可以按照以下步骤创建集群
+
+1. 在host1, 启动taosd;
+2. 修改host2:6050的配置文件taos.cfg, 将参数first设置为host1:6030，然后启动host2:6050这个taosd实例;
+3. 执行“taos -h host1" 链接到host1, 在TDengine console里执行命令：
+   ```
+   CREATE DNODE "host2:6050";
+   ```
+4. 在TDengine console里执行`"show dnodes"`查看节点是否加入成功.
+5. 按照上述第2到第5步，逐步把其他节点添加到集群中。
+
+**提示：**
+
+- first, second这两个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。
+- 两个没有配置first, second参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。
+
+###删除节点
+执行CLI程序taos, 使用root账号登录进TDengine系统，执行：
+```
+DROP DNODE "fqdn:port";
+```
+其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号
+
+###查看节点
+执行CLI程序taos,使用root账号登录进TDengine系统，执行：
+```
+SHOW DNODES;
+```
+它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个节点后，可以使用该命令查看。
+
+###查看虚拟节点组
+
+为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。
+
+执行CLI程序taos,使用root账号登录进TDengine系统，执行：
+```
+SHOW VGROUPS;
+```
+##高可用性
+TDengine通过多副本的机制来提供系统的高可用性。副本数是与DB关联的，一个集群里可以有多个DB，根据运营的需求，每个DB可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数（缺省为1）。如果副本数为1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo：
+```
+CREATE DATABASE demo replica 3;
+```
+一个DB里的数据会被切片分到多个vnode group，vnode group里的vnode数目就是DB的副本数，同一个vnode group里各vnode的数据是完全一致的。为保证高可用性，vnode group里的vnode一定要分布在不同的dnode里（实际部署时，需要在不同的物理机上），只要一个vgroup里超过半数的vnode处于工作状态，这个vgroup就能正常的对外服务。
+
+一个dnode里可能有多个DB的数据，因此一个dnode离线时，可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作，那么该vnode group就无法对外服务，无法插入或读取数据，这样会影响到它所属的DB的一部分表的d读写操作。
+
+因为vnode的引入，无法简单的给出结论：“集群中过半dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个节点不工作，那整个集群就无法正常工作了。
+
+##Mnode的高可用
+TDengine集群是由mnode (taosd的一个模块，逻辑节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。
+
+一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令：
+```
+SHOW MNODES;
+```
+来查看mnode列表，该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。
+当集群中第一个节点启动时，该节点一定会运行一个mnode实例，否则该dnode无法正常工作，因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2，启动第二个dnode时，该dnode也将运行一个mnode实例。
+
+为保证mnode服务的高可用性，numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的，如果numOfMnodes大于2，复制参数quorum自动设为2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。
+
+##负载均衡
+
+有三种情况，将触发负载均衡，而且都无需人工干预。
+
+- 当一个新节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新节点上，无需任何人工干预。
+- 当一个节点从集群中移除时，系统将自动把该节点上的数据转移到其他节点，无需任何人工干预。
+- 如果一个节点过热（数据量过大），系统将自动进行负载均衡，将该节点的一些vnode自动挪到其他节点。
+
+当上述三种情况发生时，系统将启动一各个节点的负载计算，从而决定如何挪动。
+
+##节点离线处理
+如果一个节点离线，TDengine集群将自动检测到。有如下两种情况：
+- 改节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。
+- 离线后，在offlineThreshold的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。
+
+##Arbitrator的使用
+
+如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。
+
+TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。
diff --git a/documentation20/connector-ch.md b/documentation20/connector-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..71f1dd31d30516ae95532314679f9e75525d3736
--- /dev/null
+++ b/documentation20/connector-ch.md
@@ -0,0 +1,841 @@
+# 连接器
+
+TDengine提供了丰富的应用程序开发接口，其中包括C/C++、JAVA、Python、RESTful、Go等，便于用户快速开发应用。
+
+注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
+
+## C/C++ Connector
+
+C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）：
+
+```C
+#include <taos.h>
+```
+
+在编译时需要链接TDengine动态库 _libtaos.so_ （安装后，位于 _/usr/local/taos/driver_，gcc编译时，请加上 -ltaos）。
+
+如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。
+
+
+### 基础API
+
+基础API用于完成创建数据库连接等工作，为其它API的执行提供运行时环境。
+
+- `void taos_init()`
+
+  初始化运行环境。如果应用没有主动调用该API，那么应用在调用`taos_connect`时将自动调用，故应用程序一般无需手动调用该API。 
+
+
+- `void taos_cleanup()`
+
+  清理运行环境，应用退出前应调用此API。
+
+
+- `int taos_options(TSDB_OPTION option, const void * arg, ...)`
+
+   设置客户端选项，目前只支持时区设置（_TSDB_OPTION_TIMEZONE_）和编码设置（_TSDB_OPTION_LOCALE_）。时区和编码默认为操作系统当前设置。 
+
+
+- `char *taos_get_client_info()`
+
+  获取客户端版本信息。
+
+
+- `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)`
+
+  创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含：
+
+  * ip：TDengine管理主节点的IP地址
+  * user：用户名
+  * pass：密码
+  * db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库
+  * port：端口号
+  
+  返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。
+
+
+- `char *taos_get_server_info(TAOS *taos)`
+
+  获取服务端版本信息。
+
+
+- `int taos_select_db(TAOS *taos, const char *db)`
+
+  将当前的缺省数据库设置为`db`。
+
+
+- `void taos_close(TAOS *taos)`
+
+  关闭连接, 其中`taos`是`taos_connect`函数返回的指针。
+
+
+
+### 同步查询API
+
+传统的数据库操作API，都属于同步操作。应用调用API后，一直处于阻塞状态，直到服务器返回结果。TDengine支持如下API：
+
+- `TAOS_RES* taos_query(TAOS *taos, const char *sql)`
+
+  该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。
+
+
+- `int taos_result_precision(TAOS_RES *res)`
+
+  返回结果集时间戳字段的精度，`0` 代表毫秒，`1` 代表微秒，`2` 代表纳秒。
+
+
+- `TAOS_ROW taos_fetch_row(TAOS_RES *res)`
+
+  按行获取查询结果集中的数据。
+
+
+- `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)`
+
+  批量获取查询结果集中的数据，返回值为获取到的数据的行数。
+
+
+- `int taos_num_fields(TAOS_RES *res)` 和 `int taos_field_count(TAOS_RES *res)`
+
+  这两个API等价，用于获取查询结果集中的列数。
+
+
+- `int* taos_fetch_lengths(TAOS_RES *res)`
+
+  获取结果集中每个字段的长度。 返回值是一个数组，其长度为结果集的列数。
+
+
+- `int taos_affected_rows(TAOS_RES *res)`
+
+  获取被所执行的 SQL 语句影响的行数。
+
+
+- `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)`
+
+  获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下：
+
+  ```c
+  typedef struct taosField {
+    char     name[65];  // 列名
+    uint8_t  type;      // 数据类型
+    int16_t  bytes;     // 字节数
+  } TAOS_FIELD;
+  ```
+
+
+- `void taos_stop_query(TAOS_RES *res)`
+
+  停止一个查询的执行。
+
+
+- `void taos_free_result(TAOS_RES *res)`
+
+  释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。
+
+
+- `char *taos_errstr(TAOS_RES *res)`
+
+  获取最近一次API调用失败的原因,返回值为字符串。
+
+
+- `char *taos_errno(TAOS_RES *res)`
+
+  获取最近一次API调用失败的原因，返回值为错误代码。
+
+
+**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该链接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。
+
+
+### 异步查询API
+
+同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。
+
+异步API都需要应用提供相应的回调函数，回调函数参数设置如下：前两个参数都是一致的，第三个参数依不同的API而定。第一个参数param是应用调用异步API时提供给系统的，用于回调时，应用能够找回具体操作的上下文，依具体实现而定。第二个参数是SQL操作的结果集，如果为空，比如insert操作，表示没有记录返回，如果不为空，比如select操作，表示有记录返回。
+
+异步API对于使用者的要求相对较高，用户可根据具体应用场景选择性使用。下面是三个重要的异步API： 
+
+- `void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param);`
+
+  异步执行SQL语句。
+  
+  * taos：调用taos_connect返回的数据库连接
+  * sql：需要执行的SQL语句
+  * fp：用户定义的回调函数，其第三个参数`code`用于指示操作是否成功，`0`表示成功，负数表示失败(调用`taos_errstr`获取失败原因)。应用在定义回调函数的时候，主要处理第二个参数`TAOS_RES *`，该参数是查询返回的结果集
+  * param：应用提供一个用于回调的参数
+
+- `void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param);`
+
+  批量获取异步查询的结果集，只能与`taos_query_a`配合使用。其中：
+  
+  * res：`taos_query_a`回调时返回的结果集
+  * fp：回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体；`numOfRows`是获取到的数据的行数（不是整个查询结果集的函数）。 在回调函数中，应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后，应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理，直到返回的记录数（numOfRows）为零（结果返回完成）或记录数为负值（查询出错）。
+
+
+- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);`
+
+  异步获取一条记录。其中：
+
+  * res：`taos_query_a`回调时返回的结果集
+  * fp：回调函数。其参数`param`是应用提供的一个用于回调的参数。回调时，第三个参数`row`指向一行记录。不同于`taos_fetch_rows_a`，应用无需调用`taos_fetch_row`来获取一行数据，更加简单，但数据提取性能不及批量获取的API。
+
+TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。
+
+
+### 参数绑定API
+
+除了直接调用 `taos_query` 进行查询，TDengine也提供了支持参数绑定的Prepare API，与 MySQL 一样，这些API目前也仅支持用问号`?`来代表待绑定的参数，具体如下：
+
+- `TAOS_STMT* taos_stmt_init(TAOS *taos)`
+
+  创建一个 TAOS_STMT 对象用于后续调用。
+
+- `int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length)`
+
+  解析一条sql语句，将解析结果和参数信息绑定到stmt上，如果参数length大于0，将使用此参数作为sql语句的长度，如等于0，将自动判断sql语句的长度。
+
+- `int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind)`
+
+  进行参数绑定，bind指向一个数组，需保证此数组的元素数量和顺序与sql语句中的参数完全一致。TAOS_BIND 的使用方法与 MySQL中的 MYSQL_BIND 一致，具体定义如下：
+
+  ```c
+  typedef struct TAOS_BIND {
+    int            buffer_type;
+    void *         buffer;
+    unsigned long  buffer_length;  // 未实际使用
+    unsigned long *length;
+    int *          is_null;
+    int            is_unsigned;    // 未实际使用
+    int *          error;          // 未实际使用
+  } TAOS_BIND;
+  ```
+
+- `int taos_stmt_add_batch(TAOS_STMT *stmt)`
+
+  将当前绑定的参数加入批处理中，调用此函数后，可以再次调用`taos_stmt_bind_param`绑定新的参数。需要注意，此函数仅支持 insert/import 语句，如果是select等其他SQL语句，将返回错误。
+
+- `int taos_stmt_execute(TAOS_STMT *stmt)`
+
+  执行准备好的语句。目前，一条语句只能执行一次。
+
+- `TAOS_RES* taos_stmt_use_result(TAOS_STMT *stmt)`
+
+  获取语句的结果集。结果集的使用方式与非参数化调用时一致，使用完成后，应对此结果集调用 `taos_free_result`以释放资源。
+  
+- `int taos_stmt_close(TAOS_STMT *stmt)`
+
+  执行完毕，释放所有资源。
+
+
+### 连续查询接口
+
+TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时间段，对一张或多张数据库的表(数据流)进行各种实时聚合计算操作。操作简单，仅有打开、关闭流的API。具体如下： 
+
+- `TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *))`
+
+  该API用来创建数据流，其中：
+  * taos：已经建立好的数据库连接
+  * sql：SQL查询语句（仅能使用查询语句）
+  * fp：用户定义的回调函数指针，每次流式计算完成后，TDengine将查询的结果（TAOS_ROW）、查询状态（TAOS_RES）、用户定义参数（PARAM）传递给回调函数，在回调函数内，用户可以使用taos_num_fields获取结果集列数，taos_fetch_fields获取结果集每列数据的类型。
+  * stime：是流式计算开始的时间，如果是0，表示从现在开始，如果不为零，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数）
+  * param：是应用提供的用于回调的一个参数，回调时，提供给应用
+  * callback: 第二个回调函数，会在连续查询自动停止时被调用。
+  
+  返回值为NULL，表示创建成功，返回值不为空，表示成功。
+
+
+- `void taos_close_stream (TAOS_STREAM *tstr)`
+
+  关闭数据流，其中提供的参数是taos_open_stream的返回值。用户停止流式计算的时候，务必关闭该数据流。
+  
+
+### 数据订阅接口
+
+订阅API目前支持订阅一张或多张表，并通过定期轮询的方式不断获取写入表中的最新数据。 
+
+* `TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval)`
+
+  该函数负责启动订阅服务，成功时返回订阅对象，失败时返回 `NULL`，其参数为：
+  * taos：已经建立好的数据库连接
+  * restart：如果订阅已经存在，是重新开始，还是继续之前的订阅
+  * topic：订阅的主题（即名称），此参数是订阅的唯一标识
+  * sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据
+  * fp：收到查询结果时的回调函数（稍后介绍函数原型），只在异步调用时使用，同步调用时此参数应该传 `NULL`
+  * param：调用回调函数时的附加参数，系统API将其原样传递到回调函数，不进行任何处理
+  * interval：轮询周期，单位为毫秒。异步调用时，将根据此参数周期性的调用回调函数，为避免对系统性能造成影响，不建议将此参数设置的过小；同步调用时，如两次调用`taos_consume`的间隔小于此周期，API将会阻塞，直到时间间隔超过此周期。
+
+* `typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code)`
+
+  异步模式下，回调函数的原型，其参数为：
+  * tsub：订阅对象
+  * res：查询结果集，注意结果集中可能没有记录
+  * param：调用 `taos_subscribe`时客户程序提供的附加参数
+  * code：错误码
+
+
+* `TAOS_RES *taos_consume(TAOS_SUB *tsub)`
+
+  同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。
+
+* `void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress)`
+
+  取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。
+
+
+##  Java Connector
+
+### JDBC接口
+
+如果用户使用Java开发企业级应用，可选用TDengine提供的JDBC Driver来调用服务。TDengine提供的JDBC Driver是标准JDBC规范的子集，遵循JDBC 标准(3.0)API规范，支持现有的各种Java开发框架。目前TDengine的JDBC driver并未发布到在线依赖仓库比如maven的中心仓库。因此用户开发时，需要手动把驱动包`taos-jdbcdriver-x.x.x-dist.jar`安装到开发环境的依赖仓库中。
+
+TDengine 的驱动程序包的在不同操作系统上依赖不同的本地函数库（均由C语言编写）。Linux系统上，依赖一个名为`libtaos.so` 的本地库，.so即"Shared Object"缩写。成功安装TDengine后，`libtaos.so` 文件会被自动拷贝至`/usr/local/lib/taos`目录下，该目录也包含在Linux上自动扫描路径上。Windows系统上，JDBC驱动程序依赖于一个名为`taos.dll` 的本地库，.dll是动态链接库"Dynamic Link Library"的缩写。Windows上成功安装客户端后，JDBC驱动程序包默认位于`C:/TDengine/driver/JDBC/`目录下;其依赖的动态链接库`taos.dll`文件位于`C:/TDengine/driver/C`目录下，`taos.dll` 会被自动拷贝至系统默认搜索路径`C:/Windows/System32`下。
+
+TDengine的JDBC Driver遵循标准JDBC规范，开发人员可以参考Oracle官方的JDBC相关文档来找到具体的接口和方法的定义与用法。TDengine的JDBC驱动在连接配置和支持的方法上与传统数据库驱动稍有不同。 
+
+TDengine的JDBC URL规范格式为：
+
+`jdbc:TSDB://{host_ip}:{port}/{database_name}?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
+
+其中，`{}`中的内容必须，`[]`中为可选。配置参数说明如下：
+
+- user：登陆TDengine所用用户名；默认值root
+- password：用户登陆密码；默认值taosdata
+- charset：客户端使用的字符集；默认值为系统字符集
+- cfgdir：客户端配置文件目录路径；Linux OS上默认值`/etc/taos` ，Windows OS上默认值 `C:/TDengine/cfg`
+- locale：客户端语言环境；默认值系统当前locale
+- timezone：客户端使用的时区；默认值为系统当前时区
+
+以上所有参数均可在调用java.sql.DriverManager类创建连接时指定，示例如下：
+
+```java
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Properties;
+import com.taosdata.jdbc.TSDBDriver;
+
+public Connection getConn() throws Exception{
+	Class.forName("com.taosdata.jdbc.TSDBDriver");
+  String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata";
+  Properties connProps = new Properties();
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIMEZONE, "UTC-8");
+  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+  return conn;
+}
+```
+
+这些配置参数中除了cfgdir外，均可在客户端配置文件taos.cfg中进行配置。调用java.sql.DriverManager时声明的配置参数优先级最高，JDBC URL的优先级次之，配置文件的优先级最低。例如charset同时在配置文件taos.cfg中配置，也在JDBC URL中配置，则使用JDBC URL中的配置值。
+
+此外，尽管TDengine的JDBC驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致TDengine的Java API并不能与标准完全相同。对于有大量关系型数据库开发经验而初次接触TDengine的开发者来说，有以下一些值的注意的地方：
+
+* TDengine不提供针对单条数据记录的删除和修改的操作，驱动中也没有支持相关方法
+* 目前TDengine不支持表间的join或union操作，因此也缺乏对该部分API的支持
+* TDengine支持批量写入，但是支持停留在SQL语句级别，而不是API级别，也就是说用户需要通过写特殊的SQL语句来实现批量
+* 目前TDengine不支持嵌套查询(nested query)，对每个Connection的实例，至多只能有一个打开的ResultSet实例；如果在ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver则会自动关闭上一个ResultSet
+
+对于TDengine操作的报错信息，用户可使用JDBCDriver包里提供的枚举类TSDBError.java来获取error message和error code的列表。对于更多的具体操作的相关代码，请参考TDengine提供的使用示范项目`JDBCDemo`。
+
+## Python Connector
+
+### 安装准备
+* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）](https://www.taosdata.com/cn/documentation/connector/#Windows客户端及程序接口)
+* 已安装python 2.7 or >= 3.4
+* 已安装pip
+
+### Python客户端安装
+
+#### Linux
+
+用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装： 
+
+​		`pip install src/connector/python/linux/python2/`
+
+或
+
+​		`pip install src/connector/python/linux/python3/`
+
+#### Windows
+在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos <em>cmd</em> 命令行界面
+```cmd
+cd C:\TDengine\connector\python\windows
+pip install python2\
+```
+或
+```cmd
+cd C:\TDengine\connector\python\windows
+pip install python3\
+```
+
+*如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。
+对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。
+
+### 使用
+
+#### 代码示例
+
+* 导入TDengine客户端模块
+
+```python
+import taos 
+```
+* 获取连接并获取游标对象
+```python
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+```
+*<em>host</em> 是TDengine 服务端所有IP, <em>config</em> 为客户端配置文件所在目录
+
+* 写入数据
+```python
+import datetime
+ 
+# 创建数据库
+c1.execute('create database db')
+c1.execute('use db')
+# 建表
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# 插入数据
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# 批量插入数据
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+```
+
+* 查询数据
+```python
+c1.execute('select * from tb')
+# 拉取查询结果
+data = c1.fetchall()
+# 返回的结果是一个列表，每一行构成列表的一个元素
+numOfRows = c1.rowcount
+numOfCols = len(c1.description)
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2]))
+  
+# 直接使用cursor 循环拉取查询结果
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2]))
+```
+
+* 创建订阅
+```python
+# 创建一个主题为 'test' 消费周期为1000毫秒的订阅
+# 第一个参数为 True 表示重新开始订阅，如为 False 且之前创建过主题为 'test' 的订阅，则表示继续消费此订阅的数据，而不是重新开始消费所有数据
+sub = conn.subscribe(True, "test", "select * from tb;", 1000)
+```
+
+* 消费订阅的数据
+```python
+data = sub.consume()
+for d in data:
+    print(d)
+```
+
+* 取消订阅
+```python
+sub.close()
+```
+
+
+* 关闭连接
+```python
+c1.close()
+conn.close()
+```
+#### 帮助信息
+
+用户可通过python的帮助信息直接查看模块的使用信息，或者参考tests/examples/python中的示例程序。以下为部分常用类和方法：
+
+- _TDengineConnection_类
+
+  参考python中help(taos.TDengineConnection)。
+  这个类对应客户端和TDengine建立的一个连接。在客户端多线程的场景下，这个连接实例可以是每个线程申请一个，也可以多线程共享一个连接。
+
+- _TDengineCursor_类
+
+  参考python中help(taos.TDengineCursor)。
+  这个类对应客户端进行的写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能夸线程共享使用，否则会导致返回结果出现错误。
+
+- _connect_方法
+
+  用于生成taos.TDengineConnection的实例。
+
+
+### Python客户端使用示例代码
+在tests/examples/python中，我们提供了一个示例Python程序read_example.py，可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后，通过import taos引入taos类。主要步骤如下
+
+- 通过taos.connect获取TDengineConnection对象，这个对象可以一个程序只申请一个，在多线程中共享。
+- 通过TDengineConnection对象的 .cursor()方法获取一个新的游标对象，这个游标对象必须保证每个线程独享。
+- 通过游标对象的execute()方法，执行写入或查询的SQL语句
+- 如果执行的是写入语句，execute返回的是成功写入的行数信息affected rows
+- 如果执行的是查询语句，则execute执行成功后，需要通过fetchall方法去拉取结果集。
+具体方法可以参考示例代码。
+
+## RESTful Connector
+
+为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。 
+
+### HTTP请求格式 
+
+```
+http://<ip>:<PORT>/rest/sql
+```
+
+参数说明：
+
+- IP: 集群中的任一台主机
+- PORT: 配置文件中httpPort配置项，缺省为6020
+
+例如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. 
+
+HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。
+
+- 自定义身份认证信息如下所示（<token>稍后介绍）
+
+```
+Authorization: Taosd <TOKEN>
+```
+
+- Basic身份认证信息如下所示
+
+```
+Authorization: Basic <TOKEN>
+```
+
+HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应提供数据库前缀，例如\<db-name>.\<tb-name>。如果表名不带数据库前缀，系统会返回错误。因为HTTP模块只是一个简单的转发，没有当前DB的概念。 
+
+使用curl通过自定义身份认证方式来发起一个HTTP Request，语法如下：
+
+```
+curl -H 'Authorization: Basic <TOKEN>' -d '<SQL>' <ip>:<PORT>/rest/sql
+```
+
+或者
+
+```
+curl -u username:password -d '<SQL>' <ip>:<PORT>/rest/sql
+```
+
+其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串，例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==`
+
+### HTTP返回格式
+
+返回值为JSON格式，如下:
+
+```
+{
+    "status": "succ",
+    "head": ["Time Stamp","current", …],
+    "data": [
+        ["2018-10-03 14:38:05.000", 10.3, …],
+        ["2018-10-03 14:38:15.000", 12.6, …]
+    ],
+    "rows": 2
+} 
+```
+
+说明：
+
+- status: 告知操作结果是成功还是失败
+- head: 表的定义，如果不返回结果集，仅有一列“affected_rows”
+- data: 具体返回的数据，一排一排的呈现，如果不返回结果集，仅[[affected_rows]]
+- rows: 表明总共多少行数据
+
+### 自定义授权码
+
+HTTP请求中需要带有授权码`<TOKEN>`，用于身份识别。授权码通常由管理员提供，可简单的通过发送`HTTP GET`请求来获取授权码，操作如下：
+
+```
+curl http://<ip>:6020/rest/login/<username>/<password>
+```
+
+其中，`ip`是TDengine数据库的IP地址，`username`为数据库用户名，`password`为数据库密码，返回值为`JSON`格式，各字段含义如下：
+
+- status：请求结果的标志位
+
+- code：返回值代码
+
+- desc: 授权码
+
+获取授权码示例：
+
+```
+curl http://192.168.0.1:6020/rest/login/root/taosdata
+```
+
+返回值：
+
+```
+{
+  "status": "succ",
+  "code": 0,
+  "desc": "/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"
+}
+```
+
+### 使用示例
+
+- 在demo库里查询表d1001的所有记录： 
+
+```
+curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6020/rest/sql`
+```
+返回值：
+
+```
+{
+    "status": "succ",
+    "head": ["Time Stamp","current","voltage","phase"],
+    "data": [
+        ["2018-10-03 14:38:05.000",10.3,219,0.31],
+        ["2018-10-03 14:38:15.000",12.6,218,0.33]
+    ],
+    "rows": 2
+}
+```
+
+- 创建库demo：
+
+```
+curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql`
+```
+
+返回值：
+```
+{
+    "status": "succ",
+    "head": ["affected_rows"],
+    "data": [[1]],
+    "rows": 1,
+}
+```
+
+### 其他用法
+
+#### 结果集采用Unix时间戳
+
+HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如
+
+```
+curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6020/rest/sqlt
+```
+
+返回值：
+
+```
+{
+    "status": "succ",
+    "head": ["column1","column2","column3"],
+    "data": [
+        [1538548685000,10.3,219,0.31],
+        [1538548695000,12.6,218,0.33]
+    ],
+    "rows": 2
+}
+```
+
+#### 结果集采用UTC时间字符串
+
+HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如
+```
+  curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc
+```
+
+返回值：
+
+```
+{
+    "status": "succ",
+    "head": ["column1","column2","column3"],
+    "data": [
+        ["2018-10-03T14:38:05.000+0800",10.3,219,0.31],
+        ["2018-10-03T14:38:15.000+0800",12.6,218,0.33]
+    ],
+    "rows": 2
+}
+```
+
+### 重要配置项
+
+下面仅列出一些与RESTFul接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效
+
+- httpPort: 对外提供RESTFul服务的端口号，默认绑定到6020
+- httpMaxThreads: 启动的线程数量，默认为2
+- restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240
+- httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式
+- httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131
+
+
+## Go Connector
+
+TDengine提供了GO驱动程序`taosSql`. `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengin, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`
+
+```Go
+import (
+    "database/sql"
+    _ "github.com/taosdata/driver-go/taoSql"
+)
+```
+### 常用API
+* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` 
+	
+	该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine
+	
+   **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。 
+	 
+*  `func (db *DB) Exec(query string, args ...interface{}) (Result, error)`
+
+  sql.Open内置的方法，用来执行非查询相关SQL
+  
+  
+*  `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)`
+  
+  sql.Open内置的方法，用来执行查询语句
+  
+		
+## Node.js Connector
+
+TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)：
+
+首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器.
+
+```cmd
+npm install td2.0-connector
+```
+我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下
+
+我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
+
+### Linux
+
+- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
+- `node`  必须采用v10.x版本，其他版本存在包兼容性的问题。
+- `make`
+- c语言编译器比如[GCC](https://gcc.gnu.org)
+
+### macOS
+
+- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
+
+- Xcode
+
+  - 然后通过Xcode安装
+
+    ```
+    Command Line Tools
+    ```
+
+    在
+    ```
+    Xcode -> Preferences -> Locations
+    ```
+
+    目录下可以找到这个工具。或者在终端里执行
+
+    ```
+    xcode-select --install
+    ```
+
+
+    - 该步执行后 `gcc` 和 `make`就被安装上了
+
+### Windows
+
+#### 安装方法1
+
+使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
+
+#### 安装方法2
+
+手动安装以下工具:
+
+- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) 
+- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` 
+- 进入`cmd`命令行界面, `npm config set msvs_version 2017`
+
+如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules)
+
+如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual  C++ ATL for ARM64".
+
+### 使用方法
+
+(http://docs.taosdata.com/node)
+以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node)
+
+#### 连接
+
+使用node.js连接器时，必须先<em>require</em> ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信 
+
+```javascript
+const taos = require('td2.0-connector');
+var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0})
+var cursor = conn.cursor(); // Initializing a new cursor
+```
+
+关闭连接可执行
+
+```javascript
+conn.close();
+```
+
+#### 查询
+
+可通过 ```cursor.query``` 函数来查询数据库。
+
+```javascript
+var query = cursor.query('show databases;')
+```
+
+查询的结果可以通过 ```query.execute()``` 函数获取并打印出来
+
+```javascript
+var promise = query.execute();
+promise.then(function(result) {
+  result.pretty(); 
+});
+```
+格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。
+
+```javascript
+var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5);
+query.execute().then(function(result) {
+  result.pretty();
+})
+```
+如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下：
+
+
+```javascript
+var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true)
+promise.then(function(result) {
+  result.pretty();
+})
+```
+#### 异步函数
+异步查询数据库的操作和上面类似，只需要在`cursor.execute`, `TaosQuery.execute`等函数后面加上`_a`。
+```javascript
+var promise1 = cursor.query('select count(*), avg(v1), avg(v2) from meter1;').execute_a()
+var promise2 = cursor.query('select count(*), avg(v1), avg(v2) from meter2;').execute_a();
+promise1.then(function(result) {
+  result.pretty();
+})
+promise2.then(function(result) {
+  result.pretty();
+})
+```
+
+
+### 示例
+[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
+
+[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.
+
+
diff --git a/documentation20/faq-ch.md b/documentation20/faq-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..574e69ccd9cc36e21fec896a49332bb5b6a2a167
--- /dev/null
+++ b/documentation20/faq-ch.md
@@ -0,0 +1,74 @@
+#常见问题
+
+#### 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么？☆☆☆
+
+2.0版本在之前版本的基础上，进行了完全的重构，配置文件和数据文件是不兼容的。在升级之前务必进行如下操作：
+
+1. 删除配置文件，执行 <code> sudo rm -rf /etc/taos/taos.cfg </code>
+
+2. 删除日志文件，执行 <code> sudo rm -rf /var/log/taos/ </code>
+
+3. 确保数据已经不再需要的前提下，删除数据文件，执行 <code> sudo rm -rf /var/lib/taos/ </code>
+
+4. 安装最新稳定版本的TDengine
+5.  如果数据需要迁移数据或者数据文件损坏，请联系涛思数据官方技术支持团队，进行协助解决
+
+#### 2. <a href='blog/2019/12/03/jdbcdriver找不到动态链接库/'>Windows平台下JDBCDriver找不到动态链接库</a>
+
+#### 3. <a href='blog/2019/12/03/创建数据表时提示more-dnodes-are-needed/'>创建数据表时提示more dnodes are needed</a>
+
+#### 4. <a href='blog/2019/12/06/tdengine-crash时生成core文件的方法/'>TDengine crash时生成core文件的方法</a>
+
+#### 5. 遇到错误"failed to connect to server", 我怎么办？
+
+客户端遇到链接故障，请按照下面的步骤进行检查：
+
+1. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用
+2. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd*
+3. 确认客户端连接时指定了正确的服务器IP地址
+4. ping服务器IP，如果没有反应，请检查你的网络
+5. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的
+6. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
+7. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+8. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
+   检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
+   检查服务器侧TCP端口连接是否工作：`nc -l {port}`
+   检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}`
+
+
+#### 6. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误
+
+如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。
+
+#### 7. 是否支持validation queries?
+
+TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。
+
+#### 8. 我可以删除或更新一条记录吗？
+
+不能。因为TDengine是为联网设备采集的数据设计的，不容许修改。但TDengine提供数据保留策略，只要数据记录超过保留时长，就会被自动删除。
+
+#### 10. 我怎么创建超过250列的表？
+
+使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。
+
+#### 10. 最有效的写入数据的方法是什么？
+
+批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的记录。
+
+#### 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？
+
+windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下：
+
+​      Class.forName("com.taosdata.jdbc.TSDBDriver");
+
+​      Properties properties = new Properties();
+
+​      properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8");
+
+​      Connection = DriverManager.getConnection(url, properties);
+
+#### 12. <a href='blog/2020/01/06/tdengine-go-windows驱动的编译/'>TDengine GO windows驱动的编译</a>
+
+
+
diff --git a/documentation20/faq.md b/documentation20/faq.md
new file mode 100644
index 0000000000000000000000000000000000000000..ec0bc2957a32ec4c5f47756a801194d70bb3167f
--- /dev/null
+++ b/documentation20/faq.md
@@ -0,0 +1,46 @@
+#FAQ
+
+#### 1. How to upgrade TDengine from 1.X versions to 2.X and above versions?
+
+Version 2.X is a complete refactoring of the previous version, and configuration files and data files are incompatible. Be sure to do the following before upgrading:
+
+1. Delete the configuration file, and execute <code>sudo rm -rf /etc/taos/taos</code>
+2. Delete the log file, and execute <code>sudo rm -rf /var/log/taos </code>
+3. ENSURE THAT YOUR DATAS ARE NO LONGER NEEDED! Delete the data file, and execute <code>sudo rm -rf /var/lib/taos </code>
+4. Enjoy the latest stable version of TDengine
+5. If the data needs to be migrated or the data file is corrupted, please contact the official technical support team for assistance
+
+#### 2. When encoutered with the error "failed to connect to server", what can I do?
+
+The client may encounter connection errors. Please follow the steps below for troubleshooting:
+
+1. Make sure that the client and server version Numbers are exactly the same, and that the open source community and Enterprise versions are not mixed.
+2. On the server side, execute `systemctl status taosd` to check the status of *taosd* service. If *taosd* is not running, start it and retry connecting.
+3. Make sure you have used the correct server IP address to connect to.
+4. Ping the server. If no response is received, check your network connection.
+5. Check the firewall setting, make sure the TCP/UDP ports from 6030-6039 are enabled.
+6. For JDBC, ODBC, Python, Go connections on Linux, make sure the native library *libtaos.so* are located at /usr/local/lib/taos, and /usr/local/lib/taos is in the *LD_LIBRARY_PATH*. 
+7. For JDBC, ODBC, Python, Go connections on Windows, make sure *driver/c/taos.dll* is in the system search path (or you can copy taos.dll into *C:\Windows\System32*)
+8. If the above steps can not help, try the network diagnostic tool *nc* to check if TCP/UDP port works
+      check UDP port：`nc -vuz {hostIP} {port} `
+      check TCP port on server:  `nc -l {port}`
+      check TCP port on client: ` nc {hostIP} {port}`
+
+#### 3. Why I get "Invalid SQL" error when a query is syntactically correct?
+
+If you are sure your query has correct syntax, please check the length of the SQL string. Before version 2.0, it shall be less than 64KB. 
+
+#### 4. Does TDengine support validation queries?
+
+For the time being, TDengine does not have a specific set of validation queries. However, TDengine comes with a system monitoring database named 'sys', which can usually be used as a validation query object. 
+
+#### 5. Can I delete or update a record that has been written into TDengine?
+
+The answer is NO. The design of TDengine is based on the assumption that records are generated by the connected devices, you won't be allowed to change it. But TDengine provides a retention policy, the data records will be removed once their lifetime is passed.
+
+#### 6. What is the most efficient way to write data to TDengine?
+
+TDengine supports several different writing regimes. The most efficient way to write data to TDengine is to use batch inserting. For details on batch insertion syntax, please refer to [Taos SQL](../documentation/taos-sql)
+
+
+
diff --git a/documentation20/insert-ch.md b/documentation20/insert-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..fcc061674246da3c5b5389e3a1fdcf67b36ccccb
--- /dev/null
+++ b/documentation20/insert-ch.md
@@ -0,0 +1,218 @@
+# 高效写入数据
+
+TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。
+
+## SQL写入
+
+应用通过C/C++, JDBC, GO, 或Python Connector 执行SQL insert语句来插入数据，用户还可以通过TAOS Shell，手动输入SQL insert语句插入数据。比如下面这条insert 就将一条记录写入到表d1001中：
+```mysql
+INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31);
+```
+TDengine支持一次写入多条记录，比如下面这条命令就将两条记录写入到表d1001中：
+```mysql
+INSERT INTO d1001 VALUES (1538548684000, 10.2, 220, 0.23) (1538548696650, 10.3, 218, 0.25);
+```
+
+TDengine也支持一次向多个表写入数据，比如下面这条命令就向d1001写入两条记录，向d1002写入一条记录：
+```mysql
+INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31);
+```
+
+**Tips:** 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置）。
+
+详细的SQL INSERT语法规则请见TAOS SQL
+
+##Prometheus直接写入
+[Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
+
+###从源代码编译blm_prometheus
+用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
+- Linux操作系统的服务器
+- 安装好Golang, 1.10版本以上
+- 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器）
+
+Bailongma项目中有一个文件夹blm_prometheus，存放了prometheus的写入API程序。编译过程如下：
+```
+cd blm_prometheus
+go build
+```
+
+一切正常的情况下，就会在对应的目录下生成一个blm_prometheus的可执行程序。
+
+###安装Prometheus
+通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/)
+
+### 配置Prometheus
+参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的<remote_write>部分，增加以下配置
+
+- url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节
+  启动Prometheus后，可以通过taos客户端查询确认数据是否成功写入。
+
+### 启动blm_prometheus程序
+blm_prometheus程序有以下选项，在启动blm_prometheus程序时可以通过设定这些选项来设定blm_prometheus的配置。
+```sh
+--tdengine-name
+如果TDengine安装在一台具备域名的服务器上，也可以通过配置TDengine的域名来访问TDengine。在K8S环境下，可以配置成TDengine所运行的service name
+
+--batch-size 
+blm_prometheus会将收到的prometheus的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。
+
+--dbname
+设置在TDengine中创建的数据库名称，blm_prometheus会自动在TDengine中创建一个以dbname为名称的数据库，缺省值是prometheus。
+
+--dbuser
+设置访问TDengine的用户名，缺省值是'root'
+
+--dbpassword
+设置访问TDengine的密码，缺省值是'taosdata'
+
+--port
+blm_prometheus对prometheus提供服务的端口号。
+```
+
+###启动示例
+
+通过以下命令启动一个blm_prometheus的API服务
+```
+./blm_prometheus -port 8088
+```
+假设blm_prometheus所在服务器的IP地址为"10.1.2.3"，则在prometheus的配置文件中<remote_write>部分增加url为
+```yaml
+remote_write:
+  - url: "http://10.1.2.3:8088/receive"
+```
+
+###查询prometheus写入数据
+prometheus产生的数据格式如下：
+```
+Timestamp: 1576466279341,
+Value: 37.000000, 
+apiserver_request_latencies_bucket {
+  component="apiserver", 
+  instance="192.168.99.116:8443", 
+  job="kubernetes-apiservers", 
+  le="125000", 
+  resource="persistentvolumes", s
+  cope="cluster",
+  verb="LIST", 
+  version=“v1" 
+}
+```
+其中，apiserver_request_latencies_bucket为prometheus采集的时序数据的名称，后面{}中的为该时序数据的标签。blm_prometheus会以时序数据的名称在TDengine中自动创建一个超级表，并将{}中的标签转换成TDengine的tag值，Timestamp作为时间戳，value作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。
+```
+use prometheus;
+select * from apiserver_request_latencies_bucket;
+```
+
+##Telegraf直接写入
+[Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具，TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
+
+###从源代码编译blm_telegraf
+用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
+
+- Linux操作系统的服务器
+- 安装好Golang, 1.10版本以上
+- 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器）
+
+Bailongma项目中有一个文件夹blm_telegraf，存放了Telegraf的写入API程序。编译过程如下：
+
+```
+cd blm_telegraf
+go build
+```
+
+一切正常的情况下，就会在对应的目录下生成一个blm_telegraf的可执行程序。
+
+###安装Telegraf
+目前TDengine支持Telegraf 1.7.4以上的版本。用户可以根据当前的操作系统，到Telegraf官网下载安装包，并执行安装。下载地址如下：https://portal.influxdata.com/downloads
+
+### 配置Telegraf
+修改Telegraf配置文件/etc/telegraf/telegraf.conf中与TDengine有关的配置项。 
+
+在output plugins部分，增加[[outputs.http]]配置项： 
+
+- url： bailongma API服务提供的URL, 参考下面的启动示例章节
+- data_format: "json"
+- json_timestamp_units:      "1ms"
+
+在agent部分：
+
+- hostname: 区分不同采集设备的机器名称，需确保其唯一性
+- metric_batch_size: 100，允许Telegraf每批次写入记录最大数量，增大其数量可以降低Telegraf的请求发送频率。
+
+关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的[文档](https://docs.influxdata.com/telegraf/v1.11/)。
+
+###启动blm_telegraf程序
+blm_telegraf程序有以下选项，在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。
+
+```sh
+--host 
+TDengine服务端的IP地址，缺省值为空
+
+--batch-size 
+blm_telegraf会将收到的telegraf的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。
+
+--dbname
+设置在TDengine中创建的数据库名称，blm_telegraf会自动在TDengine中创建一个以dbname为名称的数据库，缺省值是prometheus。
+
+--dbuser
+设置访问TDengine的用户名，缺省值是'root'
+
+--dbpassword
+设置访问TDengine的密码，缺省值是'taosdata'
+
+--port
+blm_telegraf对telegraf提供服务的端口号。
+```
+
+###启动示例
+通过以下命令启动一个blm_telegraf的API服务
+```
+./blm_telegraf -host 127.0.0.1 -port 8089
+```
+
+假设blm_telegraf所在服务器的IP地址为"10.1.2.3"，则在telegraf的配置文件中, 在output plugins部分，增加[[outputs.http]]配置项： 
+
+```yaml
+url = "http://10.1.2.3:8089/telegraf"
+```
+
+###查询telegraf写入数据
+telegraf产生的数据格式如下：
+```
+{
+  "fields": {
+    "usage_guest": 0, 
+    "usage_guest_nice": 0,
+    "usage_idle": 89.7897897897898, 
+    "usage_iowait": 0,
+    "usage_irq": 0,
+    "usage_nice": 0,
+    "usage_softirq": 0,
+    "usage_steal": 0,
+    "usage_system": 5.405405405405405, 
+    "usage_user": 4.804804804804805
+  },
+  
+  "name": "cpu", 
+  "tags": {
+    "cpu": "cpu2",
+    "host": "bogon" 
+  },
+  "timestamp": 1576464360 
+}
+```
+
+其中，name字段为telegraf采集的时序数据的名称，tags字段为该时序数据的标签。blm_telegraf会以时序数据的名称在TDengine中自动创建一个超级表，并将tags字段中的标签转换成TDengine的tag值，Timestamp作为时间戳，fields字段中的值作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。
+
+```
+use telegraf;
+select * from cpu;
+```
+
+## EMQ X MQTT Broker直接写入
+
+MQTT是一流行的物联网数据传输协议，[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ里做简单配置，即可将MQTT的数据直接写入TDengine。
+
+
+
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index 2d4ebb4095795fc8565f46736d0e928ce11e596c..3a03972e68d662603b4d49a7f640f6aa9e05ff92 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -6,13 +6,10 @@
 ########################################################
 
 # first full-qualified domain name (FQDN) for TDengine system
-# first                 hostname1:6030
+# firstEp               hostname1:6030
 
 # second full-qualified domain name (FQDN) for TDengine system, for cluster edition only
-# second                cluster_hostname2:6030
-
-# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only   
-# arbitrator            arbitrator_hostname:6030     
+# secondEp              cluster_hostname2:6030
 
 # the full-qualified domain name (FQDN) of dnode 
 # fqdn                  hostname
@@ -23,45 +20,54 @@
 # http service port, default tcp [6020]
 # httpPort              6020
 
-# data file's directory
-# dataDir               /var/lib/taos
-
 # log file's directory
 # logDir                /var/log/taos
 
-# number of management nodes in the system
-# numOfMnodes           3
+# data file's directory
+# dataDir               /var/lib/taos
 
-# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode
-# role                  0
+# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only   
+# arbitrator            arbitrator_hostname:6030     
 
 # number of threads per CPU core
 # numOfThreadsPerCore   1.0
 
-# max number of vgroups per db
-# maxVgroupsPerDb       0
-
-# max number of tables per vnode
-# maxTablesPerVnode     1000000
-
 # the ratio of threads responsible for querying in the total thread
 # ratioOfQueryThreads   0.5
 
-# interval of DNode report status to MNode, unit is Second, for cluster version only 
-# statusInterval        1
+# number of management nodes in the system
+# numOfMnodes           3
 
-# interval of Shell send HB to MNode, unit is Second
-# shellActivityTimer    3
+# if backup vnode directory when remove dnode
+# vnodeBak              1
+
+# Whether to start load balancing
+# balance               1
+
+# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode
+# role                  0
+
+# max timer control block
+# maxTmrCtrl            512
 
 # interval of system monitor 
 # monitorInterval       30
 
+# number of seconds allowed for a dnode to be offline, for cluster version only 
+# offlineThreshold      8640000
+
 # RPC re-try timer, millisecond
 # rpcTimer              300
 
 # RPC maximum time for ack, seconds
 # rpcMaxTime            600
 
+# interval of DNode report status to MNode, unit is Second, for cluster version only 
+# statusInterval        1
+
+# interval of Shell send HB to MNode, unit is Second
+# shellActivityTimer    3
+
 # duration of to keep tableMeta kept in Cache, seconds
 # tableMetaKeepTimer    7200
 
@@ -71,38 +77,38 @@
 # Time window minimum
 # minIntervalTime       10
 
-# max length of an SQL
-# maxSQLLength          65480
-
-# Support the maximum number of records allowed for super table time sorting
-# maxNumOfOrderedRes    100000
+# the max allowed delayed time for launching continuous query. 20ms by default
+# maxStreamCompDelay    20000
 
-# system locale
-# locale                en_US.UTF-8
+# The minimum time to wait before the first stream execution
+# maxFirstStreamCompDelay   10000
 
-# default system charset
-# charset               UTF-8
+# Retry wait time benchmark
+# retryStreamCompDelay      10
 
-# system time zone
-# timezone              Asia/Shanghai (CST, +0800)
+# the delayed time for launching each continuous query. 10% of the whole computing time window by default.
+# streamCompDelayRatio      0.1
 
-# set write ahead log (WAL) level
-# walLevel              1
+# max number of vgroups per db
+# maxVgroupsPerDb           0
 
-# enable/disable async log
-# asyncLog              1
+# max number of tables per vnode
+# maxTablesPerVnode         1000000
 
-# the compressed rpc message, option:
-#  -1 (no compression)
-#   0 (all message compressed),
-# > 0 (rpc message body which larger than this value will be compressed)
-# compressMsgSize       -1
+# Step size of increasing table number in vnode
+# tableIncStepPerVnode      1000
 
 # cache block size (Mbyte)
-# cache                 16
+# cache                     16
 
 # number of cache blocks per vnode
-# blocks                4
+# blocks                    6
+
+# number of days per DB file
+# days                  10
+
+# number of days to keep DB file
+# keep                  3650
 
 # min row of records in file block
 # minRows               100
@@ -111,17 +117,44 @@
 # maxRows               4096
 
 # enable/disable compression
-# comp                  1
+# comp                  2
 
-# number of days per DB file
-# days                  10
+# set write ahead log (WAL) level
+# walLevel              1
 
-# number of days to keep DB file
-# keep                  3650
+# When walLevel is set to 2, the cycle of fsync is executed
+# fsync                 3000
 
 # number of replications, for cluster version only 
 # replica               1
 
+# mqtt uri  
+# mqttBrokerAddress     mqtt://username:password@hostname:1883/taos/
+
+# mqtt client name 
+# mqttBrokerClientId    taos_mqtt
+
+# the compressed rpc message, option:
+#  -1 (no compression)
+#   0 (all message compressed),
+# > 0 (rpc message body which larger than this value will be compressed)
+# compressMsgSize       -1
+
+# max length of an SQL
+# maxSQLLength          65480
+
+# Support the maximum number of records allowed for super table time sorting
+# maxNumOfOrderedRes    100000
+
+# system time zone
+# timezone              Asia/Shanghai (CST, +0800)
+
+# system locale
+# locale                en_US.UTF-8
+
+# default system charset
+# charset               UTF-8
+
 # max number of connections from client for dnode
 # maxShellConns         5000
 
@@ -137,50 +170,36 @@
 # Stop writing data when the disk size of the log folder is less than this value
 # minimalDataDirGB      0.1
 
-# number of seconds allowed for a dnode to be offline, for cluster version only 
-# offlineThreshold      8640000
-
 # start http service
 # http                  1
 
-# start system monitor module
-# monitor               1
-
 # start muqq service
 # mqtt                  0
 
-# mqtt uri  
-# mqttBrokerAddress     mqtt://username:password@hostname:1883/taos/
-
-# mqtt client name 
-# mqttBrokerClientId    taos_mqtt
+# start system monitor module
+# monitor               1
 
-# maximum number of rows returned by the restful interface
-# restfulRowLimit       10240
+# Record the SQL through restful interface
+# httpEnableRecordSql   0
 
 # number of threads used to process http requests
 # httpMaxThreads        2
 
-# The minimum time to wait before the first stream execution
-# maxFirstStreamCompDelay   10000
-
-# Retry wait time benchmark
-# retryStreamCompDelay      10
-
-# the delayed time for launching each continuous query. 10% of the whole computing time window by default.
-# streamCompDelayRatio  0.1
-
-# the max allowed delayed time for launching continuous query. 20ms by default
-# maxStreamCompDelay    20000
+# maximum number of rows returned by the restful interface
+# restfulRowLimit       10240
 
 # The following parameter is used to limit the maximum number of lines in log files.
 # max number of rows per log filters
 # numOfLogLines         10000000
 
+# enable/disable async log
+# asyncLog              1
+
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
-# 131: output warning and error，135: output info, warning and error to log.
-# 199: output debug, info, warning and error to both screen and file
+# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log.
+# 199: output debug, warning and error to both screen and file
+# 207: output trace, debug, warning and error to both screen and file
 
 # debug flag for all log type, take effect when non-zero value
 # debugFlag             0
@@ -236,9 +255,6 @@
 # debug flag for http server
 # tsdbDebugFlag         131
 
-# Record the SQL through restful interface
-# httpEnableRecordSql   0
-
 # Record the SQL in taos client
 # tscEnableRecordSql    0
 
diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord
new file mode 100644
index 0000000000000000000000000000000000000000..bf4dba7258b8333e1ecff1733661b69562896e45
--- /dev/null
+++ b/packaging/deb/tarbitratord
@@ -0,0 +1,88 @@
+#!/bin/bash 
+#
+# Modified from original source: Elastic Search
+# https://github.com/elasticsearch/elasticsearch
+# Thank you to the Elastic Search authors
+#
+# chkconfig: 2345 99 01
+#
+### BEGIN INIT INFO
+# Provides:          TDEngine
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:     $local_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts TDEngine tarbitrator
+# Description:       Starts TDEngine tarbitrator, a arbitrator
+### END INIT INFO
+
+set -e
+
+PATH="/bin:/usr/bin:/sbin:/usr/sbin"
+NAME="tarbitrator"
+USER="root"
+GROUP="root"
+DAEMON="/usr/local/taos/bin/tarbitrator"
+DAEMON_OPTS=""
+PID_FILE="/var/run/$NAME.pid"
+APPARGS=""
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+. /lib/lsb/init-functions
+
+case "$1" in
+    start)
+
+        log_action_begin_msg "Starting tarbitrator..."
+        if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then
+
+            touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE"
+
+            if [ -n "$MAX_OPEN_FILES" ]; then
+                ulimit -n $MAX_OPEN_FILES
+            fi
+
+            start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS
+
+            log_end_msg $?
+        fi
+        ;;
+
+    stop)
+        log_action_begin_msg "Stopping tarbitrator..."
+        set +e
+        if [ -f "$PID_FILE" ]; then
+            start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null
+            if [ $? -eq 1 ]; then
+                log_action_cont_msg "TSD is not running but pid file exists, cleaning up"
+            elif [ $? -eq 3 ]; then
+                PID="`cat $PID_FILE`"
+                log_failure_msg "Failed to stop tarbitrator (pid $PID)"
+                exit 1
+            fi
+            rm -f "$PID_FILE"
+        else
+            log_action_cont_msg "tarbitrator was not running"
+        fi
+        log_action_end_msg 0
+        set -e
+        ;;
+
+    restart|force-reload)
+        if [ -f "$PID_FILE" ]; then
+            $0 stop
+            sleep 1
+        fi
+        $0 start
+        ;;
+    status)
+        status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME"
+        ;;
+    *)
+        exit 1
+        ;;
+esac
+
+exit 0
diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord
new file mode 100644
index 0000000000000000000000000000000000000000..5d3f2b9c79794435a21e33cc669e70f9ae9f41a0
--- /dev/null
+++ b/packaging/rpm/tarbitratord
@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# tarbitratord      This shell script takes care of starting and stopping tarbitrator.
+#
+# chkconfig: 2345 99 01
+# description: tarbitrator is a arbitrator used in TDengine cluster.
+#
+#
+### BEGIN INIT INFO
+# Provides: TDEngine
+# Required-Start: $network $local_fs $remote_fs
+# Required-Stop: $network $local_fs $remote_fs
+# Short-Description: start and stop taosd
+# Description: tarbitrator is a arbitrator used in TDengine cluster.
+### END INIT INFO
+
+# Source init functions
+. /etc/init.d/functions
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+# Default program options
+NAME=tarbitrator
+PROG=/usr/local/taos/bin/tarbitrator
+USER=root
+GROUP=root
+
+# Default directories
+LOCK_DIR=/var/lock/subsys
+PID_DIR=/var/run/$NAME
+
+# Set file names
+LOCK_FILE=$LOCK_DIR/$NAME
+PID_FILE=$PID_DIR/$NAME.pid
+
+[ -e $PID_DIR ] || mkdir -p $PID_DIR
+
+PROG_OPTS=""
+
+start() {
+    echo -n "Starting ${NAME}: "
+    # check identity
+    curid="`id -u -n`"
+    if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then
+        echo "Must be run as root or $USER, but was run as $curid"
+        return 1
+    fi
+    # Sets the maximum number of open file descriptors allowed.
+    ulimit -n $MAX_OPEN_FILES
+    curulimit="`ulimit -n`"
+    if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then
+        echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit"
+        return 1
+    fi
+
+    if [ "`id -u -n`" == root ] ; then
+        # Changes the owner of the lock, and the pid files to allow
+        # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py.
+        touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE
+        touch $PID_FILE && chown $USER:$GROUP $PID_FILE
+        daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
+    else
+        # Don't have to change user.
+        daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
+    fi
+    retval=$?
+    sleep 2
+    echo
+    [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE)
+    return $retval
+}
+
+stop() {
+    echo -n "Stopping ${NAME}: "
+    killproc -p $PID_FILE $NAME
+    retval=$?
+    echo
+    # Non-root users don't have enough permission to remove pid and lock files.
+    # So, the opentsdb_restart.py cannot get rid of the files, and the command
+    # "service opentsdb status" will complain about the existing pid file.
+    # Makes the pid file empty.
+    echo > $PID_FILE
+    [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE)
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status -p $PID_FILE -l $LOCK_FILE $NAME
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+
+exit $?
diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh
index c573a2086f05ace5b496bde9618f0bdc45026cf1..47a8febe35c299b2934df75334890cc340eadc34 100644
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@@ -99,6 +99,44 @@ else
   os_type=1
 fi
 
+
+# =============================  get input parameters =================================================
+
+# install.sh -v [server | client]  -e [yes | no] -i [systemd | service | ...]
+
+# set parameters by default value
+interactiveFqdn=yes   # [yes | no]
+verType=server        # [server | client]
+initType=systemd      # [systemd | service | ...]
+
+while getopts "hv:e:i:" arg
+do
+  case $arg in
+    e)
+      #echo "interactiveFqdn=$OPTARG"
+      interactiveFqdn=$( echo $OPTARG )
+      ;;
+    v)
+      #echo "verType=$OPTARG"
+      verType=$(echo $OPTARG)
+      ;;
+    i)
+      #echo "initType=$OPTARG"
+      initType=$(echo $OPTARG)
+      ;;
+    h)
+      echo "Usage: `basename $0` -v [server | client]  -e [yes | no]"
+      exit 0
+      ;;
+    ?) #unknow option 
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
+
 function kill_taosd() {
   pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
   if [ -n "$pid" ]; then
@@ -129,15 +167,17 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
     ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
+    ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
 
     ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/*
 
     #Make link
-    [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos             || :
-    [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd          || :
-    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
-    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
-    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
+    [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                      || :
+    [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd                   || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump          || :
+    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo          || :
+    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos          || :
+    [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
 
     if [ "$verMode" == "cluster" ]; then
         ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/*
@@ -189,50 +229,36 @@ function install_config() {
         if ((${update_flag}==1)); then
             return 0
         fi
+        
+        if [ "$interactiveFqdn" == "no" ]; then
+            return 0
+        fi
 
-        IP_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
-        IP_PATTERN="\b$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\b"
+        #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+        #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)"
+        #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)"
+        #FQDN_PATTERN=":[0-9]{1,5}$"
 
+        # first full-qualified domain name (FQDN) for TDengine cluster system
         echo
-        echo -e -n "${GREEN}Enter the IP address of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :"
-        read masterIp
+        echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join OR leave it blank to build one${NC} :"
+        read firstEp
         while true; do
-            if [ ! -z "$masterIp" ]; then
-                # check the format of the masterIp
-                if [[ $masterIp =~ $IP_PATTERN ]]; then
-                    # Write the first IP to configuration file
-                    sudo sed -i -r "s/#*\s*(masterIp\s*).*/\1$masterIp/" ${cfg_dir}/taos.cfg
-
-                    # Get the second IP address
-
-                    echo
-                    echo -e -n "${GREEN}Enter the IP address of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: "
-                    read secondIp
-                    while true; do
-
-                        if [ ! -z "$secondIp" ]; then
-                            if [[ $secondIp =~ $IP_PATTERN ]]; then
-                                # Write the second IP to configuration file
-                                sudo sed -i -r "s/#*\s*(secondIp\s*).*/\1$secondIp/" ${cfg_dir}/taos.cfg
-                                break
-                            else
-                                read -p "Please enter the correct IP address: " secondIp
-                            fi
-                        else
-                            break
-                        fi
-                    done
-    
+            if [ ! -z "$firstEp" ]; then
+                # check the format of the firstEp
+                #if [[ $firstEp == $FQDN_PATTERN ]]; then
+                    # Write the first FQDN to configuration file                    
+                    ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg    
                     break
-                else
-                    read -p "Please enter the correct IP address: " masterIp
-                fi
+                #else
+                #    read -p "Please enter the correct FQDN:port: " firstEp
+                #fi
             else
                 break
             fi
         done
 	
-	fi
+	  fi
 }
 
 
@@ -266,16 +292,24 @@ function clean_service_on_sysvinit() {
     if pidof taosd &> /dev/null; then
         ${csudo} service taosd stop || :
     fi
+    
+    if pidof tarbitrator &> /dev/null; then
+        ${csudo} service tarbitratord stop || :
+    fi
 
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --del taosd || :
+        ${csudo} chkconfig --del tarbitratord || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv -r taosd || :
+        ${csudo} insserv -r tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d -f taosd remove || :
+        ${csudo} update-rc.d -f tarbitratord remove || :
     fi
     
     ${csudo} rm -f ${service_config_dir}/taosd || :
+    ${csudo} rm -f ${service_config_dir}/tarbitratord || :
     
     if $(which init &> /dev/null); then
         ${csudo} init q || :
@@ -292,9 +326,13 @@ function install_service_on_sysvinit() {
     if ((${os_type}==1)); then
         ${csudo} cp -f ${script_dir}/init.d/taosd.deb ${install_main_dir}/init.d/taosd
         ${csudo} cp    ${script_dir}/init.d/taosd.deb ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd
+        ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
+        ${csudo} cp    ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord
     elif ((${os_type}==2)); then
         ${csudo} cp -f ${script_dir}/init.d/taosd.rpm ${install_main_dir}/init.d/taosd
         ${csudo} cp    ${script_dir}/init.d/taosd.rpm ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd
+        ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
+        ${csudo} cp    ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord
     fi
     
     #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"
@@ -303,11 +341,16 @@ function install_service_on_sysvinit() {
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --add taosd || :
         ${csudo} chkconfig --level 2345 taosd on || :
+        ${csudo} chkconfig --add tarbitratord || :
+        ${csudo} chkconfig --level 2345 tarbitratord on || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv taosd || :
         ${csudo} insserv -d taosd || :
+        ${csudo} insserv tarbitratord || :
+        ${csudo} insserv -d tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d taosd defaults || :
+        ${csudo} update-rc.d tarbitratord defaults || :
     fi
 }
 
@@ -331,8 +374,17 @@ function clean_service_on_systemd() {
         fi
         ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null
 	
-        ${csudo} rm -f ${nginx_service_config}
-	fi
+        ${csudo} rm -f ${nginx_service_config}              
+
+        tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
+        if systemctl is-active --quiet tarbitratord; then
+            echo "tarbitrator is running, stopping it..."
+            ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null
+        fi
+        ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null
+
+        ${csudo} rm -f ${tarbitratord_service_config}
+	  fi
 }
 
 # taos:2345:respawn:/etc/init.d/taosd start
@@ -364,28 +416,52 @@ function install_service_on_systemd() {
     ${csudo} systemctl enable taosd
 
     if [ "$verMode" == "cluster" ]; then		
+    
+        tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
+
+        ${csudo} bash -c "echo '[Unit]'                                  >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'After=network-online.target'             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Wants=network-online.target'             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo                                           >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo '[Service]'                               >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Type=simple'                             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator'          >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitNOFILE=infinity'                    >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitNPROC=infinity'                     >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitCORE=infinity'                      >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'TimeoutStartSec=0'                       >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StandardOutput=null'                     >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Restart=always'                          >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StartLimitBurst=3'                       >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StartLimitInterval=60s'                  >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo                                           >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo '[Install]'                               >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'WantedBy=multi-user.target'              >> ${tarbitratord_service_config}"
+        ${csudo} systemctl enable tarbitratord  
+    
         nginx_service_config="${service_config_dir}/nginxd.service"
-        ${csudo} bash -c "echo '[Unit]'                             >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'After=network-online.target'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Wants=network-online.target'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo                                      >> ${nginx_service_config}"
-        ${csudo} bash -c "echo '[Service]'                          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Type=forking'                       >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx'          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop'   >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitNOFILE=infinity'               >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitNPROC=infinity'                >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitCORE=infinity'                 >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'TimeoutStartSec=0'                  >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StandardOutput=null'                >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Restart=always'                     >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StartLimitBurst=3'                  >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StartLimitInterval=60s'             >> ${nginx_service_config}"
-        ${csudo} bash -c "echo                                      >> ${nginx_service_config}"
-        ${csudo} bash -c "echo '[Install]'                          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'WantedBy=multi-user.target'         >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Unit]'                                             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Description=Nginx For TDengine Service'             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'After=network-online.target'                        >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Wants=network-online.target'                        >> ${nginx_service_config}"
+        ${csudo} bash -c "echo                                                      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Service]'                                          >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Type=forking'                                       >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid'           >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx'             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop'      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitNOFILE=infinity'                               >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitNPROC=infinity'                                >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitCORE=infinity'                                 >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'TimeoutStartSec=0'                                  >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StandardOutput=null'                                >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Restart=always'                                     >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StartLimitBurst=3'                                  >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StartLimitInterval=60s'                             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo                                                      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Install]'                                          >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'WantedBy=multi-user.target'                         >> ${nginx_service_config}"
         if ! ${csudo} systemctl enable nginxd &> /dev/null; then
             ${csudo} systemctl daemon-reexec
             ${csudo} systemctl enable nginxd
@@ -489,7 +565,7 @@ function update_TDengine() {
         install_service
         install_config
 		
-		if [ "$verMode" == "cluster" ]; then    
+		    if [ "$verMode" == "cluster" ]; then    
             # Check if openresty is installed
             openresty_work=false
 
@@ -502,7 +578,7 @@ function update_TDengine() {
                     echo -e "\033[44;31;5mNginx for TDengine does not work! Please try again!\033[0m"
                 fi
             fi
-		fi 
+		    fi 
 
         echo
         echo -e "\033[44;32;1mTDengine is updated successfully!${NC}"
@@ -583,8 +659,8 @@ function install_TDengine() {
         install_config	
 
         # Ask if to start the service
-        echo
-        echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
+        #echo
+        #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
         echo
         echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg"
         if ((${service_mod}==0)); then
@@ -600,13 +676,16 @@ function install_TDengine() {
                 echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
            else
                 echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
-            fi
-		else
+           fi
+		    else
             echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
         fi
 		
+		    echo		    
+		    echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}"
         echo
         echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
+        echo       
     else # Only install client
         install_bin
         install_config
@@ -620,7 +699,7 @@ function install_TDengine() {
 
 
 ## ==============================Main program starts from here============================
-if [ -z $1 ]; then
+if [ "$verType" == "server" ]; then
     # Install server and client
     if [ -x ${bin_dir}/taosd ]; then
         update_flag=1
@@ -628,7 +707,8 @@ if [ -z $1 ]; then
     else
         install_TDengine
     fi
-else
+elif [ "$verType" == "client" ]; then
+    interactiveFqdn=no
     # Only install client
     if [ -x ${bin_dir}/taos ]; then
         update_flag=1
@@ -636,4 +716,6 @@ else
     else
         install_TDengine client
     fi
+else 
+    echo  "please input correct verType"   
 fi
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index c214626efd3765db29702cce88ac3f9362d355ef..1cb412d9dd227d791cd904e247bf86237d59263b 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taos
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -54,14 +54,18 @@ nginx_dir="${code_dir}/../../enterprise/src/plugins/web"
 # temp use rpm's taosd. TODO: later modify according to os type
 init_file_deb=${script_dir}/../deb/taosd
 init_file_rpm=${script_dir}/../rpm/taosd
+init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord
+init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord
 
 # make directories.
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg
-mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/*
+mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
 mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/taosd.deb
 mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taosd.rpm
+mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || :
+mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || :
 
 if [ "$verMode" == "cluster" ]; then
     sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >> remove_temp.sh
diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh
index 28cc835f3057bf37bcd157aeb744fffa89771d0b..9fb8731449ec4db6a56e617febd6e073119e3f21 100755
--- a/packaging/tools/remove.sh
+++ b/packaging/tools/remove.sh
@@ -26,6 +26,7 @@ v15_java_app_dir="/usr/local/lib/taos"
 
 service_config_dir="/etc/systemd/system"
 taos_service_name="taosd"
+tarbitrator_service_name="tarbitratord"
 nginx_service_name="nginxd"
 csudo=""
 if command -v sudo > /dev/null; then
@@ -59,6 +60,12 @@ function kill_taosd() {
   fi
 }
 
+function kill_tarbitrator() {
+  pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo} kill -9 $pid   || :
+  fi
+}
 function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/taos      || :
@@ -102,9 +109,18 @@ function clean_service_on_systemd() {
     ${csudo} rm -f ${taosd_service_config}
 
     if [ "$verMode" == "cluster" ]; then
-		nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
-	
-   	 	if [ -d ${bin_dir}/web ]; then
+        tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
+
+        if systemctl is-active --quiet ${tarbitrator_service_name}; then
+            echo "TDengine tarbitrator is running, stopping it..."
+            ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
+        fi
+        ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
+
+        ${csudo} rm -f ${tarbitratord_service_config}
+        
+		    nginx_service_config="${service_config_dir}/${nginx_service_name}.service"	
+   	 	  if [ -d ${bin_dir}/web ]; then
    	        if systemctl is-active --quiet ${nginx_service_name}; then
    	            echo "Nginx for TDengine is running, stopping it..."
    	            ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null
@@ -124,16 +140,25 @@ function clean_service_on_sysvinit() {
         echo "TDengine taosd is running, stopping it..."
         ${csudo} service taosd stop || :
     fi
-
+    
+    if pidof tarbitrator &> /dev/null; then
+        echo "TDengine tarbitrator is running, stopping it..."
+        ${csudo} service tarbitratord stop || :
+    fi
+    
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --del taosd || :
+        ${csudo} chkconfig --del tarbitratord || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv -r taosd || :
+        ${csudo} insserv -r tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d -f taosd remove || :
+        ${csudo} update-rc.d -f tarbitratord remove || :
     fi
     
     ${csudo} rm -f ${service_config_dir}/taosd || :
+    ${csudo} rm -f ${service_config_dir}/tarbitratord || :
    
     if $(which init &> /dev/null); then
         ${csudo} init q || :
@@ -148,6 +173,7 @@ function clean_service() {
     else
         # must manual stop taosd
         kill_taosd
+        kill_tarbitrator
     fi
 }
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 57d723437981205458d32357c8a05ce9ba46e451..ede66d95bb5699ef0480e3168227b218372bc8b1 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -10,6 +10,8 @@ ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)
 ADD_SUBDIRECTORY(kit)
 ADD_SUBDIRECTORY(plugins)
+ADD_SUBDIRECTORY(sync)
+ADD_SUBDIRECTORY(balance)
 ADD_SUBDIRECTORY(mnode)
 ADD_SUBDIRECTORY(vnode)
 ADD_SUBDIRECTORY(tsdb)
diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..45b7c4ed57c678d28bfd5d9e2734b2e5ff724fe3
--- /dev/null
+++ b/src/balance/CMakeLists.txt
@@ -0,0 +1,16 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sdb/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  ADD_LIBRARY(balance ${SRC})
+ENDIF ()
diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
new file mode 100644
index 0000000000000000000000000000000000000000..7aed8c05ca075d0f8adf58a1c8c4809cc293fc95
--- /dev/null
+++ b/src/balance/src/balance.c
@@ -0,0 +1,1011 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "tutil.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+extern void *       tsMnodeTmr;
+static void *       tsBalanceTimer = NULL;
+static int32_t      tsBalanceDnodeListSize = 0;
+static SDnodeObj ** tsBalanceDnodeList = NULL;
+static int32_t      tsBalanceDnodeListMallocSize = 16;
+static pthread_mutex_t tsBalanceMutex;
+
+static void  balanceStartTimer(int64_t mseconds);
+static void  balanceInitDnodeList();
+static void  balanceCleanupDnodeList();
+static void  balanceAccquireDnodeList();
+static void  balanceReleaseDnodeList();
+static void  balanceMonitorDnodeModule();
+static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static void balanceLock() {
+  pthread_mutex_lock(&tsBalanceMutex);
+}
+
+static void balanceUnLock() {
+  pthread_mutex_unlock(&tsBalanceMutex);
+}
+
+static bool balanceCheckFree(SDnodeObj *pDnode) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+    mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
+    return false;
+  }
+  
+  if (pDnode->openVnodes >= TSDB_MAX_VNODES) {
+    mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES);
+    return false;
+  }
+
+  if (pDnode->diskAvailable <= tsMinimalDataDirGB) {
+    mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable);
+    return false;
+  }
+
+  if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) {
+    mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId);
+    return false;
+  }
+
+  return true;
+}
+
+static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+  mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
+
+  SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
+  if (pDnode != NULL) {
+    atomic_sub_fetch_32(&pDnode->openVnodes, 1);
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */
+  int32_t   numOfVnodes = 0;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i;
+    if (pTmpVodeGid == pVnodeGid) {
+      continue;
+    }
+    vnodeGid[numOfVnodes] = *pTmpVodeGid;
+    ++numOfVnodes;
+  }
+  memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
+  pVgroup->numOfVnodes = numOfVnodes;
+
+  mnodeUpdateVgroup(pVgroup);
+}
+
+static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+  // SVnodeGid tmp = *pVnodeGid1;
+  // *pVnodeGid1 = *pVnodeGid2;
+  // *pVnodeGid2 = tmp;
+}
+
+int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+  int32_t dnode = 0;
+  int32_t vnodes = 0;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
+      if (balanceCheckFree(pDnode)) {
+        SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
+        pVnodeGid->dnodeId = pDnode->dnodeId;
+        pVnodeGid->pDnode = pDnode;
+        dnode++;
+        vnodes++;
+        break;
+      }
+    }
+  }
+
+  if (vnodes != pVgroup->numOfVnodes) {
+    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
+           pVgroup->numOfVnodes, vnodes);
+    balanceReleaseDnodeList();
+    balanceUnLock();
+
+    if (mnodeGetOnlineDnodesNum() == 0) {
+      return TSDB_CODE_MND_NOT_READY;
+    } else {
+      return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+    }
+  }
+
+  /*
+   * make the choice more random.
+   * replica 1: no choice
+   * replica 2: there are 2 combinations
+   * replica 3 or larger: there are 6 combinations
+   */
+  if (pVgroup->numOfVnodes == 1) {
+  } else if (pVgroup->numOfVnodes == 2) {
+    if (rand() % 2 == 0) {
+      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+    }
+  } else {
+    int32_t randVal = rand() % 6;
+    if (randVal == 1) {  // 1, 0, 2
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+    } else if (randVal == 2) {  // 1, 2, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else if (randVal == 3) {  // 2, 1, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+    } else if (randVal == 4) {  // 2, 0, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    }
+    if (randVal == 5) {  // 0, 2, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else {
+    }  // 0, 1, 2
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+  return TSDB_CODE_SUCCESS;
+}
+
+static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+  if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
+    return false;
+  }
+
+  bool isReady = false;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
+    if (pVnode == pRmVnode) continue;
+
+    mTrace("vgId:%d, change vgroup status, dnode:%d status:%d", pVgroup->vgId, pVnode->pDnode->dnodeId,
+           pVnode->pDnode->status);
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
+
+    if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+      isReady = true;
+    }
+  }
+
+  return isReady;
+}
+
+/**
+ * desc: remove one vnode from vgroup
+ * all vnodes in vgroup should in ready state, except the balancing one
+ **/
+static void balanceRemoveVnode(SVgObj *pVgroup) {
+  if (pVgroup->numOfVnodes <= 1) return;
+
+  SVnodeGid *pRmVnode = NULL;
+  SVnodeGid *pSelVnode = NULL;
+  int32_t    maxScore = 0;
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]);
+    SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId);
+
+    if (pDnode == NULL) {
+      mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+      break;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else if (pVnode->dnodeId == pVgroup->lbDnodeId) {
+      mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else {
+      if (pSelVnode == NULL) {
+        pSelVnode = pVnode;
+        maxScore = pDnode->score;
+      } else {
+        if (maxScore < pDnode->score) {
+          pSelVnode = pVnode;
+          maxScore = pDnode->score;
+        }
+      }
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  if (pRmVnode != NULL) {
+    pSelVnode = pRmVnode;
+  }
+
+  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+    mDebug("vgId:%d, is not ready", pVgroup->vgId);
+  } else {
+    mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
+    balanceDiscardVnode(pVgroup, pSelVnode);
+  }
+}
+
+static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+ for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pGid = &pVgroup->vnodeGid[i];
+    if (pGid->dnodeId == 0) break;
+    if (pGid->dnodeId == pDnode->dnodeId) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * desc: add vnode to vgroup, find a new one if dest dnode is null
+ **/
+static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+  if (pDestDnode == NULL) {
+    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+      if (pDnode == pSrcDnode) continue;
+      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (!balanceCheckFree(pDnode)) continue;
+      
+      pDestDnode = pDnode;
+      mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
+      break;
+    }
+  }
+
+  if (pDestDnode == NULL) {
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes;
+  pVnodeGid->dnodeId = pDestDnode->dnodeId;
+  pVnodeGid->pDnode = pDestDnode;
+  pVgroup->numOfVnodes++;
+
+  if (pSrcDnode != NULL) {
+    pVgroup->lbDnodeId = pSrcDnode->dnodeId;
+  }
+
+  atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
+
+  mnodeUpdateVgroup(pVgroup);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static bool balanceMonitorBalance() {
+  if (tsBalanceDnodeListSize < 2) return false;
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[src];
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+           pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
+           pDnode->openVnodes);
+  }
+
+  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  if (scoresDiff < 0.01) {
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    return false;
+  }
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
+    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
+    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+    if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
+      continue;
+    }
+
+    void *pIter = NULL;
+    while (1) {
+      SVgObj *pVgroup;
+      pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+      if (pVgroup == NULL) break;
+
+      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+        for (int32_t dest = 0; dest < src; dest++) {
+          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
+          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+
+          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          if (srcScore + 0.0001 < destScore) continue;
+          if (!balanceCheckFree(pDestDnode)) continue;
+          
+          mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
+                 pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
+                 srcScore, pDestDnode->score, destScore);
+          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          mnodeDecVgroupRef(pVgroup);
+          sdbFreeIter(pIter);
+          return true;
+        }
+      }
+
+      mnodeDecVgroupRef(pVgroup);
+    }
+
+    sdbFreeIter(pIter);
+  }
+
+  return false;
+}
+
+// if mgmt changed to master
+// 1. reset balanceAccessSquence to zero
+// 2. reset state of dnodes to offline
+// 3. reset lastAccess of dnodes to zero
+void balanceReset() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    // while master change, should reset dnode to offline
+    mInfo("dnode:%d set access:%d to 0", pDnode->dnodeId, pDnode->lastAccess);
+    pDnode->lastAccess = 0;
+    if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
+      pDnode->status = TAOS_DN_STATUS_OFFLINE;
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  tsAccessSquence = 0;
+}
+
+static int32_t balanceMonitorVgroups() {
+  void *  pIter = NULL;
+  SVgObj *pVgroup = NULL;
+  bool    hasUpdatingVgroup = false;
+
+  while (1) {
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    int32_t dbReplica = pVgroup->pDb->cfg.replications;
+    int32_t vgReplica = pVgroup->numOfVnodes;
+    
+    if (vgReplica > dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceRemoveVnode(pVgroup);
+    } else if (vgReplica < dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceAddVnode(pVgroup, NULL, NULL);
+    }
+
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+
+  return hasUpdatingVgroup;
+}
+
+static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+  mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
+
+  void *  pIter = NULL;
+  bool    hasThisDnode = false;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    mnodeDecVgroupRef(pVgroup);
+
+    if (hasThisDnode) break;
+  }
+
+  sdbFreeIter(pIter);
+
+  if (!hasThisDnode) {
+    mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
+    mnodeDropDnode(pDnode, NULL);
+    return true;
+  }
+
+  return false;
+}
+
+static bool balanceMontiorDropping() {
+  void *pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    mnodeDecDnodeRef(pDnode);
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
+      if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; 
+      if (mnodeGetDnodesNum() <= 1) continue;
+
+      mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
+              tsAccessSquence - pDnode->lastAccess);
+
+      pDnode->status = TAOS_DN_STATUS_DROPPING;
+      mnodeUpdateDnode(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return true;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      bool ret = balanceMonitorDnodeDropping(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return ret;
+    }
+  }
+
+  sdbFreeIter(pIter);
+
+  return false;
+}
+
+static bool balanceStart() {
+  if (!sdbIsMaster()) return false;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  balanceMonitorDnodeModule();
+
+  bool updateSoon = balanceMontiorDropping();
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorVgroups();
+  }
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorBalance();
+  }
+ 
+  balanceReleaseDnodeList();
+
+  balanceUnLock();
+
+  return updateSoon;
+}
+
+static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+      if (pVgroup->vnodeGid[i].pDnode == pDnode) {
+        pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE;
+      }
+    }
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceCheckDnodeAccess() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (tsAccessSquence - pDnode->lastAccess > 3) {
+      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
+        pDnode->status = TAOS_DN_STATUS_OFFLINE;
+        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        balanceSetVgroupOffline(pDnode);
+      }
+    }
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+
+  tsBalanceTimer = NULL;
+  tsAccessSquence ++;
+
+  balanceCheckDnodeAccess();  
+  bool updateSoon = false;
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      updateSoon = balanceStart();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    updateSoon = balanceStart();
+  }
+
+  if (updateSoon) {
+    balanceStartTimer(1000);
+  } else {
+    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
+  }
+}
+
+static void balanceStartTimer(int64_t mseconds) {
+  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
+}
+
+void balanceSyncNotify() {
+  if (sdbIsMaster()) {
+    balanceLock();
+    balanceAccquireDnodeList();
+    balanceMonitorDnodeModule();
+    balanceReleaseDnodeList();
+    balanceUnLock();
+  }
+}
+
+void balanceAsyncNotify() {
+  balanceStartTimer(500); 
+}
+
+int32_t balanceInit() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
+  
+  pthread_mutex_init(&tsBalanceMutex, NULL);
+  balanceInitDnodeList();
+  balanceStartTimer(2000);
+  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
+
+  balanceReset();
+  
+  return 0;
+}
+
+void balanceCleanUp() {
+  if (tsBalanceTimer != NULL) {
+    taosTmrStopA(&tsBalanceTimer);
+    pthread_mutex_destroy(&tsBalanceMutex);
+    tsBalanceTimer = NULL;
+    mDebug("stop balance timer");
+  }
+  balanceCleanupDnodeList();
+}
+
+int32_t balanceDropDnode(SDnodeObj *pDnode) {
+  int32_t    totalFreeVnodes = 0;
+  void *     pIter = NULL;
+  SDnodeObj *pTempDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pTempDnode);
+    if (pTempDnode == NULL) break;
+
+    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+      totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
+    }
+
+    mnodeDecDnodeRef(pTempDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  if (pDnode->openVnodes > totalFreeVnodes) {
+    mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
+    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+  }
+
+  pDnode->status = TAOS_DN_STATUS_DROPPING;
+  mnodeUpdateDnode(pDnode);
+  
+  balanceStartTimer(1100);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float balanceCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+void balanceCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
+                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
+}
+
+float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                        balanceCalcBandwidthScore(pDnode);
+  float moduleScore = balanceCalcModuleScore(pDnode);
+  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+static void balanceInitDnodeList() {
+  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
+}
+
+static void balanceCleanupDnodeList() {
+  if (tsBalanceDnodeList != NULL) {
+    free(tsBalanceDnodeList);
+    tsBalanceDnodeList = NULL;
+  }
+}
+
+static void balanceCheckDnodeListSize(int32_t dnodesNum) {
+  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
+    tsBalanceDnodeListMallocSize = dnodesNum * 2;
+    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
+  }
+}
+
+void balanceAccquireDnodeList() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  balanceCheckDnodeListSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {  
+    if (dnodeIndex >= dnodesNum) break;
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    balanceCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
+        break;
+      }
+      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
+    }
+    tsBalanceDnodeList[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  sdbFreeIter(pIter);
+
+  tsBalanceDnodeListSize = dnodeIndex;
+}
+
+void balanceReleaseDnodeList() {
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0)  {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+  
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                      balanceCalcBandwidthScore(pDnode);
+    float moduleScore = balanceCalcModuleScore(pDnode);
+    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
+
+static void balanceMonitorDnodeModule() {
+  int32_t numOfMnodes = mnodeGetMnodesNum();
+  if (numOfMnodes >= tsNumOfMnodes) return;
+
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode == NULL) break;
+
+    if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      continue;
+    }
+
+    if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE) {
+      continue;
+    }
+
+    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, add mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
+    mnodeAddMnode(pDnode->dnodeId);
+    
+    numOfMnodes = mnodeGetMnodesNum();
+    if (numOfMnodes >= tsNumOfMnodes) return;
+  }
+}
+
+int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+  if (!sdbIsMaster()) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  if (tsEnableBalance != 0) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_BALANCE_ENABLED;
+  }
+
+  SVgObj *pVgroup = mnodeGetVgroup(vnodeId);
+  if (pVgroup == NULL) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_VGROUP_NOT_EXIST;
+  }
+
+  SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId);
+  if (pDestDnode == NULL) {
+    mnodeDecVgroupRef(pVgroup);
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  balanceLock();
+  balanceAccquireDnodeList();
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, pSrcDnode->dnodeId);
+    code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
+  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, dnodeId);
+    code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
+  } else if (!balanceCheckFree(pDestDnode)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
+           dnodeId);
+    code = TSDB_CODE_MND_DNODE_NOT_FREE;
+  } else {
+    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+
+  mnodeDecVgroupRef(pVgroup);
+  mnodeDecDnodeRef(pDestDnode);
+
+  return code;
+}
\ No newline at end of file
diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt
index 00fa1a1479e51c5120641041155b9aa8fc2ff189..eaebd2f19dfb77ffcd54d3d8d2a0dc597c8ed875 100644
--- a/src/client/CMakeLists.txt
+++ b/src/client/CMakeLists.txt
@@ -4,11 +4,11 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 INCLUDE_DIRECTORIES(jni)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 17840df4a4062002b54f4aea5293427a5a7a1c3b..7a44870938f943e1aec667f4ecf0890fd394ff43 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -31,8 +31,8 @@ extern "C" {
 #include "tutil.h"
 
 #include "qExecutor.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "tcmdtype.h"
 
 // forward declaration
@@ -60,15 +60,14 @@ typedef struct SCMCorVgroupInfo {
 } SCMCorVgroupInfo;
 
 typedef struct STableMeta {
-  STableComInfo tableInfo;
-  uint8_t       tableType;
-  int16_t       sversion;
-  int16_t       tversion;
+  STableComInfo  tableInfo;
+  uint8_t        tableType;
+  int16_t        sversion;
+  int16_t        tversion;
   SCMVgroupInfo  vgroupInfo;
   SCMCorVgroupInfo  corVgroupInfo;
-  int32_t       sid;       // the index of one table in a virtual node
-  uint64_t      uid;       // unique id of a table
-  SSchema       schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
+  STableId       id;
+  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
 } STableMeta;
 
 typedef struct STableMetaInfo {
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 9dd33e03cb21d8dfc807a8d8e1f91042621c7633..85cff4ba17859fe0ca48ce1e5b9a280765e0c79a 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -430,7 +430,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   pRes->code = code;
 
   if (code != TSDB_CODE_SUCCESS) {
-    tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code));
+    tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code));
     goto _error;
   } else {
     tscDebug("%p get tableMeta successfully", pSql);
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 72ccd5adc6ee835e9e25f9b9b8b2511cad797910..17f6c97ea16223d692d4b8d06c00856ab477d09b 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2131,6 +2131,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
 }
 
 bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
+  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo == NULL) {
+    return true;
+  }
+
   STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx);
   
   // required number of results are not reached, continue load data block
diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 80fc82d90b1856753c4c88268c650147fed9e124..bf76b8cbe8a9fe31ef139db1a8b6f266a4b13c3b 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -691,9 +691,15 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   
   pModel = createColumnModel(pSchema, size, capacity);
 
+  int32_t pg = DEFAULT_PAGE_SIZE;
+  int32_t overhead = sizeof(tFilePage);
+  while((pg - overhead) < pModel->rowSize * 2) {
+    pg *= 2;
+  }
+
   size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
   for (int32_t i = 0; i < numOfSubs; ++i) {
-    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
+    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel);
     (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
   }
 
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 95098bbab1867d08e8d5e337a7dd7601b9befbfb..ae2370cd5628a59ea89f868a929df073118cbf3f 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -104,7 +104,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
   } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) {
     // do nothing
   } else if (pToken->type == TK_INTEGER) {
-    useconds = str2int64(pToken->z);
+    useconds = tsosStr2int64(pToken->z);
   } else {
     // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
     if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
@@ -629,8 +629,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
 }
 
 static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
-  pBlocks->tid = pTableMeta->sid;
-  pBlocks->uid = pTableMeta->uid;
+  pBlocks->tid = pTableMeta->id.tid;
+  pBlocks->uid = pTableMeta->id.uid;
   pBlocks->sversion = pTableMeta->sversion;
   pBlocks->numOfRows += numOfRows;
 }
@@ -686,7 +686,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   
   STableDataBlocks *dataBuf = NULL;
-  int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
+  int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE,
                                         sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name,
                                         pTableMeta, &dataBuf);
   if (ret != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index f58431849c7165d19a115eec3d7c177a708ffcb8..44236face07bf0b6d654d7d91935f4e9ac25f6d2 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -635,7 +635,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
   int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    if (pTableMetaInfo->pTableMeta->uid == uid) {
+    if (pTableMetaInfo->pTableMeta->id.uid == uid) {
       tableIndex = i;
       break;
     }
@@ -3053,7 +3053,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
-  pLeft->uid = pTableMetaInfo->pTableMeta->uid;
+  pLeft->uid = pTableMetaInfo->pTableMeta->id.uid;
   pLeft->tagColId = pTagSchema1->colId;
   strcpy(pLeft->tableId, pTableMetaInfo->name);
 
@@ -3065,7 +3065,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
-  pRight->uid = pTableMetaInfo->pTableMeta->uid;
+  pRight->uid = pTableMetaInfo->pTableMeta->id.uid;
   pRight->tagColId = pTagSchema2->colId;
   strcpy(pRight->tableId, pTableMetaInfo->name);
 
@@ -3603,7 +3603,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex);
 
   STagCond* pTagCond = &pQueryInfo->tagCond;
-  pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->uid;
+  pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid;
 
   assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN);
 
@@ -3840,7 +3840,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     
     // add to source column list
     STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    int64_t uid = pTableMetaInfo->pTableMeta->uid;
+    int64_t uid = pTableMetaInfo->pTableMeta->id.uid;
     int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
     
     size_t num = taosArrayGetSize(colList);
@@ -4506,8 +4506,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
     pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
-    pUpdateMsg->tid       = htonl(pTableMeta->sid);
-    pUpdateMsg->uid       = htobe64(pTableMeta->uid);
+    pUpdateMsg->tid       = htonl(pTableMeta->id.tid);
+    pUpdateMsg->uid       = htobe64(pTableMeta->id.uid);
     pUpdateMsg->colId     = htons(pTagsSchema->colId);
     pUpdateMsg->type      = pTagsSchema->type;
     pUpdateMsg->bytes     = htons(pTagsSchema->bytes);
@@ -4992,6 +4992,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
   pMsg->compression = pCreateDb->compressionLevel;
   pMsg->walLevel = (char)pCreateDb->walLevel;
   pMsg->replications = pCreateDb->replica;
+  pMsg->quorum = pCreateDb->quorum;
   pMsg->ignoreExist = pCreateDb->ignoreExists;
 }
 
@@ -5045,7 +5046,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
 
     if (pExpr->functionId != TSDB_FUNC_TAG) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
       SColumnIndex    index = {.tableIndex = 0, .columnIndex = columnInfo};
       SSchema*        pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
 
@@ -5522,6 +5523,13 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 
+  if (pCreate->quorum != -1 &&
+      (pCreate->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCreate->quorum > TSDB_MAX_DB_REPLICA_OPTION)) {
+    snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum,
+             TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
+  }
+
   int32_t val = htonl(pCreate->daysPerFile);
   if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
     snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val,
diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c
index 9b8f48b109eb773a4808ec95231e175d613ec6c0..63607b7a86e0cce38c9a2f07f41b7055b538cb96 100644
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@@ -162,8 +162,8 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
     .numOfColumns = pTableMetaMsg->numOfColumns,
   };
   
-  pTableMeta->sid = pTableMetaMsg->sid;
-  pTableMeta->uid = pTableMetaMsg->uid;
+  pTableMeta->id.tid = pTableMetaMsg->sid;
+  pTableMeta->id.uid = pTableMetaMsg->uid;
   pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
 
   tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 6c1faf6f9d9cba9d2a66bc3c9070321d981f521d..521280af8762e1636e7fb8935619b7170cc1e875 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -605,9 +605,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     } 
 
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
-    pTableIdInfo->tid = htonl(pTableMeta->sid);
-    pTableIdInfo->uid = htobe64(pTableMeta->uid);
-    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
+    pTableIdInfo->tid = htonl(pTableMeta->id.tid);
+    pTableIdInfo->uid = htobe64(pTableMeta->id.uid);
+    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
     pMsg += sizeof(STableIdInfo);
@@ -640,7 +640,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
   }
   
   tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
-      pTableMeta->sid, pTableMeta->uid);
+      pTableMeta->id.tid, pTableMeta->id.uid);
   
   return pMsg;
 }
@@ -714,8 +714,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
     if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
         pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
-      tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
-          pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
+      tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
+          pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
                pColSchema->name);
 
       return TSDB_CODE_TSC_INVALID_SQL;
@@ -833,8 +833,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
       if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) ||
           (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) {
-        tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
-                 pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns,
+        tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
+                 pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, total, numOfTagColumns,
                  pCol->colIndex.columnIndex, pColSchema->name);
 
         return TSDB_CODE_TSC_INVALID_SQL;
@@ -855,7 +855,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
     STagCond* pTagCond = &pQueryInfo->tagCond;
     
-    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
+    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
     if (pCond != NULL && pCond->cond != NULL) {
       pQueryMsg->tagCondLen = htons(pCond->len);
       memcpy(pMsg, pCond->cond, pCond->len);
@@ -1739,7 +1739,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
+  tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name);
   free(pTableMeta);
   
   return TSDB_CODE_SUCCESS;
@@ -2215,7 +2215,7 @@ int tscRenewTableMeta(SSqlObj *pSql, char *tableId) {
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   if (pTableMetaInfo->pTableMeta) {
     tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
-             tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta);
+             tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta);
   }
 
   taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 8e554d62ffa1da2cdcefb9eaabf8ea400aef283f..6e572d94d11ee21f0b875e761b84659d74b7a07f 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -241,7 +241,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
-    SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
+    SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
     SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
@@ -334,7 +334,7 @@ void tscSaveSubscriptionProgress(void* sub) {
 
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
-  if (tmkdir(path, 0777) != 0) {
+  if (taosMkDir(path, 0777) != 0) {
     tscError("failed to create subscribe dir: %s", path);
   }
 
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 5d26d09fae1dcc6c947375004703a9e6621ec629..15e02799aabd8dea2cd7d7aa3c4342fb38b96080 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -180,7 +180,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
   pSupporter->limit = pQueryInfo->limit;
 
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index);
-  pSupporter->uid = pTableMetaInfo->pTableMeta->uid;
+  pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid;
   assert (pSupporter->uid != 0);
 
   getTmpfilePath("join-", pSupporter->path);
@@ -355,7 +355,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     // set the join condition tag column info, to do extract method
     if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       assert(pQueryInfo->tagCond.joinInfo.hasJoin);
-      int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+      int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
 
       pExpr->param[0].i64Key = colId;
       pExpr->numOfParams = 1;
@@ -499,7 +499,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
   // set the tags value for ts_comp function
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0);
-    int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+    int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid);
     pExpr->param->i64Key = tagColId;
     pExpr->numOfParams = 1;
   }
@@ -560,7 +560,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
   qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
 
   SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
 
@@ -1034,7 +1034,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
     int32_t tableIndexOfSub = -1;
     for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j);
-      if (pTableMetaInfo->pTableMeta->uid == pExpr->uid) {
+      if (pTableMetaInfo->pTableMeta->id.uid == pExpr->uid) {
         tableIndexOfSub = j;
         break;
       }
@@ -1205,7 +1205,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       STagCond* pTagCond = &pSupporter->tagCond;
       assert(pTagCond->joinInfo.hasJoin);
 
-      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->uid);
+      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
       SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
 
       int16_t bytes = 0;
@@ -1237,7 +1237,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0);
 
       if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-        int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+        int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid);
         pExpr->param->i64Key = tagColId;
         pExpr->numOfParams = 1;
       }
@@ -1505,12 +1505,11 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 
   SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql);
 
-  // todo add to async res or not??
   if (pNew == NULL) {
-    tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
-             trsupport->pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex);
+    tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d",
+             trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex);
 
-    pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    pParentSql->res.code = terrno;
     trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
 
     return pParentSql->res.code;
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 5662b4a8856e638ebb41c0e21b5e23647494224c..42356108b18dcbe6c78a619fd6f100ea493600c5 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -148,7 +148,7 @@ void taos_init_imp() {
   refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
   if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "client");
+    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
   }
 
   tscDebug("client is initialized successfully");
@@ -220,7 +220,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 
         if (strlen(tsLocale) == 0) { // locale does not set yet
           char* defaultLocale = setlocale(LC_CTYPE, "");
-          tstrncpy(tsLocale, defaultLocale, sizeof(tsLocale));
+          tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
         }
 
         // set the user specified locale
@@ -234,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
           tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale);
         }
 
-        tstrncpy(tsLocale, locale, sizeof(tsLocale));
+        tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -249,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
               tscInfo("charset changed from %s to %s", tsCharset, charset);
             }
 
-            tstrncpy(tsCharset, charset, sizeof(tsCharset));
+            tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -286,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
             tscInfo("charset changed from %s to %s", tsCharset, pStr);
           }
 
-          tstrncpy(tsCharset, pStr, sizeof(tsCharset));
+          tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscInfo("charset:%s not valid", pStr);
@@ -304,7 +304,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        tstrncpy(tsTimezone, pStr, sizeof(tsTimezone));
+        tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN);
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 17adc0c03d804280c43df0326a955cd29fd1aa60..73d6f0e5921dd5d26d8c4be16bf7a18c9bc8bb73 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -356,9 +356,9 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
   if (pObj->signature == pObj) {
-    pthread_mutex_lock(&pObj->mutex);
+    //pthread_mutex_lock(&pObj->mutex);
     tfree(pSql->sqlstr);
-    pthread_mutex_unlock(&pObj->mutex);
+    //pthread_mutex_unlock(&pObj->mutex);
   }
   
   tscFreeSqlResult(pSql);
@@ -955,7 +955,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
   pExpr->interBytes    = interSize;
   
   if (pTableMetaInfo->pTableMeta) {
-    pExpr->uid = pTableMetaInfo->pTableMeta->uid;
+    pExpr->uid = pTableMetaInfo->pTableMeta->id.uid;
   }
   
   return pExpr;
@@ -1482,7 +1482,7 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i
   int32_t k = -1;
 
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-    if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->uid == uid) {
+    if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->id.uid == uid) {
       k = i;
       break;
     }
@@ -1675,6 +1675,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
   if (pNew == NULL) {
     tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1688,6 +1689,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
 
     free(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1706,6 +1708,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1743,6 +1746,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1756,7 +1760,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery
   }
 
-  uint64_t uid = pTableMetaInfo->pTableMeta->uid;
+  uint64_t uid = pTableMetaInfo->pTableMeta->id.uid;
   tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true);
 
   int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo);
@@ -1827,8 +1831,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   if (pFinalInfo->pTableMeta == NULL) {
-    tscError("%p new subquery failed for get tableMeta is NULL from cache", pSql);
+    tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name);
     tscFreeSqlObj(pNew);
+
+    if (pPrevSql != NULL) {
+      assert(pPrevSql->res.code != TSDB_CODE_SUCCESS);
+      terrno = pPrevSql->res.code;
+    } else {
+      terrno = TSDB_CODE_TSC_APP_ERROR;
+    }
+
     return NULL;
   }
   
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 57a32f3d6930b5d323b81bd9873c0bda44cf83e6..e856481a9a5c84967910bb609fe5347b171bab7a 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -1,12 +1,11 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(common ${SRC})
-  TARGET_LINK_LIBRARIES(common tutil)
-ENDIF ()
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(inc)
+
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(common ${SRC})
+TARGET_LINK_LIBRARIES(common tutil)
diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h
index 2ed4b81204050ffe2c67b6f51929829d7403b557..68875341bae20bacab048412a81cd23256cb2f83 100644
--- a/src/common/inc/tdataformat.h
+++ b/src/common/inc/tdataformat.h
@@ -235,11 +235,11 @@ typedef struct {
   int maxPoints;  // max number of points
   int bufSize;
 
-  int      numOfRows;
-  int      numOfCols;  // Total number of cols
-  int      sversion;   // TODO: set sversion
-  void *   buf;
-  SDataCol cols[];
+  int       numOfRows;
+  int       numOfCols;  // Total number of cols
+  int       sversion;   // TODO: set sversion
+  void *    buf;
+  SDataCol *cols;
 } SDataCols;
 
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
@@ -249,13 +249,14 @@ typedef struct {
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
-void       tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
+int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 void       tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows);
+void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                              int limit2, int tRows);
 
 // ----------------- K-V data row structure
 /*
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index b9afef65d97d305db96b43a4f7a8bf90814b6419..b8d7297ee0caf3157890061fed757fbe7431231a 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -43,9 +43,9 @@ extern uint32_t tsMaxTmrCtrl;
 extern float    tsNumOfThreadsPerCore;
 extern float    tsRatioOfQueryThreads;
 extern int8_t   tsDaylight;
-extern char     tsTimezone[64];
-extern char     tsLocale[64];
-extern char     tsCharset[64];  // default encode string
+extern char     tsTimezone[];
+extern char     tsLocale[];
+extern char     tsCharset[];  // default encode string
 extern int32_t  tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
 
@@ -80,6 +80,7 @@ extern int16_t tsCompression;
 extern int16_t tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
+extern int32_t tsQuorum;
 
 // balance
 extern int32_t tsEnableBalance;
diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c
index 12ea4ad78d18a5f2b09daabb71093c153d30aca7..adfce5580e9eecb2e0168038d004e5f6cc934716 100644
--- a/src/common/src/tdataformat.c
+++ b/src/common/src/tdataformat.c
@@ -310,9 +310,15 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
 }
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
-  SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols);
+  SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols));
   if (pCols == NULL) return NULL;
 
+  pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
+  if (pCols->cols == NULL) {
+    tdFreeDataCols(pCols);
+    return NULL;
+  }
+
   pCols->maxRowSize = maxRowSize;
   pCols->maxCols = maxCols;
   pCols->maxPoints = maxRows;
@@ -320,15 +326,27 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
 
   pCols->buf = malloc(pCols->bufSize);
   if (pCols->buf == NULL) {
-    free(pCols);
+    tdFreeDataCols(pCols);
     return NULL;
   }
 
   return pCols;
 }
 
-void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
-  // assert(schemaNCols(pSchema) <= pCols->numOfCols);
+int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
+  if (schemaNCols(pSchema) > pCols->maxCols) {
+    pCols->maxCols = schemaNCols(pSchema);
+    pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
+    if (pCols->cols == NULL) return -1;
+  }
+
+  if (schemaTLen(pSchema) > pCols->maxRowSize) {
+    pCols->maxRowSize = schemaTLen(pSchema);
+    pCols->bufSize = schemaTLen(pSchema) * pCols->maxPoints;
+    pCols->buf = realloc(pCols->buf, pCols->bufSize);
+    if (pCols->buf == NULL) return -1;
+  }
+
   tdResetDataCols(pCols);
   pCols->numOfCols = schemaNCols(pSchema);
 
@@ -337,11 +355,14 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
     dataColInit(pCols->cols + i, schemaColAt(pSchema, i), &ptr, pCols->maxPoints);
     ASSERT((char *)ptr - (char *)(pCols->buf) <= pCols->bufSize);
   }
+  
+  return 0;
 }
 
 void tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     tfree(pCols->buf);
+    tfree(pCols->cols);
     free(pCols);
   }
 }
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 08349374b13ddbd041877d7006dc4a00727b5829..1a510299c1b73af8189775fdc7e5d9af442e7411 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -50,7 +50,7 @@ int32_t tsShellActivityTimer = 3;  // second
 float   tsNumOfThreadsPerCore = 1.0;
 float   tsRatioOfQueryThreads = 0.5;
 int8_t  tsDaylight = 0;
-char    tsTimezone[64] = {0};
+char    tsTimezone[TSDB_TIMEZONE_LEN] = {0};
 char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int32_t tsEnableCoreFile = 0;
@@ -109,6 +109,7 @@ int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
 int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
+int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = 100;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -296,7 +297,7 @@ static void doInitGlobalConfig() {
   SGlobalCfg cfg = {0};
   
   // ip address
-  cfg.option = "first";
+  cfg.option = "firstEp";
   cfg.ptr = tsFirst;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
@@ -306,7 +307,7 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "second";
+  cfg.option = "secondEp";
   cfg.ptr = tsSecond;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
@@ -742,6 +743,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "quorum";
+  cfg.ptr = &tsQuorum;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
+  cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "mqttBrokerAddress";
   cfg.ptr = tsMqttBrokerAddress;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
@@ -1338,4 +1349,4 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d
   }
 
   return true;
-}
\ No newline at end of file
+}
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
index f4c9ec765aab2f2fe29b958f56fcf38634fa9639..02d642d6430d8c2dccaa9be03258861ff051a374 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
@@ -260,50 +260,66 @@ public class TSDBDriver implements java.sql.Driver {
 		if (url.indexOf("?") != -1) {
 			String dbName = url.substring(0, url.indexOf("?"));
 			urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName);
-			url = url.trim().substring(1);
+			url = url.trim().substring(url.indexOf("?") + 1);
 		} else {
 			// without user & password so return
-			String dbName = url.trim();
-			urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName);
+			if(!url.trim().isEmpty()) {
+				String dbName = url.trim();
+				urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName);
+			}
 			this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty("user"));
 			return urlProps;
 		}
 
-		String[] queryStrings = url.trim().split("&");
 		String user = "";
+
+		if (url.indexOf("&") == -1) {
+			String[] kvPair = url.trim().split("=");
+			if (kvPair.length == 2) {
+				setPropertyValue(urlProps, kvPair);	
+				return urlProps;
+			} 
+		}
+
+		String[] queryStrings = url.trim().split("&");		
 		for (String queryStr : queryStrings) {
 			String[] kvPair = queryStr.trim().split("=");
 			if (kvPair.length < 2){
 				continue;
 			}
-			switch (kvPair[0].toLowerCase()) {
-				case PROPERTY_KEY_USER:
-					urlProps.setProperty(PROPERTY_KEY_USER, kvPair[1]);
-					user = kvPair[1];
-					break;
-				case PROPERTY_KEY_PASSWORD:
-					urlProps.setProperty(PROPERTY_KEY_PASSWORD, kvPair[1]);
-					break;
-                case PROPERTY_KEY_TIME_ZONE:
-                    urlProps.setProperty(PROPERTY_KEY_TIME_ZONE, kvPair[1]);
-                    break;
-                case PROPERTY_KEY_LOCALE:
-                    urlProps.setProperty(PROPERTY_KEY_LOCALE, kvPair[1]);
-                    break;
-                case PROPERTY_KEY_CHARSET:
-                    urlProps.setProperty(PROPERTY_KEY_CHARSET, kvPair[1]);
-                    break;
-                case PROPERTY_KEY_CONFIG_DIR:
-                    urlProps.setProperty(PROPERTY_KEY_CONFIG_DIR, kvPair[1]);
-                    break;
-			}
+			setPropertyValue(urlProps, kvPair);			
 		}
 
+		user = urlProps.getProperty(PROPERTY_KEY_USER).toString();
 		this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user);
 
 		return urlProps;
 	}
 
+	public void setPropertyValue(Properties property, String[] keyValuePair) {
+		switch (keyValuePair[0].toLowerCase()) {
+			case PROPERTY_KEY_USER:
+				property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]);					
+				break;
+			case PROPERTY_KEY_PASSWORD:
+				property.setProperty(PROPERTY_KEY_PASSWORD, keyValuePair[1]);
+				break;
+			case PROPERTY_KEY_TIME_ZONE:
+				property.setProperty(PROPERTY_KEY_TIME_ZONE, keyValuePair[1]);
+				break;
+			case PROPERTY_KEY_LOCALE:
+				property.setProperty(PROPERTY_KEY_LOCALE, keyValuePair[1]);
+				break;
+			case PROPERTY_KEY_CHARSET:
+				property.setProperty(PROPERTY_KEY_CHARSET, keyValuePair[1]);
+				break;
+			case PROPERTY_KEY_CONFIG_DIR:
+				property.setProperty(PROPERTY_KEY_CONFIG_DIR, keyValuePair[1]);
+				break;
+		}
+	}
+
+
 	public int getMajorVersion() {
 		return 1;
 	}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ea5a431a5ef6dca232467aa80d430c6ac33dd69
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
@@ -0,0 +1,47 @@
+package com.taosdata.jdbc;
+
+import org.junit.Test;
+
+import java.sql.SQLException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+public class TSDBDriverTest {
+
+    @Test
+    public void urlParserTest() throws SQLException {
+        TSDBDriver driver = new TSDBDriver();
+        String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password";
+
+        Properties properties = new Properties(); 
+        driver.parseURL(url, properties);
+        assertEquals(properties.get("host"), "127.0.0.1");
+        assertEquals(properties.get("port"), "0");
+        assertEquals(properties.get("dbname"), "db");                
+        assertEquals(properties.get("user"), "root");   
+        assertEquals(properties.get("password"), "your_password");
+
+        url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8";
+        properties = new Properties(); 
+        driver.parseURL(url, properties);
+        assertEquals(properties.get("host"), "127.0.0.1");
+        assertEquals(properties.get("port"), "0");
+        assertEquals(properties.get("dbname"), "log");        
+        assertEquals(properties.get("charset"), "UTF-8"); 
+        
+        url = "jdbc:TSDB://127.0.0.1:0/";
+        properties = new Properties(); 
+        driver.parseURL(url, properties);
+        assertEquals(properties.get("host"), "127.0.0.1");
+        assertEquals(properties.get("port"), "0"); 
+        assertEquals(properties.get("dbname"), null);
+        
+        url = "jdbc:TSDB://127.0.0.1:0/db";
+        properties = new Properties(); 
+        driver.parseURL(url, properties);
+        assertEquals(properties.get("host"), "127.0.0.1");
+        assertEquals(properties.get("port"), "0"); 
+        assertEquals(properties.get("dbname"), "db");
+    }
+}
\ No newline at end of file
diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt
index e8796306f3ef00893de91b9cb491215327d38012..4ccdb765c32926efa35e131322df2c584bd4c2b2 100644
--- a/src/cq/CMakeLists.txt
+++ b/src/cq/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(inc)
diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt
index 99c729dff46f695e945b747a499ed809c4da2b31..82781f531a30ace6066b1dc65500e06608cc054e 100644
--- a/src/cq/test/CMakeLists.txt
+++ b/src/cq/test/CMakeLists.txt
@@ -1,17 +1,12 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
-
-  LIST(APPEND CQTEST_SRC ./cqtest.c)
-  ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
-  TARGET_LINK_LIBRARIES(cqtest tcq)
-
-ENDIF ()
-
-
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/cq/inc)
+
+LIST(APPEND CQTEST_SRC ./cqtest.c)
+ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
+TARGET_LINK_LIBRARIES(cqtest tcq)
diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt
index de6e15e6b9fa8546ad232c16c8ac2c9472806cb4..f7c2961352a531cbaaee3eddafec2609059892fc 100644
--- a/src/dnode/CMakeLists.txt
+++ b/src/dnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
@@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync)
 
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)
@@ -26,10 +26,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     TARGET_LINK_LIBRARIES(taosd grant)
   ENDIF ()
 
-  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
   SET(PREPARE_ENV_CMD "prepare_env_cmd")
   SET(PREPARE_ENV_TARGET "prepare_env_target")
   ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c
index 901e0061e9e89035b322650d2f22881e574c1c38..251968487892811b6c43bd3e28df75c8da87c46c 100644
--- a/src/dnode/src/dnodeSystem.c
+++ b/src/dnode/src/dnodeSystem.c
@@ -20,7 +20,6 @@
 #include "tglobal.h"
 #include "dnodeInt.h"
 #include "dnodeMain.h"
-#include "tfile.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static sem_t exitSem;
@@ -40,7 +39,7 @@ int32_t main(int32_t argc, char *argv[]) {
         exit(EXIT_FAILURE);
       }
     } else if (strcmp(argv[i], "-V") == 0) {
-#ifdef _SYNC
+#ifdef _ACCT
       char *versionStr = "enterprise";
 #else
       char *versionStr = "community";
diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c
index bbea1a5e0bb52a56cfbf5a23dd29a17e094586a9..cb53bb5e60b1d4f0316c776644e7582af0579ebe 100644
--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@@ -49,7 +49,7 @@ static taos_qset       readQset;
 int32_t dnodeInitVnodeRead() {
   readQset = taosOpenQset();
 
-  readPool.min = 2;
+  readPool.min = tsNumOfCores;
   readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
   if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
   readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
@@ -206,10 +206,14 @@ static void *dnodeProcessReadQueue(void *param) {
            taosMsg[pReadMsg->rpcMsg.msgType], type);
     int32_t code = vnodeProcessRead(pVnode, pReadMsg);
 
-    if (type == TAOS_QTYPE_RPC) {
+    if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
       dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
     } else {
-      dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      if (code == TSDB_CODE_QRY_HAS_RSP) {
+        dnodeSendRpcReadRsp(pVnode, pReadMsg, TSDB_CODE_SUCCESS);
+      } else {
+        dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      }
     }
 
     taosFreeQitem(pReadMsg);
diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index dc09a03e1497cb52718279b542e3318fb2789d08..51bc8890fcd98b6f5dee4c88d2858990e68bc2a3 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -38,6 +38,8 @@ typedef struct {
 
 typedef struct {
   SRspRet  rspRet;
+  int32_t  processedCount;
+  int32_t  code;
   void    *pCont;
   int32_t  contLen;
   SRpcMsg  rpcMsg;
@@ -187,13 +189,16 @@ void dnodeFreeVnodeWqueue(void *wqueue) {
 void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
   SWriteMsg *pWrite = (SWriteMsg *)param;
 
-  if (code > 0) return;
+  if (code < 0) pWrite->code = code;
+  int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
+
+  if (count <= 1) return;
 
   SRpcMsg rpcRsp = {
     .handle  = pWrite->rpcMsg.handle,
     .pCont   = pWrite->rspRet.rsp,
     .contLen = pWrite->rspRet.len,
-    .code    = code,
+    .code    = pWrite->code,
   };
 
   rpcSendResponse(&rpcRsp);
@@ -239,7 +244,10 @@ static void *dnodeProcessWriteQueue(void *param) {
       }
 
       int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
-      if (pWrite) pWrite->rpcMsg.code = code;
+      if (pWrite) { 
+        pWrite->rpcMsg.code = code;
+        if (code <= 0) pWrite->processedCount = 1; 
+      }
     }
 
     walFsync(vnodeGetWal(pVnode));
diff --git a/src/inc/mnode.h b/src/inc/mnode.h
index 01358fdb447d93e9f68cb30376e753ed112fab25..03a25e0f92b881b6b1b9bcfcf6605257b776ae1b 100644
--- a/src/inc/mnode.h
+++ b/src/inc/mnode.h
@@ -42,6 +42,7 @@ typedef struct SMnodeMsg {
   int8_t    expected;
   int8_t    retry;
   int32_t   code;
+  void *    pObj;
   struct SAcctObj * pAcct;
   struct SDnodeObj *pDnode;
   struct SUserObj * pUser;
diff --git a/src/inc/query.h b/src/inc/query.h
index d201b649f9de6aa55e0b9f1e0aa8c9aff11a6092..ec1e458b628a60f0db86a48e9da9af7257454ed2 100644
--- a/src/inc/query.h
+++ b/src/inc/query.h
@@ -28,7 +28,7 @@ typedef void* qinfo_t;
  * @param qinfo
  * @return
  */
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo);
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo);
 
 
 /**
@@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs
  * @param qinfo
  * @return
  */
-void qTableQuery(qinfo_t qinfo);
+bool qTableQuery(qinfo_t qinfo);
 
 /**
  * Retrieve the produced results information, if current query is not paused or completed,
@@ -48,7 +48,7 @@ void qTableQuery(qinfo_t qinfo);
  * @param qinfo
  * @return
  */
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext);
 
 /**
  *
@@ -60,16 +60,14 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
  * @param contLen payload length
  * @return
  */
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
 
 /**
- * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
- * so it can be only called once for each retrieve
  *
  * @param qinfo
  * @return
  */
-bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
+void* qGetResultRetrieveMsg(qinfo_t qinfo);
 
 /**
  * kill current ongoing query and free query handle automatically
@@ -89,7 +87,7 @@ void  qQueryMgmtNotifyClosed(void* pExecutor);
 void  qCleanupQueryMgmt(void* pExecutor);
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo);
 void** qAcquireQInfo(void* pMgmt, uint64_t key);
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree);
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle);
 
 #ifdef __cplusplus
 }
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 1bb185e448cf7f6d01bc3eff055e7545d43c7451..c70ca8662eaf1b3b2c73f526f7892585632da189 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -250,7 +250,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_STATE_LEN            20
 #define TSDB_COUNTRY_LEN          20
 #define TSDB_LOCALE_LEN           64
-#define TSDB_TIMEZONE_LEN         64
+#define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
 
 #define TSDB_FQDN_LEN             128
@@ -338,6 +338,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MIN_DB_REPLICA_OPTION      1
 #define TSDB_MAX_DB_REPLICA_OPTION      3
 #define TSDB_DEFAULT_DB_REPLICA_OPTION  1
+#define TSDB_DEFAULT_DB_QUORUM_OPTION   1
 
 #define TSDB_MAX_JOIN_TABLE_NUM         5
 #define TSDB_MAX_UNION_CLAUSE           5
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index 1f03f1fe097930cf2fff8b30a0bfbb81515e2fb2..ece08ae173f171364943a94e1962f90d486f4811 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -46,197 +46,200 @@ static STaosError errors[] = {
 #endif
 
 // rpc
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS,       0, 0x0001, "action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED,            0, 0x0002, "auth required")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE,             0, 0x0003, "auth failure")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT,                 0, 0x0004, "redirect")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY,                0, 0x0005, "not ready")    // peer is not ready to process data
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED,        0, 0x0006, "message already processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED,0, 0x0007, "last session not finished")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID,       0, 0x0008, "mismatched meter id")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW,                 0, 0x0009, "too slow")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS,             0, 0x000A, "max sessions")    // too many sessions
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL,          0, 0x000B, "network unavailable")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR,                0, 0x000C, "rpc app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE,      0, 0x000D, "unexpected response")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE,            0, 0x000E, "invalid value")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID,          0, 0x000F, "invalid transaction id")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID,       0, 0x0010, "invalid session id")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE,         0, 0x0011, "invalid message type")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE,    0, 0x0012, "invalid response type")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP,       0, 0x0013, "invalid timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS,       0, 0x0001, "Action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED,            0, 0x0002, "Authentication required")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE,             0, 0x0003, "Authentication failure")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT,                 0, 0x0004, "Redirect")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY,                0, 0x0005, "System not ready")    // peer is not ready to process data
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED,        0, 0x0006, "Message already processed")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED,0, 0x0007, "Last session not finished")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID,       0, 0x0008, "Mismatched meter id")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW,                 0, 0x0009, "Processing of request timed out")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS,             0, 0x000A, "Number of sessions reached limit")    // too many sessions
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL,          0, 0x000B, "Unable to establish connection")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR,                0, 0x000C, "Unexpected generic error in RPC")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE,      0, 0x000D, "Unexpected response")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE,            0, 0x000E, "Invalid value")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID,          0, 0x000F, "Invalid transaction id")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID,       0, 0x0010, "Invalid session id")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE,         0, 0x0011, "Invalid message type")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE,    0, 0x0012, "Invalid response type")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP,       0, 0x0013, "Invalid timestamp")
 
 //common & util
-TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT,          0, 0x0100, "operations not support")
-TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED,         0, 0x0101, "memory corrupted")
-TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY,            0, 0x0102, "out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG,          0, 0x0103, "invalid config message")
-TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED,           0, 0x0104, "file is corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT,          0, 0x0100, "Operation not supported")
+TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED,         0, 0x0101, "Memory corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY,            0, 0x0102, "Out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG,          0, 0x0103, "Invalid config message")
+TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED,           0, 0x0104, "Data file corrupted")
 
 //client
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL,              0, 0x0200, "invalid sql")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE,          0, 0x0201, "client invalid qhandle")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP,       0, 0x0202, "client time/server time can not be mixed up")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE,            0, 0x0203, "clientinvalid value")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION,          0, 0x0204, "client invalid version")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE,               0, 0x0205, "client invalid ie")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN,             0, 0x0206, "client invalid fqdn")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH,      0, 0x0207, "client invalid username length")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH,      0, 0x0208, "client invalid password length")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH,        0, 0x0209, "client invalid database length")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH,  0, 0x020A, "client invalid table length")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION,       0, 0x020B, "client invalid connection")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY,            0, 0x020C, "client out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE,             0, 0x020D, "client no disk space")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED,       0, 0x020E, "client query cache erased")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED,          0, 0x020F, "client query cancelled")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY,      0, 0x0210, "client sorted res too many")      // too many result for ordered super table projection query
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR,                0, 0x0211, "client app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS,       0, 0x0212, "client action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED,             0, 0x0213, "client disconnected")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH,            0, 0x0214, "client no write auth")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL,              0, 0x0200, "Invalid SQL statement")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE,          0, 0x0201, "Invalid qhandle")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP,       0, 0x0202, "Invalid combination of client/service time")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE,            0, 0x0203, "Invalid value in client")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION,          0, 0x0204, "Invalid client version")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE,               0, 0x0205, "Invalid client ie")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN,             0, 0x0206, "Invalid host name")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH,      0, 0x0207, "Invalid user name")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH,      0, 0x0208, "Invalid password")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH,        0, 0x0209, "Database name too long")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH,  0, 0x020A, "Table name too long")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION,       0, 0x020B, "Invalid connection")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY,            0, 0x020C, "System out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE,             0, 0x020D, "System out of disk space")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED,       0, 0x020E, "Query cache erased")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED,          0, 0x020F, "Query terminated")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY,      0, 0x0210, "Result set too large to be sorted")      // too many result for ordered super table projection query
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR,                0, 0x0211, "Application error")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS,       0, 0x0212, "Action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED,             0, 0x0213, "Disconnected from service")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH,            0, 0x0214, "No write permission")
 
 // mnode
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED,        0, 0x0300, "mnode message not processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS,       0, 0x0301, "mnode action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED,  0, 0x0302, "mnode action need to be reprocessed")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS,                0, 0x0303, "mnode no rights")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR,                0, 0x0304, "mnode app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION,       0, 0x0305, "mnode invalid message connection")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION,      0, 0x0306, "mnode invalid message version")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN,          0, 0x0307, "mnode invalid message length")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE,         0, 0x0308, "mnode invalid message type")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS,     0, 0x0309, "mnode too many shell conns")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY,            0, 0x030A, "mnode out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ,          0, 0x030B, "mnode invalid show handle")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID,         0, 0x030C, "mnode invalid query id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID,        0, 0x030D, "mnode invalid stream id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID,          0, 0x030E, "mnode invalid connection")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE,    0, 0x0320, "sdb object already there")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR,                0, 0x0321, "sdb app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE,   0, 0x0322, "sdb invalid table type")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE,        0, 0x0323, "sdb object not there")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW,      0, 0x0324, "sdb invalid meta row")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE,      0, 0x0325, "sdb invalid key type")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST,      0, 0x0330, "dnode already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST,          0, 0x0331, "dnode not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         0, 0x0332, "vgroup not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER,         0, 0x0333, "cant not remove master")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES,         0, 0x0334, "no enough dnodes")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "cluster cfg inconsistent")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "invalid dnode cfg option")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED,          0, 0x0337, "balance already enabled")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE,      0, 0x0338, "vgroup not in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  0, 0x0339, "vgroup already in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE,           0, 0x033A, "dnode not avaliable")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       0, 0x033B, "cluster id not match")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       0, 0x0340, "mnode accounts already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT,             0, 0x0341, "mnode invalid account")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION,      0, 0x0342, "mnode invalid acct option")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST,       0, 0x0350, "mnode user already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER,             0, 0x0351, "mnode invalid user")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT,      0, 0x0352, "mnode invalid user format")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT,      0, 0x0353, "mnode invalid password format")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN,        0, 0x0354, "mnode can not get user from conn")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS,           0, 0x0355, "mnode too many users")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST,      0, 0x0360, "mnode table already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID,         0, 0x0361, "mnode invalid table id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME,       0, 0x0362, "mnode invalid table name")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE,       0, 0x0363, "mnode invalid table type")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS,            0, 0x0364, "mnode too many tags")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES,      0, 0x0366, "mnode not enough time series")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE,          0, 0x0367, "mnode no super table")           // operation only available for super table
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG,        0, 0x0368, "mnode column name too long")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST,         0, 0x0369, "mnode tag already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            0, 0x036A, "mnode tag not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST,       0, 0x036B, "mnode field already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          0, 0x036C, "mnode field not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME,      0, 0x036D, "mnode invalid stable name")
-
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED,          0, 0x0380, "mnode db not selected")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST,         0, 0x0381, "mnode database aleady exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION,        0, 0x0382, "mnode invalid db option")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB,               0, 0x0383, "mnode invalid database")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN,     0, 0x0384, "mnode monitor db forbidden")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES,       0, 0x0385, "mnode too many databases")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "mnode db in dropping")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED,        0, 0x0300, "Message not processed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS,       0, 0x0301, "Message is progressing")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED,  0, 0x0302, "Messag need to be reprocessed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS,                0, 0x0303, "Insufficient privilege for operation")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR,                0, 0x0304, "Unexpected generic error in mnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION,       0, 0x0305, "Invalid message connection")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION,      0, 0x0306, "Incompatible protocol version")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN,          0, 0x0307, "Invalid message length")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE,         0, 0x0308, "Invalid message type")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS,     0, 0x0309, "Too many connections")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY,            0, 0x030A, "Out of memory in mnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ,          0, 0x030B, "Data expired")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID,         0, 0x030C, "Invalid query id")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID,        0, 0x030D, "Invalid stream id")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID,          0, 0x030E, "Invalid connection id")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE,    0, 0x0320, "Object already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR,                0, 0x0321, "Unexpected generic error in sdb")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE,   0, 0x0322, "Invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE,        0, 0x0323, "Object not there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW,      0, 0x0324, "Invalid meta row")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE,      0, 0x0325, "Invalid key type")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST,      0, 0x0330, "DNode already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST,          0, 0x0331, "DNode does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         0, 0x0332, "VGroup does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER,         0, 0x0333, "Master DNode cannot be removed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES,         0, 0x0334, "Out of DNodes")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "Cluster cfg inconsistent")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "Invalid dnode cfg option")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED,          0, 0x0337, "Balance already enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE,      0, 0x0338, "Vgroup not in dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  0, 0x0339, "Vgroup already in dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE,           0, 0x033A, "Dnode not avaliable")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       0, 0x033B, "Cluster id not match")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY,                0, 0x033C, "Cluster not ready")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       0, 0x0340, "Account already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT,             0, 0x0341, "Invalid account")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION,      0, 0x0342, "Invalid account options")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST,       0, 0x0350, "User already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER,             0, 0x0351, "Invalid user")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT,      0, 0x0352, "Invalid user format")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT,      0, 0x0353, "Invalid password format")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN,        0, 0x0354, "Can not get user from conn")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS,           0, 0x0355, "Too many users")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST,      0, 0x0360, "Table already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID,         0, 0x0361, "Table name too long")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME,       0, 0x0362, "Table does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE,       0, 0x0363, "Invalid table type in tsdb")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS,            0, 0x0364, "Too many tags")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES,      0, 0x0366, "Too many time series")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE,          0, 0x0367, "Not super table")           // operation only available for super table
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG,        0, 0x0368, "Tag name too long")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST,         0, 0x0369, "Tag already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            0, 0x036A, "Tag does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST,       0, 0x036B, "Field already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          0, 0x036C, "Field does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME,      0, 0x036D, "Super table does not exist")
+
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED,          0, 0x0380, "Database not specified or available")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST,         0, 0x0381, "Database already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION,        0, 0x0382, "Invalid database options")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB,               0, 0x0383, "Invalid database name")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN,     0, 0x0384, "Cannot delete monitor database")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES,       0, 0x0385, "Too many databases for account")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database not available")
 
 // dnode
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "dnode message not processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            0, 0x0401, "dnode out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS,          0, 0x0402, "dnode no disk write access")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN,          0, 0x0403, "dnode invalid message length")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            0, 0x0401, "Dnode out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS,          0, 0x0402, "No permission for disk files in dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN,          0, 0x0403, "Invalid message length")
 
 // vnode 
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS,       0, 0x0500, "vnode action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED,        0, 0x0501, "vnode message not processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED,  0, 0x0502, "vnode action need to be reprocessed")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID,        0, 0x0503, "vnode invalid vgroup id")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED,              0, 0x0504, "vnode init failed")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE,             0, 0x0505, "vnode no diskspace")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "vnode no disk permissions")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "vnode no such file or directory")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "vnode out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "vnode app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_STATUS,           0, 0x0510, "vnode not in ready state")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "vnode not in synced state")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "vnode no write auth")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS,       0, 0x0500, "Action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED,        0, 0x0501, "Message not processed")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED,  0, 0x0502, "Action need to be reprocessed")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID,        0, 0x0503, "Invalid Vgroup ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED,              0, 0x0504, "Vnode initialization failed")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE,             0, 0x0505, "System out of disk space")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "No write permission for disk files")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "Missing data file")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "Out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "Unexpected generic error in vnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_STATUS,           0, 0x0510, "Database not ready")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")
 
 // tsdb
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         0, 0x0600, "tsdb invalid table id")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE,       0, 0x0601, "tsdb invalid table type")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION,    0, 0x0602, "tsdb invalid table schema version")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST,      0, 0x0603, "tsdb table already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG,           0, 0x0604, "tsdb invalid configuration")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED,              0, 0x0605, "tsdb init failed")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE,             0, 0x0606, "tsdb no diskspace")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS,      0, 0x0607, "tsdb no disk permissions")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED,           0, 0x0608, "tsdb file corrupted")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY,            0, 0x0609, "tsdb out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE,      0, 0x060A, "tsdb tag version is out of date")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE,   0, 0x060B, "tsdb timestamp is out of range")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP,      0, 0x060C, "tsdb submit message is messed up")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION,           0, 0x060D, "tsdb invalid action")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG,    0, 0x060E, "tsdb invalid create table message")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "tsdb no table data in memory skiplist")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "tsdb file already exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "tsdb need to reconfigure table")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "tsdb create table information")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         0, 0x0600, "Invalid table ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE,       0, 0x0601, "Invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION,    0, 0x0602, "Invalid table schema version")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST,      0, 0x0603, "Table already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG,           0, 0x0604, "Invalid configuration")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED,              0, 0x0605, "Tsdb init failed")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE,             0, 0x0606, "No diskspace for tsdb")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS,      0, 0x0607, "No permission for disk files")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED,           0, 0x0608, "Data file(s) corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY,            0, 0x0609, "Out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE,      0, 0x060A, "Tag too old")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE,   0, 0x060B, "Timestamp data out of range")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP,      0, 0x060C, "Submit message is messed up")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION,           0, 0x060D, "Invalid operation")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG,    0, 0x060E, "Invalid creation of table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "No table data in memory skiplist")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "File already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "Need to reconfigure table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "Invalid information to create table")
 
 // query
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "query invalid handle")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG,              0, 0x0701, "query invalid message")    // failed to validate the sql expression msg by vnode
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE,             0, 0x0702, "query no diskspace")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "query out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "query app error")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "query duplicated join key")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT,        0, 0x0706, "query tag conditon too many")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "Invalid handle")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG,              0, 0x0701, "Invalid message")    // failed to validate the sql expression msg by vnode
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE,             0, 0x0702, "No diskspace for query")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "System out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "Unexpected generic error in query")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "Duplicated join key")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT,        0, 0x0706, "Tag conditon too many")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY,                0, 0x0707, "Query not ready")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP,                  0, 0x0708, "Query should response")
 
 // grant
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "grant expired")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED,          0, 0x0801, "grant dnode limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED,           0, 0x0802, "grant account limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED,     0, 0x0803, "grant timeseries limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED,             0, 0x0804, "grant db limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED,           0, 0x0805, "grant user limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED,           0, 0x0806, "grant conn limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED,         0, 0x0807, "grant stream limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED,          0, 0x0808, "grant speed limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED,        0, 0x0809, "grant storage limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED,      0, 0x080A, "grant query time limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "grant cpu limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "License expired")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED,          0, 0x0801, "DNode creation limited by licence")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED,           0, 0x0802, "Account creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED,     0, 0x0803, "Table creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED,             0, 0x0804, "DB creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED,           0, 0x0805, "User creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED,           0, 0x0806, "Conn creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED,         0, 0x0807, "Stream creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED,          0, 0x0808, "Write speed limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED,        0, 0x0809, "Storage capacity limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED,      0, 0x080A, "Query time limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores limited by license")
 
 // sync
-TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "sync invalid configuration")
-TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "sync module not enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
 
 // wal
-TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "wal app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
 
 #ifdef TAOS_ERROR_C
 };
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index ad2be97946838c71b63ab1b94424ee23c390e32e..edddc717c08e6b94f15bd66261f563560e00ad64 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -522,6 +522,7 @@ typedef struct {
   int8_t   compression;
   int8_t   walLevel;
   int8_t   replications;
+  int8_t   quorum;
   int8_t   ignoreExist;
 } SCMCreateDbMsg, SCMAlterDbMsg;
 
diff --git a/src/inc/tsync.h b/src/inc/tsync.h
index 65b91d87e479abd4c0b497cf4b95bc561db8fae5..2dfac7ec3282bc2f404eeffa4ccc7e85ad6cf78a 100644
--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@@ -26,6 +26,7 @@ extern "C" {
 typedef enum _TAOS_SYNC_ROLE {
   TAOS_SYNC_ROLE_OFFLINE,
   TAOS_SYNC_ROLE_UNSYNCED,
+  TAOS_SYNC_ROLE_SYNCING,
   TAOS_SYNC_ROLE_SLAVE,
   TAOS_SYNC_ROLE_MASTER,
 } ESyncRole;
@@ -78,6 +79,9 @@ typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
 // when role is changed, call this to notify app
 typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
 
+// if a number of retrieving data failed, call this to start flow control 
+typedef void     (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
+
 // when data file is synced successfully, notity app
 typedef int      (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
 
@@ -93,6 +97,7 @@ typedef struct {
   FWriteToCache   writeToCache;
   FConfirmForward confirmForward;
   FNotifyRole     notifyRole;
+  FNotifyFlowCtrl notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
 } SSyncInfo;
 
diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index 0b63e9e71d5cc76cd2b4b1eec3997181e7c5c706..e77f5741d302f5b7ae8c22ca04d099f87ad9f2a0 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -104,123 +104,125 @@
 #define TK_MAXTABLES                       86
 #define TK_CACHE                           87
 #define TK_REPLICA                         88
-#define TK_DAYS                            89
-#define TK_MINROWS                         90
-#define TK_MAXROWS                         91
-#define TK_BLOCKS                          92
-#define TK_CTIME                           93
-#define TK_WAL                             94
-#define TK_FSYNC                           95
-#define TK_COMP                            96
-#define TK_PRECISION                       97
-#define TK_LP                              98
-#define TK_RP                              99
-#define TK_TAGS                           100
-#define TK_USING                          101
-#define TK_AS                             102
-#define TK_COMMA                          103
-#define TK_NULL                           104
-#define TK_SELECT                         105
-#define TK_UNION                          106
-#define TK_ALL                            107
-#define TK_FROM                           108
-#define TK_VARIABLE                       109
-#define TK_INTERVAL                       110
-#define TK_FILL                           111
-#define TK_SLIDING                        112
-#define TK_ORDER                          113
-#define TK_BY                             114
-#define TK_ASC                            115
-#define TK_DESC                           116
-#define TK_GROUP                          117
-#define TK_HAVING                         118
-#define TK_LIMIT                          119
-#define TK_OFFSET                         120
-#define TK_SLIMIT                         121
-#define TK_SOFFSET                        122
-#define TK_WHERE                          123
-#define TK_NOW                            124
-#define TK_RESET                          125
-#define TK_QUERY                          126
-#define TK_ADD                            127
-#define TK_COLUMN                         128
-#define TK_TAG                            129
-#define TK_CHANGE                         130
-#define TK_SET                            131
-#define TK_KILL                           132
-#define TK_CONNECTION                     133
-#define TK_STREAM                         134
-#define TK_COLON                          135
-#define TK_ABORT                          136
-#define TK_AFTER                          137
-#define TK_ATTACH                         138
-#define TK_BEFORE                         139
-#define TK_BEGIN                          140
-#define TK_CASCADE                        141
-#define TK_CLUSTER                        142
-#define TK_CONFLICT                       143
-#define TK_COPY                           144
-#define TK_DEFERRED                       145
-#define TK_DELIMITERS                     146
-#define TK_DETACH                         147
-#define TK_EACH                           148
-#define TK_END                            149
-#define TK_EXPLAIN                        150
-#define TK_FAIL                           151
-#define TK_FOR                            152
-#define TK_IGNORE                         153
-#define TK_IMMEDIATE                      154
-#define TK_INITIALLY                      155
-#define TK_INSTEAD                        156
-#define TK_MATCH                          157
-#define TK_KEY                            158
-#define TK_OF                             159
-#define TK_RAISE                          160
-#define TK_REPLACE                        161
-#define TK_RESTRICT                       162
-#define TK_ROW                            163
-#define TK_STATEMENT                      164
-#define TK_TRIGGER                        165
-#define TK_VIEW                           166
-#define TK_COUNT                          167
-#define TK_SUM                            168
-#define TK_AVG                            169
-#define TK_MIN                            170
-#define TK_MAX                            171
-#define TK_FIRST                          172
-#define TK_LAST                           173
-#define TK_TOP                            174
-#define TK_BOTTOM                         175
-#define TK_STDDEV                         176
-#define TK_PERCENTILE                     177
-#define TK_APERCENTILE                    178
-#define TK_LEASTSQUARES                   179
-#define TK_HISTOGRAM                      180
-#define TK_DIFF                           181
-#define TK_SPREAD                         182
-#define TK_TWA                            183
-#define TK_INTERP                         184
-#define TK_LAST_ROW                       185
-#define TK_RATE                           186
-#define TK_IRATE                          187
-#define TK_SUM_RATE                       188
-#define TK_SUM_IRATE                      189
-#define TK_AVG_RATE                       190
-#define TK_AVG_IRATE                      191
-#define TK_TBID                           192
-#define TK_SEMI                           193
-#define TK_NONE                           194
-#define TK_PREV                           195
-#define TK_LINEAR                         196
-#define TK_IMPORT                         197
-#define TK_METRIC                         198
-#define TK_TBNAME                         199
-#define TK_JOIN                           200
-#define TK_METRICS                        201
-#define TK_STABLE                         202
-#define TK_INSERT                         203
-#define TK_INTO                           204
-#define TK_VALUES                         205
+#define TK_QUORUM                          89
+#define TK_DAYS                            90
+#define TK_MINROWS                         91
+#define TK_MAXROWS                         92
+#define TK_BLOCKS                          93
+#define TK_CTIME                           94
+#define TK_WAL                             95
+#define TK_FSYNC                           96
+#define TK_COMP                            97
+#define TK_PRECISION                       98
+#define TK_LP                              99
+#define TK_RP                             100
+#define TK_TAGS                           101
+#define TK_USING                          102
+#define TK_AS                             103
+#define TK_COMMA                          104
+#define TK_NULL                           105
+#define TK_SELECT                         106
+#define TK_UNION                          107
+#define TK_ALL                            108
+#define TK_FROM                           109
+#define TK_VARIABLE                       110
+#define TK_INTERVAL                       111
+#define TK_FILL                           112
+#define TK_SLIDING                        113
+#define TK_ORDER                          114
+#define TK_BY                             115
+#define TK_ASC                            116
+#define TK_DESC                           117
+#define TK_GROUP                          118
+#define TK_HAVING                         119
+#define TK_LIMIT                          120
+#define TK_OFFSET                         121
+#define TK_SLIMIT                         122
+#define TK_SOFFSET                        123
+#define TK_WHERE                          124
+#define TK_NOW                            125
+#define TK_RESET                          126
+#define TK_QUERY                          127
+#define TK_ADD                            128
+#define TK_COLUMN                         129
+#define TK_TAG                            130
+#define TK_CHANGE                         131
+#define TK_SET                            132
+#define TK_KILL                           133
+#define TK_CONNECTION                     134
+#define TK_STREAM                         135
+#define TK_COLON                          136
+#define TK_ABORT                          137
+#define TK_AFTER                          138
+#define TK_ATTACH                         139
+#define TK_BEFORE                         140
+#define TK_BEGIN                          141
+#define TK_CASCADE                        142
+#define TK_CLUSTER                        143
+#define TK_CONFLICT                       144
+#define TK_COPY                           145
+#define TK_DEFERRED                       146
+#define TK_DELIMITERS                     147
+#define TK_DETACH                         148
+#define TK_EACH                           149
+#define TK_END                            150
+#define TK_EXPLAIN                        151
+#define TK_FAIL                           152
+#define TK_FOR                            153
+#define TK_IGNORE                         154
+#define TK_IMMEDIATE                      155
+#define TK_INITIALLY                      156
+#define TK_INSTEAD                        157
+#define TK_MATCH                          158
+#define TK_KEY                            159
+#define TK_OF                             160
+#define TK_RAISE                          161
+#define TK_REPLACE                        162
+#define TK_RESTRICT                       163
+#define TK_ROW                            164
+#define TK_STATEMENT                      165
+#define TK_TRIGGER                        166
+#define TK_VIEW                           167
+#define TK_COUNT                          168
+#define TK_SUM                            169
+#define TK_AVG                            170
+#define TK_MIN                            171
+#define TK_MAX                            172
+#define TK_FIRST                          173
+#define TK_LAST                           174
+#define TK_TOP                            175
+#define TK_BOTTOM                         176
+#define TK_STDDEV                         177
+#define TK_PERCENTILE                     178
+#define TK_APERCENTILE                    179
+#define TK_LEASTSQUARES                   180
+#define TK_HISTOGRAM                      181
+#define TK_DIFF                           182
+#define TK_SPREAD                         183
+#define TK_TWA                            184
+#define TK_INTERP                         185
+#define TK_LAST_ROW                       186
+#define TK_RATE                           187
+#define TK_IRATE                          188
+#define TK_SUM_RATE                       189
+#define TK_SUM_IRATE                      190
+#define TK_AVG_RATE                       191
+#define TK_AVG_IRATE                      192
+#define TK_TBID                           193
+#define TK_SEMI                           194
+#define TK_NONE                           195
+#define TK_PREV                           196
+#define TK_LINEAR                         197
+#define TK_IMPORT                         198
+#define TK_METRIC                         199
+#define TK_TBNAME                         200
+#define TK_JOIN                           201
+#define TK_METRICS                        202
+#define TK_STABLE                         203
+#define TK_INSERT                         204
+#define TK_INTO                           205
+#define TK_VALUES                         206
+
 
 #define TK_SPACE                          300
 #define TK_COMMENT                        301
diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt
index 9854a44c3b64f889d02de8c8493cc92180ee2258..01c5f5ea3adf580eca86777e93c7c2d88bf535f4 100644
--- a/src/kit/shell/CMakeLists.txt
+++ b/src/kit/shell/CMakeLists.txt
@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 01dcc73ef65e4d75ea9f3d7e7a83ab0ee8892950..616a3bfd7f26b55bb2b71aa9bb3f7720bd69e43c 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -73,6 +73,8 @@ TAOS *shellInit(SShellArguments *args) {
   // Connect to the database.
   TAOS *con = taos_connect(args->host, args->user, args->password, args->database, args->port);
   if (con == NULL) {
+    printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno));
+    fflush(stdout);
     return con;
   }
 
diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt
index 627092a85b5abff2d71cc482eb9e8d28171a46b1..a1593ce5a1b2594101c4d4e42497b2136d5e28d4 100644
--- a/src/kit/taosdemo/CMakeLists.txt
+++ b/src/kit/taosdemo/CMakeLists.txt
@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt
index c63168237cac898db6c0ecf303164ed272b89545..2a14dbfb4ac164d064d094a89799c7f1fd8a1a9a 100644
--- a/src/kit/taosdump/CMakeLists.txt
+++ b/src/kit/taosdump/CMakeLists.txt
@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c
index a1b1998298d3c8245046869cc37d8b6d7c72ca7a..7ae209bbdfa9daa4ccdbbece2e2d0573f1755e20 100644
--- a/src/kit/taosdump/taosdump.c
+++ b/src/kit/taosdump/taosdump.c
@@ -545,7 +545,7 @@ int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric
   strcpy(tableRecord.name, meter);
   strcpy(tableRecord.metric, metric);
 
-  twrite(*fd, &tableRecord, sizeof(STableRecord));
+  taosTWrite(*fd, &tableRecord, sizeof(STableRecord));
   return 0;
 }
 
@@ -600,7 +600,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu
     strncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
     strcpy(tableRecord.metric, metric);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 
@@ -1196,7 +1196,7 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
   while ((row = taos_fetch_row(tmpResult)) != NULL) {  
     memset(&tableRecord, 0, sizeof(STableRecord));
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
   }  
   
   taos_free_result(tmpResult);
@@ -1282,7 +1282,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
     strncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 
diff --git a/src/kit/taosmigrate/CMakeLists.txt b/src/kit/taosmigrate/CMakeLists.txt
index 85b2f33f01f65cd6d49d47440455ed8ef53d0bdf..fdd9dd88861c356d63174d9515df3c3a8910a4e4 100644
--- a/src/kit/taosmigrate/CMakeLists.txt
+++ b/src/kit/taosmigrate/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) 
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/inc)
@@ -12,7 +12,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(. SRC)  
  
   ADD_EXECUTABLE(taosmigrate ${SRC})
-  TARGET_LINK_LIBRARIES(taosmigrate common tutil cJson)
+  TARGET_LINK_LIBRARIES(taosmigrate common cJson osdetail tutil)
 ENDIF () 
 
 SET_SOURCE_FILES_PROPERTIES(./taosmigrate.c PROPERTIES COMPILE_FLAGS -w)
diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt
index 2e975f089c7f7b792afc4b10605d1552e0820856..f5b9fbf74763bc331c809e0f5d5d6b9eca6593d8 100644
--- a/src/mnode/CMakeLists.txt
+++ b/src/mnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index 62028bc820d192b7c2c750dbcc1081e068bbf547..f6e3904c647d113d0e26e5740e8a1b4a87d4e4b1 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -171,6 +171,7 @@ typedef struct {
   int8_t  compression;
   int8_t  walLevel;
   int8_t  replications;
+  int8_t  quorum;
   int8_t  reserved[12];
 } SDbCfg;
 
@@ -250,6 +251,7 @@ typedef struct {
   int32_t  rowSize;
   int32_t  numOfRows;
   void *   pIter;
+  void **  ppShow;
   int16_t  offset[TSDB_MAX_COLUMNS];
   int16_t  bytes[TSDB_MAX_COLUMNS];
   int32_t  numOfReads;
diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h
index f37178652472940f047e1444999f9643f7ab7311..13b6ec44111cd9526522c8d213a767c66ad834c1 100644
--- a/src/mnode/inc/mnodeDnode.h
+++ b/src/mnode/inc/mnodeDnode.h
@@ -41,7 +41,7 @@ void    mgmtMonitorDnodeModule();
 
 int32_t mnodeGetDnodesNum();
 int32_t mnodeGetOnlinDnodesCpuCoreNum();
-int32_t mnodeGetOnlinDnodesNum();
+int32_t mnodeGetOnlineDnodesNum();
 void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mnodeIncDnodeRef(SDnodeObj *pDnode);
 void    mnodeDecDnodeRef(SDnodeObj *pDnode);
diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h
index cadd1b1f3d438d316c6dc8d10cea1e5d4e0dbed3..f4854f69a0f315d06d402eef3a125b2a3e45ab0e 100644
--- a/src/mnode/inc/mnodeSdb.h
+++ b/src/mnode/inc/mnodeSdb.h
@@ -47,15 +47,16 @@ typedef enum {
   SDB_OPER_LOCAL
 } ESdbOper;
 
-typedef struct {
+typedef struct SSdbOper {
   ESdbOper type;
-  void *   table;
-  void *   pObj;
-  void *   rowData;
   int32_t  rowSize;
   int32_t  retCode; // for callback in sdb queue
   int32_t  processedCount; // for sync fwd callback
-  int32_t  (*cb)(struct SMnodeMsg *pMsg, int32_t code);
+  int32_t  (*reqFp)(struct SMnodeMsg *pMsg);
+  int32_t  (*writeCb)(struct SMnodeMsg *pMsg, int32_t code);
+  void *   table;
+  void *   pObj;
+  void *   rowData;
   struct SMnodeMsg *pMsg;
 } SSdbOper;
 
@@ -86,6 +87,7 @@ void    sdbUpdateMnodeRoles();
 int32_t sdbInsertRow(SSdbOper *pOper);
 int32_t sdbDeleteRow(SSdbOper *pOper);
 int32_t sdbUpdateRow(SSdbOper *pOper);
+int32_t sdbInsertRowImp(SSdbOper *pOper);
 
 void    *sdbGetRow(void *handle, void *key);
 void    *sdbFetchRow(void *handle, void *pIter, void **ppRow);
diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c
index 287e8bb7232b0813fba8e573f1a6a64e1824cf1e..c40a696edecb1d6689361e26e30f751820b4208e 100644
--- a/src/mnode/src/mnodeAcct.c
+++ b/src/mnode/src/mnodeAcct.c
@@ -85,7 +85,7 @@ static int32_t mnodeAcctActionRestored() {
   if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
     mInfo("dnode first deploy, create root acct");
     int32_t code = mnodeCreateRootAcct();
-    if (code != TSDB_CODE_SUCCESS) {
+    if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mError("failed to create root account, reason:%s", tstrerror(code));
       return code;
     }
diff --git a/src/mnode/src/mnodeBalance.c b/src/mnode/src/mnodeBalance.c
deleted file mode 100644
index 95a97bd055e53ec14ce764e03765afc590c6f88e..0000000000000000000000000000000000000000
--- a/src/mnode/src/mnodeBalance.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tglobal.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
-#include "mnodeSdb.h"
-
-#ifndef _SYNC
-
-int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
-void    balanceCleanUp() {}
-void    balanceAsyncNotify() {}
-void    balanceSyncNotify() {}
-void    balanceReset() {}
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
-
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1000.0;
-
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
-      float openVnodes = pDnode->openVnodes;
-      if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
-
-      float usage = openVnodes / pDnode->numOfCores;
-      if (usage <= vnodeUsage) {
-        pSelDnode = pDnode;
-        vnodeUsage = usage;
-      }
-    }
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  sdbFreeIter(pIter);
-
-  if (pSelDnode == NULL) {
-    mError("failed to alloc vnode to vgroup");
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  }
-
-  pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
-  pVgroup->vnodeGid[0].pDnode = pSelDnode;
-
-  mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
-  return TSDB_CODE_SUCCESS;
-}
-
-#endif 
diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c
index 8933107dd1e747e6a70bae6378073be6c86cc943..41727712b542ffed15b136d6d252e2f8bf51d8cf 100644
--- a/src/mnode/src/mnodeCluster.c
+++ b/src/mnode/src/mnodeCluster.c
@@ -71,7 +71,7 @@ static int32_t mnodeClusterActionRestored() {
   if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
     mInfo("dnode first deploy, create cluster");
     int32_t code = mnodeCreateCluster();
-    if (code != TSDB_CODE_SUCCESS) {
+    if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mError("failed to create cluster, reason:%s", tstrerror(code));
       return code;
     }
@@ -159,16 +159,15 @@ int32_t mnodeGetClusterId() {
 
 void mnodeUpdateClusterId() {
   SClusterObj *pCluster = NULL;
-  mnodeGetNextCluster(NULL, &pCluster);
+  void *pIter = mnodeGetNextCluster(NULL, &pCluster);
   if (pCluster != NULL) {
     tsClusterId = pCluster->clusterId;
-    mnodeDecClusterRef(pCluster);
     mInfo("cluster id is %d", tsClusterId);
-  } else {
-    //assert(false);
   }
-}
 
+  mnodeDecClusterRef(pCluster);
+  sdbFreeIter(pIter);
+}
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index a159e98ed5e9efb086608e7e1b3965bb714eb292..82ac0868d13254a1b6f7658503d1221bb4446209 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -301,12 +301,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
-#ifndef _SYNC
-  if (pCfg->replications != 1) {
-    mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
+  if (pCfg->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCfg->quorum > TSDB_MAX_DB_REPLICA_OPTION) {
+    mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_REPLICA_OPTION,
+           TSDB_MAX_DB_REPLICA_OPTION);
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
-#endif
 
   return TSDB_CODE_SUCCESS;
 }
@@ -327,12 +326,15 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->compression < 0) pCfg->compression = tsCompression;
   if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
+  if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
 }
 
 static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
   SDbObj *pDb = pMsg->pDb;
-  if (pDb != NULL) {
+  if (code == TSDB_CODE_SUCCESS) {
     mLInfo("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+  } else {
+    mError("db:%s, failed to create by %s, reason:%s", pDb->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
   }
 
   return code;
@@ -376,7 +378,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
     .precision           = pCreate->precision,
     .compression         = pCreate->compression,
     .walLevel            = pCreate->walLevel,
-    .replications        = pCreate->replications
+    .replications        = pCreate->replications,
+    .quorum              = pCreate->quorum
   };
 
   mnodeSetDefaultDbCfg(&pDb->cfg);
@@ -393,17 +396,16 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
     .pObj    = pDb,
     .rowSize = sizeof(SDbObj),
     .pMsg    = pMsg,
-    .cb      = mnodeCreateDbCb
+    .writeCb = mnodeCreateDbCb
   };
 
   code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
-    mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
     mnodeDestroyDb(pDb);
-    return code;
-  } else {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
+
+  return code;
 }
 
 bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
@@ -515,6 +517,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
+    pShow->bytes[cols] = 2;
+    pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+    strcpy(pSchema[cols].name, "quorum");
+    pSchema[cols].bytes = htons(pShow->bytes[cols]);
+    cols++;
+
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
     strcpy(pSchema[cols].name, "days");
@@ -661,6 +669,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
       *(int16_t *)pWrite = pDb->cfg.replications;
       cols++;
 
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      *(int16_t *)pWrite = pDb->cfg.quorum;
+      cols++;
+
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int16_t *)pWrite = pDb->cfg.daysPerFile;
       cols++;
@@ -761,8 +773,8 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) {
   };
 
   int32_t code = sdbUpdateRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_MND_SDB_ERROR;
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("db:%s, failed to set dropping state, reason:%s", pDb->name, tstrerror(code));
   }
 
   return code;
@@ -810,6 +822,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   int8_t  compression    = pAlter->compression;
   int8_t  walLevel       = pAlter->walLevel;
   int8_t  replications   = pAlter->replications;
+  int8_t  quorum         = pAlter->quorum;
   int8_t  precision      = pAlter->precision;
   
   terrno = TSDB_CODE_SUCCESS;
@@ -908,6 +921,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     }
   }
 
+  if (quorum >= 0 && quorum != pDb->cfg.quorum) {
+    mDebug("db:%s, quorum:%d change to %d", pDb->name, pDb->cfg.quorum, quorum);
+    newCfg.compression = quorum;
+  }
+
   return newCfg;
 }
 
@@ -954,12 +972,12 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
       .table = tsDbSdb,
       .pObj  = pDb,
       .pMsg  = pMsg,
-      .cb    = mnodeAlterDbCb
+      .writeCb = mnodeAlterDbCb
     };
 
     code = sdbUpdateRow(&oper);
-    if (code == TSDB_CODE_SUCCESS) {
-      if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+    if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+      mError("db:%s, failed to alter, reason:%s", pDb->name, tstrerror(code));
     }
   }
 
@@ -1002,16 +1020,16 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
   mInfo("db:%s, drop db from sdb", pDb->name);
 
   SSdbOper oper = {
-    .type  = SDB_OPER_GLOBAL,
-    .table = tsDbSdb,
-    .pObj  = pDb,
-    .pMsg  = pMsg,
-    .cb    = mnodeDropDbCb
+    .type   = SDB_OPER_GLOBAL,
+    .table  = tsDbSdb,
+    .pObj   = pDb,
+    .pMsg   = pMsg,
+    .writeCb = mnodeDropDbCb
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("db:%s, failed to drop, reason:%s", pDb->name, tstrerror(code));
   }
 
   return code;
@@ -1038,7 +1056,7 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) {
   }
 
   int32_t code = mnodeSetDbDropping(pMsg->pDb);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
     return code;
   }
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 4f9421efdefdce18b8342ad31bc49754acdedfb9..be15abe354fef372b3fe6d392b47100e663ed158 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -79,9 +79,6 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
-#ifndef _SYNC 
-  mnodeDropAllDnodeVgroups(pDnode);
-#endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
   balanceAsyncNotify();
 
@@ -209,7 +206,7 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() {
   return cpuCores;
 }
 
-int32_t mnodeGetOnlinDnodesNum() {
+int32_t mnodeGetOnlineDnodesNum() {
   SDnodeObj *pDnode = NULL;
   void *     pIter = NULL;
   int32_t    onlineDnodes = 0;
@@ -264,7 +261,8 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
     .pObj = pDnode
   };
 
-  if (sdbUpdateRow(&oper) != 0) {
+  int32_t code = sdbUpdateRow(&oper);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("dnodeId:%d, failed update", pDnode->dnodeId);
   }
 }
@@ -504,13 +502,12 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
   };
 
   int32_t code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     int dnodeId = pDnode->dnodeId;
     tfree(pDnode);
-    mError("failed to create dnode:%d, result:%s", dnodeId, tstrerror(code));
+    mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code));
   } else {
-    mInfo("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+    mLInfo("dnode:%d is created", pDnode->dnodeId);
   }
 
   return code;
@@ -525,9 +522,10 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) {
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    mLInfo("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("dnode:%d, failed to drop from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+  } else {
+    mLInfo("dnode:%d, is dropped from cluster", pDnode->dnodeId);
   }
 
   return code;
@@ -552,12 +550,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 
   mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
 
-#ifndef _SYNC
-  int32_t code = mnodeDropDnode(pDnode, pMsg);
-#else
   int32_t code = balanceDropDnode(pDnode);
-#endif
-
   mnodeDecDnodeRef(pDnode);
   return code;
 }
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index d2f389ca0b6afde4dd9e48c05a99c266ebbbca44..5f82a9afad878bf5c3f920de2d2a2ffd4c1bf018 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
+#include "tglobal.h"
 #include "trpc.h"
 #include "tsync.h"
 #include "tbalance.h"
@@ -31,8 +32,6 @@
 #include "mnodeShow.h"
 #include "mnodeUser.h"
 
-#include "tglobal.h"
-
 static void *        tsMnodeSdb = NULL;
 static int32_t       tsMnodeUpdateSize = 0;
 static SRpcEpSet     tsMnodeEpSetForShell;
@@ -190,18 +189,7 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) {
 }
 
 char *mnodeGetMnodeRoleStr(int32_t role) {
-  switch (role) {
-    case TAOS_SYNC_ROLE_OFFLINE:
-      return "offline";
-    case TAOS_SYNC_ROLE_UNSYNCED:
-      return "unsynced";
-    case TAOS_SYNC_ROLE_SLAVE:
-      return "slave";
-    case TAOS_SYNC_ROLE_MASTER:
-      return "master";
-    default:
-      return "undefined";
-  }
+  return syncRole[role];
 }
 
 void mnodeUpdateMnodeEpSet() {
@@ -290,9 +278,8 @@ int32_t mnodeAddMnode(int32_t dnodeId) {
   };
 
   int32_t code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     tfree(pMnode);
-    code = TSDB_CODE_MND_SDB_ERROR;
   }
 
   mnodeUpdateMnodeEpSet();
@@ -324,9 +311,6 @@ int32_t mnodeDropMnode(int32_t dnodeId) {
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_SDB_ERROR;
-  }
 
   sdbDecRef(tsMnodeSdb, pMnode);
 
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index 9121f31131bc0cafa85830409bf9d40390da6790..ff4ddf1b2a08759cddb6154f0b478de6411f4ec0 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -68,7 +68,7 @@ int32_t mnodeInitProfile() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
 
-  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, false, mnodeFreeConn, "conn");
+  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, true, mnodeFreeConn, "conn");
   return 0;
 }
 
@@ -119,8 +119,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
     return NULL;
   }
 
-  if (pConn->ip != ip || pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
-    mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user,
+  if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
+    mDebug("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user,
            taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port);
     taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false);
     return NULL;
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 26efcfeac037ff3abe3ae33dfa380772fad7ad5b..f3f6e33431a67e884ff04599c3ad44869771e1bb 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -101,6 +101,11 @@ static int32_t sdbInitWriteWorker();
 static void    sdbCleanupWriteWorker();
 static int32_t sdbAllocWriteQueue();
 static void    sdbFreeWritequeue();
+static int32_t sdbUpdateRowImp(SSdbOper *pOper);
+static int32_t sdbDeleteRowImp(SSdbOper *pOper);
+static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper);
+static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper);
+static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper);
 
 int32_t sdbGetId(void *handle) {
   return ((SSdbTable *)handle)->autoIndex;
@@ -260,8 +265,20 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
              tstrerror(code));
   }
 
-  if (pOper->cb != NULL) {
-    pOper->retCode = (*pOper->cb)(pMsg, pOper->retCode);
+  // failed to forward, need revert insert
+  if (pOper->retCode != TSDB_CODE_SUCCESS) {
+    SWalHead *pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+    int32_t   action = pHead->msgType % 10;
+    sdbError("table:%s record:%p:%s ver:%" PRIu64 ", action:%d failed to foward reason:%s",
+             ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont),
+             pHead->version, action, tstrerror(pOper->retCode));
+    if (action == SDB_ACTION_INSERT) {
+      sdbDeleteHash(pOper->table, pOper);
+    }
+  }
+
+  if (pOper->writeCb != NULL) {
+    pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
   }
   dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
 
@@ -269,6 +286,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   if (ahandle == NULL) {
     sdbDecRef(pOper->table, pOper->pObj);
   }
+
   taosFreeQitem(pOper);
 }
 
@@ -396,8 +414,8 @@ void sdbIncRef(void *handle, void *pObj) {
 
   SSdbTable *pTable = handle;
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
-  atomic_add_fetch_32(pRefCount, 1);
-  sdbTrace("add ref to table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+  int32_t    refCount = atomic_add_fetch_32(pRefCount, 1);
+  sdbTrace("add ref to table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
 }
 
 void sdbDecRef(void *handle, void *pObj) {
@@ -406,11 +424,11 @@ void sdbDecRef(void *handle, void *pObj) {
   SSdbTable *pTable = handle;
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
   int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
-  sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+  sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
 
   int32_t *updateEnd = pObj + pTable->refCountPos - 4;
   if (refCount <= 0 && *updateEnd) {
-    sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+    sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
     SSdbOper oper = {.pObj = pObj};
     (*pTable->destroyFp)(&oper);
   }
@@ -609,7 +627,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
   if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
   if (sdbGetRowFromObj(pTable, pOper->pObj)) {
-    sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj));
+    sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName,
+             sdbGetKeyStrFromObj(pTable, pOper->pObj));
     sdbDecRef(pTable, pOper->pObj);
     return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE;
   }
@@ -634,9 +653,20 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
     return TSDB_CODE_SUCCESS;
   }
 
+  if (pOper->reqFp) {
+    return (*pOper->reqFp)(pOper->pMsg);
+  } else {
+    return sdbInsertRowImp(pOper);
+  }
+}
+
+int32_t sdbInsertRowImp(SSdbOper *pOper) {
+  SSdbTable *pTable = (SSdbTable *)pOper->table;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
+
   int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
   SSdbOper *pNewOper = taosAllocateQitem(size);
-  
+
   SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
   pHead->version = 0;
   pHead->len = pOper->rowSize;
@@ -655,7 +685,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
 
   sdbIncRef(pNewOper->table, pNewOper->pObj);
   taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-  return TSDB_CODE_SUCCESS;
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 bool sdbCheckRowDeleted(void *pTableInput, void *pRow) {
@@ -664,7 +695,6 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) {
 
   int32_t *updateEnd = pRow + pTable->refCountPos - 4;
   return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1;
-  // return (*updateEnd == 1);
 }
 
 int32_t sdbDeleteRow(SSdbOper *pOper) {
@@ -692,13 +722,24 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
     return TSDB_CODE_SUCCESS;
   }
 
+  if (pOper->reqFp) {
+    return (*pOper->reqFp)(pOper->pMsg);
+  } else {
+    return sdbDeleteRowImp(pOper);
+  }
+}
+
+int32_t sdbDeleteRowImp(SSdbOper *pOper) {
+  SSdbTable *pTable = (SSdbTable *)pOper->table;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
+
   int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
   SSdbOper *pNewOper = taosAllocateQitem(size);
 
   SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
   pHead->version = 0;
   pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
-  
+
   pOper->rowData = pHead->cont;
   (*pTable->encodeFp)(pOper);
   pHead->len = pOper->rowSize;
@@ -711,7 +752,8 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
   }
 
   taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-  return TSDB_CODE_SUCCESS;
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 int32_t sdbUpdateRow(SSdbOper *pOper) {
@@ -735,6 +777,17 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
     return TSDB_CODE_SUCCESS;
   }
 
+  if (pOper->reqFp) {
+    return (*pOper->reqFp)(pOper->pMsg);
+  } else {
+    return sdbUpdateRowImp(pOper);
+  }
+}
+
+int32_t sdbUpdateRowImp(SSdbOper *pOper) {
+  SSdbTable *pTable = (SSdbTable *)pOper->table;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
+
   int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
   SSdbOper *pNewOper = taosAllocateQitem(size);
 
@@ -755,7 +808,8 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
 
   sdbIncRef(pNewOper->table, pNewOper->pObj);
   taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-  return TSDB_CODE_SUCCESS;
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c
index d7f058b3a80e5085965b027a0ec7d92683ac3bf0..733bd43c74ae219f82c97820e867afef79616826 100644
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@@ -49,8 +49,8 @@ static int32_t mnodeProcessUseMsg(SMnodeMsg *mnodeMsg);
 static void  mnodeFreeShowObj(void *data);
 static bool  mnodeAccquireShowObj(SShowObj *pShow);
 static bool  mnodeCheckShowFinished(SShowObj *pShow);
-static void *mnodePutShowObj(SShowObj *pShow, int32_t size);
-static void  mnodeReleaseShowObj(void *pShow, bool forceRemove);
+static void *mnodePutShowObj(SShowObj *pShow);
+static void  mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove);
 
 extern void *tsMnodeTmr;
 static void *tsMnodeShowCache = NULL;
@@ -65,7 +65,7 @@ int32_t mnodeInitShow() {
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
   
-  tsMnodeShowCache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, false, mnodeFreeShowObj, "show");
+  tsMnodeShowCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 5, false, mnodeFreeShowObj, "show");
   return 0;
 }
 
@@ -121,13 +121,13 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
   }
 
   int32_t showObjSize = sizeof(SShowObj) + htons(pShowMsg->payloadLen);
-  SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
+  SShowObj *pShow = calloc(1, showObjSize);
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
   tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
-  pShow = mnodePutShowObj(pShow, showObjSize);
+  pShow = mnodePutShowObj(pShow);
   if (pShow == NULL) {    
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
@@ -270,7 +270,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
     }
   }
 
-  pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum());
+  pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
   pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
   mnodeGetMnodeEpSetForShell(&pHBRsp->epSet);
 
@@ -377,37 +377,41 @@ static bool mnodeCheckShowFinished(SShowObj *pShow) {
 }
 
 static bool mnodeAccquireShowObj(SShowObj *pShow) {
-  SShowObj *pSaved = taosCacheAcquireByKey(tsMnodeShowCache, &pShow->index, sizeof(int32_t));
-  if (pSaved == pShow) {
-    mDebug("%p, show is accquired from cache", pShow);
+  SShowObj **ppShow = taosCacheAcquireByKey(tsMnodeShowCache, &pShow, sizeof(int64_t));
+  if (ppShow) {
+    mDebug("%p, show is accquired from cache, data:%p, index:%d", pShow, ppShow, pShow->index);
     return true;
-  } else {
-    return false;
   }
+
+  return false;
 }
 
-static void *mnodePutShowObj(SShowObj *pShow, int32_t size) {
+static void* mnodePutShowObj(SShowObj *pShow) {
   if (tsMnodeShowCache != NULL) {
     pShow->index = atomic_add_fetch_32(&tsShowObjIndex, 1);
-    SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, &pShow->index, sizeof(int32_t), pShow, size, 6);
-    mDebug("%p, show is put into cache, index:%d", newQhandle, pShow->index);
-    free(pShow);
-
-    return newQhandle;
+    SShowObj **ppShow = taosCachePut(tsMnodeShowCache, &pShow, sizeof(int64_t), &pShow, sizeof(int64_t), 6);
+    pShow->ppShow = (void**)ppShow;
+    mDebug("%p, show is put into cache, data:%p index:%d", pShow, ppShow, pShow->index);
+    return pShow;
   }
 
   return NULL;
 }
 
 static void mnodeFreeShowObj(void *data) {
-  SShowObj *pShow = data;
+  SShowObj *pShow = *(SShowObj **)data;
   sdbFreeIter(pShow->pIter);
-  mDebug("%p, show is destroyed", pShow);
+
+  mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
+  tfree(pShow);
 }
 
-static void mnodeReleaseShowObj(void *pShow, bool forceRemove) {
-  mDebug("%p, show is released, force:%s", pShow, forceRemove ? "true" : "false");
-  taosCacheRelease(tsMnodeShowCache, &pShow, forceRemove);
+static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) {
+  SShowObj **ppShow = (SShowObj **)pShow->ppShow;
+  mDebug("%p, show is released, force:%s data:%p index:%d", pShow, forceRemove ? "true" : "false", ppShow,
+         pShow->index);
+
+  taosCacheRelease(tsMnodeShowCache, (void **)(&ppShow), forceRemove);
 }
 
 void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 053a1522a73262138121e434cefc843fb6ec5c6e..009633b43b1e8529e591cc43260d034fa7e39c71 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -128,9 +128,11 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     // add ref
     pTable->superTable = mnodeGetSuperTableByUid(pTable->suid);
-    mnodeAddTableIntoStable(pTable->superTable, pTable);
-    grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
-    if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
+    if (pTable->superTable != NULL) {
+      mnodeAddTableIntoStable(pTable->superTable, pTable);
+      grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
+      if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
+    }
   } else {
     grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
     if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->numOfColumns - 1);
@@ -865,18 +867,17 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
     .pObj = pStable,
     .rowSize = sizeof(SSuperTableObj) + schemaSize,
     .pMsg = pMsg,
-    .cb = mnodeCreateSuperTableCb
+    .writeCb = mnodeCreateSuperTableCb
   };
 
   int32_t code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mnodeDestroySuperTable(pStable);
     pMsg->pTable = NULL;
     mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
-    return code;
-  } else {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
+
+  return code;
 }
 
 static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
@@ -924,13 +925,15 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
     .table = tsSuperTableSdb,
     .pObj  = pStable,
     .pMsg  = pMsg,
-    .cb    = mnodeDropSuperTableCb
+    .writeCb = mnodeDropSuperTableCb
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("app:%p:%p, table:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+           tstrerror(code));
   }
+
   return code;
 }
 
@@ -995,15 +998,10 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeAddSuperTableTagCb
+    .writeCb = mnodeAddSuperTableTagCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
@@ -1034,15 +1032,10 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeDropSuperTableTagCb
+    .writeCb = mnodeDropSuperTableTagCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
@@ -1083,15 +1076,10 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeModifySuperTableTagNameCb
+    .writeCb = mnodeModifySuperTableTagNameCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) {
@@ -1162,15 +1150,10 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeAddSuperTableColumnCb
+    .writeCb = mnodeAddSuperTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+ return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
@@ -1212,15 +1195,10 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeDropSuperTableColumnCb
+    .writeCb = mnodeDropSuperTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
@@ -1261,15 +1239,10 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .pMsg = pMsg,
-    .cb = mnodeChangeSuperTableColumnCb
+    .writeCb = mnodeChangeSuperTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 // show super tables
@@ -1645,20 +1618,12 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
   return pCreate;
 }
 
-static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
+static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
   SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
   assert(pTable);
 
-  if (code == TSDB_CODE_SUCCESS) {
-    mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle,
-           pMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, tstrerror(code));
-  } else {
-    mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg,
-           pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb};
-    sdbDeleteRow(&desc);
-    return code;
-  }
+  mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+         pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid);
 
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
   SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable);
@@ -1679,6 +1644,34 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
+static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
+  SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+  SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
+  assert(pTable);
+
+  if (code == TSDB_CODE_SUCCESS) {
+    if (pCreate->getMeta) {
+      mDebug("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+             pTable->info.tableId, pMsg->rpcMsg.handle);
+
+      pMsg->retry = 0;
+      dnodeReprocessMnodeWriteMsg(pMsg);
+    } else {
+      mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+             pMsg->rpcMsg.handle);
+
+      dnodeSendRpcMnodeWriteRsp(pMsg, TSDB_CODE_SUCCESS);
+    }
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  } else {
+    mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg,
+           pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
+    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb};
+    sdbDeleteRow(&desc);
+    return code;
+  }
+}
+
 static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   SVgObj *pVgroup = pMsg->pVgroup;
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
@@ -1752,23 +1745,23 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   pMsg->pTable = (STableObj *)pTable;
   mnodeIncTableRef(pMsg->pTable);
 
-  SSdbOper desc = {0};
-  desc.type = SDB_OPER_GLOBAL;
-  desc.pObj = pTable;
-  desc.table = tsChildTableSdb;
-  desc.pMsg = pMsg;
-  desc.cb = mnodeDoCreateChildTableCb;
+  SSdbOper desc = {
+    .type  = SDB_OPER_GLOBAL,
+    .pObj  = pTable,
+    .table = tsChildTableSdb,
+    .pMsg  = pMsg,
+    .reqFp = mnodeDoCreateChildTableFp
+  };
   
   int32_t code = sdbInsertRow(&desc);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mnodeDestroyChildTable(pTable);
     pMsg->pTable = NULL;
-    mError("app:%p:%p, table:%s, update sdb error, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
+    mError("app:%p:%p, table:%s, failed to create, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
            tstrerror(code));
-    return code;       
-  } else {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
+
+  return code;
 }
 
 static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
@@ -1813,7 +1806,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
     return terrno;
   } else {
     mDebug("app:%p:%p, table:%s, send create msg to vnode again", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
-    return mnodeDoCreateChildTableCb(pMsg, TSDB_CODE_SUCCESS);
+    return mnodeDoCreateChildTableFp(pMsg);
   }
 }
 
@@ -1878,13 +1871,15 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
     .table = tsChildTableSdb,
     .pObj  = pTable,
     .pMsg  = pMsg,
-    .cb    = mnodeDropChildTableCb
+    .writeCb = mnodeDropChildTableCb
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("app:%p:%p, ctable:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+           tstrerror(code));
   }
+
   return code;
 }
 
@@ -1980,15 +1975,10 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
     .table = tsChildTableSdb,
     .pObj = pTable,
     .pMsg = pMsg,
-    .cb = mnodeAlterNormalTableColumnCb
+    .writeCb = mnodeAlterNormalTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
@@ -2018,15 +2008,10 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
     .table = tsChildTableSdb,
     .pObj = pTable,
     .pMsg = pMsg,
-    .cb = mnodeAlterNormalTableColumnCb
+    .writeCb = mnodeAlterNormalTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
@@ -2060,15 +2045,10 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char
     .table = tsChildTableSdb,
     .pObj = pTable,
     .pMsg = pMsg,
-    .cb = mnodeAlterNormalTableColumnCb
+    .writeCb = mnodeAlterNormalTableColumnCb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return sdbUpdateRow(&oper);
 }
 
 static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) {
@@ -2374,19 +2354,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
   }
 
   if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
-    SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont;
-    if (pCreate->getMeta) {
-      mDebug("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p result:%s",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle,
-             tstrerror(rpcMsg->code));
-
-      mnodeMsg->retry = 0;
-      dnodeReprocessMnodeWriteMsg(mnodeMsg);
-    } else {
-      mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-             pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
-
-      dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+     SSdbOper desc = {
+      .type  = SDB_OPER_GLOBAL,
+      .pObj  = pTable,
+      .table = tsChildTableSdb,
+      .pMsg  = mnodeMsg,
+      .writeCb = mnodeDoCreateChildTableCb
+    };
+    
+    int32_t code = sdbInsertRowImp(&desc);
+    if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+      mnodeMsg->pTable = NULL;
+      mnodeDestroyChildTable(pTable);
+      dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
     }
   } else {
     if (mnodeMsg->retry++ < 10) {
diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c
index 8c783eebaf64fe377b9756e26196573c8fdfcaca..a875cff4a2f1dfda9eaa43fb9ce65e821e4b1fc6 100644
--- a/src/mnode/src/mnodeUser.c
+++ b/src/mnode/src/mnodeUser.c
@@ -182,9 +182,10 @@ static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) {
   };
 
   int32_t code = sdbUpdateRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("user:%s, failed to alter by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
+  } else {
     mLInfo("user:%s, is altered by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
@@ -236,11 +237,11 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
   };
 
   code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
     tfree(pUser);
   } else {
     mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
@@ -255,9 +256,10 @@ static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) {
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("user:%s, failed to drop by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
+  } else {
     mLInfo("user:%s, is dropped by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 089168c31ea8465e017441b04cbcd415ce16fb2c..91aa3fdec2bc9a627fa9d9f01b841f4d90cfa805 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -256,7 +256,8 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
     .pObj = pVgroup
   };
 
-  if (sdbUpdateRow(&oper) != TSDB_CODE_SUCCESS) {
+  int32_t code = sdbUpdateRow(&oper);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("vgId:%d, failed to update vgroup", pVgroup->vgId);
   }
   mnodeSendAlterVgroupMsg(pVgroup);
@@ -433,11 +434,12 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
     maxVgroupsPerDb = MAX(maxVgroupsPerDb, 2);
   }
 
+  int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   if (pDb->numOfVgroups < maxVgroupsPerDb) {
     mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle,
            pMsg, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb);
     pthread_mutex_unlock(&pDb->mutex);
-    int32_t code = mnodeCreateVgroup(pMsg);
+    code = mnodeCreateVgroup(pMsg);
     if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       return code;
     } else {
@@ -448,10 +450,10 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
   SVgObj *pVgroup = pDb->vgList[0];
   if (pVgroup == NULL) {
     pthread_mutex_unlock(&pDb->mutex);
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+    return code;
   }
 
-  int32_t code = mnodeAllocVgroupIdPool(pVgroup);
+  code = mnodeAllocVgroupIdPool(pVgroup);
   if (code != TSDB_CODE_SUCCESS) {
     pthread_mutex_unlock(&pDb->mutex);
     return code;
@@ -476,23 +478,11 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
   return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
-static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
+static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) {
   SVgObj *pVgroup = pMsg->pVgroup;
   SDbObj *pDb = pMsg->pDb;
   assert(pVgroup);
 
-  if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
-           tstrerror(code));
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
-    sdbDeleteRow(&desc);
-    return code;
-  } else {
-    pVgroup->status = TAOS_VG_STATUS_READY;
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
-    (void)sdbUpdateRow(&desc);
-  }
-
   mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
         pDb->name, pVgroup->numOfVnodes);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
@@ -508,6 +498,29 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
+static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
+  SVgObj *pVgroup = pMsg->pVgroup;
+  SDbObj *pDb = pMsg->pDb;
+  assert(pVgroup);
+
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+           tstrerror(code));
+    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
+    sdbDeleteRow(&desc);
+    return code;
+  } else {
+    mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+        pDb->name, pVgroup->numOfVnodes);
+    pVgroup->status = TAOS_VG_STATUS_READY;
+    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
+    (void)sdbUpdateRow(&desc);
+
+    dnodeReprocessMnodeWriteMsg(pMsg);
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  }
+}
+
 int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
   SDbObj *pDb = pMsg->pDb;
@@ -517,30 +530,34 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
   pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
-  if (balanceAllocVnodes(pVgroup) != 0) {
-    mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
+  int32_t code = balanceAllocVnodes(pVgroup);
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("db:%s, no enough dnode to alloc %d vnodes to vgroup, reason:%s", pDb->name, pVgroup->numOfVnodes,
+           tstrerror(code));
     free(pVgroup);
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+    return code;
+  }
+
+  if (pMsg->pVgroup != NULL) {
+    mnodeDecVgroupRef(pMsg->pVgroup);
   }
 
   pMsg->pVgroup = pVgroup;
   mnodeIncVgroupRef(pVgroup);
 
   SSdbOper oper = {
-    .type = SDB_OPER_GLOBAL,
-    .table = tsVgroupSdb,
-    .pObj = pVgroup,
+    .type    = SDB_OPER_GLOBAL,
+    .table   = tsVgroupSdb,
+    .pObj    = pVgroup,
     .rowSize = sizeof(SVgObj),
-    .pMsg = pMsg,
-    .cb = mnodeCreateVgroupCb
+    .pMsg    = pMsg,
+    .reqFp   = mnodeCreateVgroupFp
   };
 
-  int32_t code = sdbInsertRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  code = sdbInsertRow(&oper);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     pMsg->pVgroup = NULL;
     mnodeDestroyVgroup(pVgroup);
-  } else {
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
@@ -784,7 +801,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
   pCfg->walLevel            = pDb->cfg.walLevel;
   pCfg->replications        = (int8_t) pVgroup->numOfVnodes;
   pCfg->wals                = 3;
-  pCfg->quorum              = 1;
+  pCfg->quorum              = pDb->cfg.quorum;
   
   SMDVnodeDesc *pNodes = pVnode->nodes;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
@@ -801,7 +818,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
 SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup) {
   SRpcEpSet epSet = {
     .numOfEps = pVgroup->numOfVnodes,
-    .inUse = 0,
+    .inUse = pVgroup->inUse,
   };
   for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
     strcpy(epSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn);
@@ -891,18 +908,28 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   if (mnodeMsg->received != mnodeMsg->expected) return;
 
   if (mnodeMsg->received == mnodeMsg->successed) {
-    dnodeReprocessMnodeWriteMsg(mnodeMsg);
+     SSdbOper oper = {
+      .type    = SDB_OPER_GLOBAL,
+      .table   = tsVgroupSdb,
+      .pObj    = pVgroup,
+      .rowSize = sizeof(SVgObj),
+      .pMsg    = mnodeMsg,
+      .writeCb = mnodeCreateVgroupCb
+    };
+
+    int32_t code = sdbInsertRowImp(&oper);
+    if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+      mnodeMsg->pVgroup = NULL;
+      mnodeDestroyVgroup(pVgroup);
+      dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+    }
   } else {
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
       .table = tsVgroupSdb,
       .pObj = pVgroup
     };
-    int32_t code = sdbDeleteRow(&oper);
-    if (code != 0) {
-      code = TSDB_CODE_MND_SDB_ERROR;
-    }
-
+    sdbDeleteRow(&oper);
     dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
   }
 }
diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt
index f88209ad9bee533ac24a25f653f80c8b796bfd2f..a8664669d00e15af8f97d681675e15c071f6635b 100644
--- a/src/os/CMakeLists.txt
+++ b/src/os/CMakeLists.txt
@@ -1,6 +1,16 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-ADD_SUBDIRECTORY(linux)
-ADD_SUBDIRECTORY(windows)
-ADD_SUBDIRECTORY(darwin)
+IF (TD_LINUX_64)
+  ADD_SUBDIRECTORY(src/linux64)
+ELSEIF (TD_LINUX_32)
+  ADD_SUBDIRECTORY(src/linux32)
+ELSEIF (TD_DARWIN_64)
+  ADD_SUBDIRECTORY(src/darwin64)  
+ELSEIF (TD_WINDOWS_64)
+  ADD_SUBDIRECTORY(src/windows64)
+ELSEIF (TD_WINDOWS_32)
+  ADD_SUBDIRECTORY(src/windows32)
+ENDIF ()
+
+ADD_SUBDIRECTORY(src/detail)
diff --git a/src/os/darwin/CMakeLists.txt b/src/os/darwin/CMakeLists.txt
deleted file mode 100644
index 116d0ec8c0c7198bc836d6d2dc741daba3d97137..0000000000000000000000000000000000000000
--- a/src/os/darwin/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-IF (TD_DARWIN_64)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(os ${SRC})
-ENDIF ()
diff --git a/src/os/inc/os.h b/src/os/inc/os.h
new file mode 100644
index 0000000000000000000000000000000000000000..896f3afc7ac26bb750d7f8b808534c72fe2a1637
--- /dev/null
+++ b/src/os/inc/os.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_H
+#define TDENGINE_OS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _TD_DARWIN_64
+#include "osDarwin64.h"
+#endif
+
+#ifdef _TD_LINUX_64
+#include "osLinux64.h"
+#endif
+
+#ifdef _TD_LINUX_32
+#include "osLinux32.h"
+#endif
+
+#ifdef _TD_ALPINE
+#include "osAlpine.h"
+#endif
+
+#ifdef _TD_WINDOWS_64
+#include "osWindows64.h"
+#endif
+
+#ifdef _TD_WINDOWS_32
+#include "osWindows32.h"
+#endif
+
+#include "osSpec.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osAlpine.h b/src/os/inc/osAlpine.h
new file mode 100644
index 0000000000000000000000000000000000000000..b8212373ce84fa7b028c0b60e0149547c789e950
--- /dev/null
+++ b/src/os/inc/osAlpine.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <linux/sysctl.h>
+
+typedef int(*__compar_fn_t)(const void *, const void *);
+void  error (int, int, const char *);
+#ifndef PTHREAD_MUTEX_RECURSIVE_NP
+  #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/darwin/inc/os.h b/src/os/inc/osDarwin64.h
similarity index 96%
rename from src/os/darwin/inc/os.h
rename to src/os/inc/osDarwin64.h
index 1c719933a4621c4103f335e07020af0b3449646c..3ceb7ea8f59849f6fd73782c200f1facff90d62b 100644
--- a/src/os/darwin/inc/os.h
+++ b/src/os/inc/osDarwin64.h
@@ -201,9 +201,10 @@ int tsem_destroy(dispatch_semaphore_t *sem);
 
 void osInit();
 
+ssize_t tread(int fd, void *buf, size_t count);
+
 ssize_t twrite(int fd, void *buf, size_t n);
 
-char *taosCharsetReplace(char *charsetstr);
 
 bool taosCheckPthreadValid(pthread_t thread);
 
@@ -217,8 +218,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
 
 void taosPrintOsInfo();
 
-char *taosCharsetReplace(char *charsetstr);
-
 void taosPrintOsInfo();
 
 void taosGetSystemInfo();
@@ -233,6 +232,8 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
 
 void taosSetCoreDump();
 
+int tSystem(const char * cmd);
+
 typedef int(*__compar_fn_t)(const void *, const void *);
 
 // for send function in tsocket.c
@@ -255,6 +256,17 @@ typedef int(*__compar_fn_t)(const void *, const void *);
 #define BUILDIN_CLZ(val)  __builtin_clz(val)
 #define BUILDIN_CTZ(val)  __builtin_ctz(val)
 
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h
new file mode 100644
index 0000000000000000000000000000000000000000..1778b0e31582ed58c51f36f9604ae20c542d871e
--- /dev/null
+++ b/src/os/inc/osLinux32.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h
new file mode 100644
index 0000000000000000000000000000000000000000..0a99f4b74593e43a5c9e579abd82c5511dc5ac4d
--- /dev/null
+++ b/src/os/inc/osLinux64.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+#include <linux/sysctl.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osSpec.h b/src/os/inc/osSpec.h
new file mode 100644
index 0000000000000000000000000000000000000000..d93a6472064cc68f334eb53ec58218984c696765
--- /dev/null
+++ b/src/os/inc/osSpec.h
@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_SPEC_H
+#define TDENGINE_OS_SPEC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define tclose(x) taosCloseSocket(x)
+#define tfree(x)         \
+  do {                   \
+    if (x) {             \
+      free((void *)(x)); \
+      x = 0;             \
+    }                    \
+  } while (0);
+
+#define tstrncpy(dst, src, size)   \
+  do {                             \
+    strncpy((dst), (src), (size)); \
+    (dst)[(size)-1] = 0;           \
+  } while (0);
+
+#ifndef STDERR_FILENO
+#define STDERR_FILENO (2)
+#endif
+
+#define FD_VALID(x) ((x) > STDERR_FILENO)
+#define FD_INITIALIZER  ((int32_t)-1)
+
+#define WCHAR wchar_t
+
+#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
+#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
+
+#ifndef NDEBUG
+#define ASSERT(x) assert(x)
+#else
+#define ASSERT(x)
+#endif
+
+#ifdef UNUSED
+#undefine UNUSED
+#endif
+#define UNUSED(x) ((void)(x))
+
+#ifdef UNUSED_FUNC
+#undefine UNUSED_FUNC
+#endif
+
+#ifdef UNUSED_PARAM
+#undef UNUSED_PARAM
+#endif
+
+#if defined(__GNUC__)
+#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
+#define UNUSED_FUNC __attribute__((unused))
+#else
+#define UNUSED_PARAM(x) x
+#define UNUSED_FUNC
+#endif
+
+#ifdef tListLen
+#undefine tListLen
+#endif
+#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
+
+#if defined(__GNUC__)
+#define FORCE_INLINE inline __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
+  
+#define DEFAULT_COMP(x, y)       \
+  do {                           \
+    if ((x) == (y)) {            \
+      return 0;                  \
+    } else {                     \
+      return (x) < (y) ? -1 : 1; \
+    }                            \
+  } while (0)
+
+#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
+
+// align to 8bytes
+#define ALIGN8(n) ALIGN_NUM(n, 8)
+
+#define POW2(x) ((x) * (x))
+
+#ifndef TAOS_OS_FUNC_MATH
+  #define SWAP(a, b, c)        \
+    do {                       \
+        typeof(a) __tmp = (a); \
+        (a) = (b);             \
+        (b) = __tmp;           \
+    } while (0)
+
+  #define MAX(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a > __b) ? __a : __b; \
+    })
+
+  #define MIN(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a < __b) ? __a : __b; \
+    })
+#endif
+
+#ifndef TAOS_OS_DEF_TIME
+  #define MILLISECOND_PER_SECOND ((int64_t)1000L)
+#endif
+#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
+#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
+#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
+#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
+#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
+#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
+
+#ifndef TAOS_OS_FUNC_SEMPHONE
+  #define tsem_t sem_t
+  #define tsem_init sem_init
+  #define tsem_wait sem_wait
+  #define tsem_post sem_post
+  #define tsem_destroy sem_destroy
+#endif
+
+#ifndef TAOS_OS_FUNC_ATOMIC
+  #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+
+  #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
+
+  #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+#endif
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count);
+ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
+#ifndef TAOS_OS_FUNC_FILE_OP
+  #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
+  #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
+  #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
+  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
+#endif
+
+#ifndef TAOS_OS_FUNC_NETWORK
+  #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) read(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) write(fd, buf, len)
+  #define taosCloseSocket(x) \
+    {                        \
+      if (FD_VALID(x)) {     \
+        close(x);            \
+        x = FD_INITIALIZER;  \
+      }                      \
+    }
+#endif
+
+#ifndef TAOS_OS_FUNC_LZ4
+  #define BUILDIN_CLZL(val) __builtin_clzl(val)
+  #define BUILDIN_CTZL(val) __builtin_ctzl(val)
+  #define BUILDIN_CLZ(val) __builtin_clz(val)
+  #define BUILDIN_CTZ(val) __builtin_ctz(val)
+#endif
+
+#ifndef TAOS_OS_FUNC_WCHAR
+  #define twcslen wcslen
+#endif  
+
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
+void osInit();
+
+// TAOS_OS_FUNC_PTHREAD
+bool taosCheckPthreadValid(pthread_t thread);
+int64_t taosGetPthreadId();
+
+// TAOS_OS_FUNC_SOCKET
+int taosSetNonblocking(int sock, int on);
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
+void taosBlockSIGPIPE();
+
+// TAOS_OS_FUNC_SYSINFO
+void taosGetSystemInfo();
+void taosPrintOsInfo();
+void taosKillSystem();
+int tSystem(const char * cmd) ;
+
+// TAOS_OS_FUNC_CORE
+void taosSetCoreDump();
+
+// TAOS_OS_FUNC_UTIL
+int64_t tsosStr2int64(char *str);
+
+// TAOS_OS_FUNC_TIMER
+void taosMsleep(int mseconds);
+int taosInitTimer(void (*callback)(int), int ms);
+void taosUninitTimer();
+
+// TAOS_OS_FUNC_RAND
+uint32_t taosRand(void);
+void taosRandStr(char* str, int32_t size);
+uint32_t trand(void);
+
+// TAOS_OS_FUNC_FILE
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath);
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
+
+// USE_LIBICONV
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
+bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len);
+int     tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes);
+bool    taosValidateEncodec(const char *encodec);
+char *  taosCharsetReplace(char *charsetstr);
+
+// TAOS_OS_FUNC_MALLOC
+#define TAOS_ALLOC_MODE_DEFAULT 0
+#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
+#define TAOS_ALLOC_MODE_DETECT_LEAK 2
+void   taosSetAllocMode(int mode, const char *path, bool autoDump);
+void   taosDumpMemoryLeak();
+void * tmalloc(size_t size);
+void * tcalloc(size_t nmemb, size_t size);
+size_t tsizeof(void *ptr);
+void   tmemset(void *ptr, int c);
+void * trealloc(void *ptr, size_t size);
+void   tzfree(void *ptr);
+
+// TAOS_OS_FUNC_DIR
+void taosRemoveDir(char *rootDir);
+int  taosMkDir(const char *pathname, mode_t mode); 
+void taosMvDir(char* destDir, char *srcDir);
+
+#ifdef TAOS_RANDOM_FILE_FAIL
+  void taosSetRandomFileFailFactor(int factor);
+  void taosSetRandomFileFailOutput(const char *path);
+  ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
+  #undef taosTRead
+  #undef taosTWrite
+  #undef taosLSeek
+  #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
+#endif
+
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+  ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
+  ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+  ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
+  ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+  #undef taosSend
+  #undef taosSendto
+  #undef taosReadSocket
+  #undef taosWriteSocket
+  #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len)
+#endif
+
+#ifdef TAOS_MEM_CHECK
+  void *  taos_malloc(size_t size, const char *file, uint32_t line);
+  void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
+  void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
+  void    taos_free(void *ptr, const char *file, uint32_t line);
+  char *  taos_strdup(const char *str, const char *file, uint32_t line);
+  char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
+  ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+  #define malloc(size) taos_malloc(size, __FILE__, __LINE__)
+  #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
+  #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
+  #define free(ptr) taos_free(ptr, __FILE__, __LINE__)
+  #define strdup(str) taos_strdup(str, __FILE__, __LINE__)
+  #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+  #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+#endif  // TAOS_MEM_CHECK
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osWindows32.h b/src/os/inc/osWindows32.h
new file mode 100644
index 0000000000000000000000000000000000000000..4744d4beb55d533d48c6631ee3e0ae2f390cff6b
--- /dev/null
+++ b/src/os/inc/osWindows32.h
@@ -0,0 +1,54 @@
+/*
+* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+*
+* This program is free software: you can use, redistribute, and/or modify
+* it under the terms of the GNU Affero General Public License, version 3
+* or later ("AGPL"), as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TDENGINE_PLATFORM_WINDOWS32_H
+#define TDENGINE_PLATFORM_WINDOWS32_H
+
+#include <assert.h>
+#include <ctype.h>
+#include <direct.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <float.h>
+#include <locale.h>
+#include <intrin.h>
+#include <io.h>
+#include <math.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <inttypes.h>
+#include "winsock2.h"
+#include <WS2tcpip.h>
+#include <winbase.h>
+#include <Winsock2.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file
diff --git a/src/os/windows/inc/os.h b/src/os/inc/osWindows64.h
similarity index 99%
rename from src/os/windows/inc/os.h
rename to src/os/inc/osWindows64.h
index a1901aa6e767b109d154be7c4557738976a13b1c..b4687afb82fd05999b60bd50ba2eb797e6a1d11c 100644
--- a/src/os/windows/inc/os.h
+++ b/src/os/inc/osWindows64.h
@@ -48,6 +48,8 @@
 extern "C" {
 #endif
 
+#define TAOS_OS_FUNC_WCHAR
+
 // for function open in stat.h 
 #define S_IRWXU                  _S_IREAD
 #define S_IRWXG                  _S_IWRITE
@@ -78,10 +80,10 @@ extern "C" {
 #define wcsncasecmp _wcsnicmp
 #define strtok_r strtok_s
 #ifdef _TD_GO_DLL_
-  int64_t str2int64(char *str);
+  int64_t tsosStr2int64(char *str);
   uint64_t htonll(uint64_t val);
 #else
-  #define str2int64 _atoi64
+  #define tsosStr2int64 _atoi64
 #endif
 
 #define snprintf _snprintf
@@ -359,7 +361,6 @@ int taosSetNonblocking(int sock, int on);
 
 int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
 
-char *taosCharsetReplace(char *charsetstr);
 
 void taosPrintOsInfo();
 
diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h
deleted file mode 100644
index 00b9f33f1bb3c2443a2ed25097a3a7215c489666..0000000000000000000000000000000000000000
--- a/src/os/linux/inc/os.h
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_PLATFORM_LINUX_H
-#define TDENGINE_PLATFORM_LINUX_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <argp.h>
-#include <arpa/inet.h>
-#include <assert.h>
-#include <ctype.h>
-#include <dirent.h>
-#include <endian.h>
-#include <errno.h>
-#include <float.h>
-#include <ifaddrs.h>
-#include <libgen.h>
-#include <limits.h>
-#include <locale.h>
-#include <math.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/ip.h>
-#include <netinet/tcp.h>
-#include <netinet/udp.h>
-#include <pthread.h>
-#include <pwd.h>
-#include <regex.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/epoll.h>
-#include <sys/eventfd.h>
-#include <sys/file.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/sendfile.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/syscall.h>
-#include <sys/statvfs.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <sys/un.h>
-#include <syslog.h>
-#include <termios.h>
-#include <unistd.h>
-#include <wchar.h>
-#include <wordexp.h>
-#include <wctype.h>
-#include <inttypes.h>
-#include <fcntl.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <error.h>
-
-#define taosCloseSocket(x) \
-  {                        \
-    if (FD_VALID(x)) {     \
-      close(x);            \
-      x = FD_INITIALIZER;  \
-    }                      \
-  }
-  
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags);
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen);
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count);
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count);
-
-#define send(sockfd, buf, len, flags) taos_send_random_fail(sockfd, buf, len, flags)
-#define sendto(sockfd, buf, len, flags, dest_addr, addrlen) \
-          taos_sendto_random_fail(sockfd, buf, len, flags, dest_addr, addrlen)
-#define taosWriteSocket(fd, buf, len) taos_write_random_fail(fd, buf, len)
-#define taosReadSocket(fd, buf, len) taos_read_random_fail(fd, buf, len)
-
-#else
-
-#define taosWriteSocket(fd, buf, len) write(fd, buf, len)
-#define taosReadSocket(fd, buf, len) read(fd, buf, len)
-
-#endif  /* TAOS_RANDOM_NETWORK_FAIL */
-
-#define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-
-#define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
-
-#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define SWAP(a, b, c)      \
-  do {                     \
-    typeof(a) __tmp = (a); \
-    (a) = (b);             \
-    (b) = __tmp;           \
-  } while (0)
-
-#define MAX(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a > __b) ? __a : __b; \
-  })
-
-#define MIN(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a < __b) ? __a : __b; \
-  })
-
-#define MILLISECOND_PER_SECOND ((int64_t)1000L)
-
-#define tsem_t sem_t
-#define tsem_init sem_init
-#define tsem_wait sem_wait
-#define tsem_post sem_post
-#define tsem_destroy sem_destroy
-
-void osInit();
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size);
-
-ssize_t twrite(int fd, void *buf, size_t n);
-
-ssize_t tread(int fd, void *buf, size_t count);
-
-bool taosCheckPthreadValid(pthread_t thread);
-
-void taosResetPthread(pthread_t *thread);
-
-int64_t taosGetPthreadId();
-
-int taosSetNonblocking(int sock, int on);
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
-
-void taosPrintOsInfo();
-
-char *taosCharsetReplace(char *charsetstr);
-
-void taosGetSystemInfo();
-
-void taosKillSystem();
-
-bool taosSkipSocketCheck();
-
-int64_t str2int64(char *str);
-
-void taosSetCoreDump();
-
-void taosBlockSIGPIPE();
-
-int tSystem(const char * cmd) ;
-
-
-#ifdef _ALPINE
-  typedef int(*__compar_fn_t)(const void *, const void *);
-  void  error (int, int, const char *);
-  #ifndef PTHREAD_MUTEX_RECURSIVE_NP
-    #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
-  #endif
-#endif
-
-#ifndef _TD_ARM_32_
-#define BUILDIN_CLZL(val) __builtin_clzl(val)
-#define BUILDIN_CTZL(val) __builtin_ctzl(val)
-#else
-#define BUILDIN_CLZL(val) __builtin_clzll(val)
-#define BUILDIN_CTZL(val) __builtin_ctzll(val)
-#endif
-#define BUILDIN_CLZ(val) __builtin_clz(val)
-#define BUILDIN_CTZ(val) __builtin_ctz(val)
-
-#undef threadlocal
-#ifdef _ISOC11_SOURCE
-  #define threadlocal _Thread_local
-#elif defined(__APPLE__)
-  #define threadlocal
-#elif defined(__GNUC__) && !defined(threadlocal)
-  #define threadlocal __thread
-#else
-  #define threadlocal
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c
deleted file mode 100644
index 216d8942bcac8953ecfd753658e71f6c2a422f4e..0000000000000000000000000000000000000000
--- a/src/os/linux/src/linuxPlatform.c
+++ /dev/null
@@ -1,318 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taosdef.h"
-#include "tglobal.h"
-#include "ttimer.h"
-#include "tulog.h"
-#include "tutil.h"
-
-int64_t str2int64(char *str) {
-  char *endptr = NULL;
-  return strtoll(str, &endptr, 10);
-}
-
-/*
-  to make taosMsleep work,
-   signal SIGALRM shall be blocked in the calling thread,
-
-  sigset_t set;
-  sigemptyset(&set);
-  sigaddset(&set, SIGALRM);
-  pthread_sigmask(SIG_BLOCK, &set, NULL);
-*/
-void taosMsleep(int mseconds) {
-  struct timeval timeout;
-  int            seconds, useconds;
-
-  seconds = mseconds / 1000;
-  useconds = (mseconds % 1000) * 1000;
-  timeout.tv_sec = seconds;
-  timeout.tv_usec = useconds;
-
-  /* sigset_t set; */
-  /* sigemptyset(&set); */
-  /* sigaddset(&set, SIGALRM); */
-  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
-
-  select(0, NULL, NULL, NULL, &timeout);
-
-  /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
-}
-
-bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
-
-int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
-
-int taosSetNonblocking(int sock, int on) {
-  int flags = 0;
-  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
-    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  if (on)
-    flags |= O_NONBLOCK;
-  else
-    flags &= ~O_NONBLOCK;
-
-  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
-    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  return 0;
-}
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
-  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
-}
-static void taosDeleteTimer(void *tharg) {
-  timer_t *pTimer = tharg;
-  timer_delete(*pTimer);
-}
-
-static pthread_t timerThread;
-static timer_t         timerId;
-static volatile bool stopTimer = false;
-
-void *taosProcessAlarmSignal(void *tharg) {
-  // Block the signal
-  sigset_t sigset;
-  sigemptyset(&sigset);
-  sigaddset(&sigset, SIGALRM);
-  sigprocmask(SIG_BLOCK, &sigset, NULL);
-  void (*callback)(int) = tharg;
-
-  struct sigevent sevent = {{0}};
-
-  #ifdef _ALPINE
-    sevent.sigev_notify = SIGEV_THREAD;
-    sevent.sigev_value.sival_int = syscall(__NR_gettid);
-  #else
-    sevent.sigev_notify = SIGEV_THREAD_ID;
-    sevent._sigev_un._tid = syscall(__NR_gettid);
-  #endif
-  
-  sevent.sigev_signo = SIGALRM;
-
-  if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) {
-    uError("Failed to create timer");
-  }
-
-  pthread_cleanup_push(taosDeleteTimer, &timerId);
-
-  struct itimerspec ts;
-  ts.it_value.tv_sec = 0;
-  ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK;
-  ts.it_interval.tv_sec = 0;
-  ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK;
-
-  if (timer_settime(timerId, 0, &ts, NULL)) {
-    uError("Failed to init timer");
-    return NULL;
-  }
-
-  int signo;
-  while (!stopTimer) {
-    if (sigwait(&sigset, &signo)) {
-      uError("Failed to wait signal: number %d", signo);
-      continue;
-    }
-    /* printf("Signal handling: number %d ......\n", signo); */
-
-    callback(0);
-  }
-  
-  pthread_cleanup_pop(1);
-
-  return NULL;
-}
-
-
-int taosInitTimer(void (*callback)(int), int ms) {
-  pthread_attr_t tattr;
-  pthread_attr_init(&tattr);
-  int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback);
-  pthread_attr_destroy(&tattr);
-  if (code != 0) {
-    uError("failed to create timer thread");
-    return -1;
-  }
-  return 0;
-}
-
-void taosUninitTimer() {
-  stopTimer = true;
-  pthread_join(timerThread, NULL);
-}
-
-ssize_t tread(int fd, void *buf, size_t count) {
-  size_t  leftbytes = count;
-  ssize_t readbytes;
-  char *  tbuf = (char *)buf;
-
-  while (leftbytes > 0) {
-    readbytes = read(fd, (void *)tbuf, leftbytes);
-    if (readbytes < 0) {
-      if (errno == EINTR) {
-        continue;
-      } else {
-        return -1;
-      }
-    } else if (readbytes == 0) {
-      return (ssize_t)(count - leftbytes);
-    }
-
-    leftbytes -= readbytes;
-    tbuf += readbytes;
-  }
-
-  return (ssize_t)count;
-}
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size) {
-  size_t  leftbytes = size;
-  ssize_t sentbytes;
-
-  while (leftbytes > 0) {
-    /*
-     * TODO : Think to check if file is larger than 1GB
-     */
-    //if (leftbytes > 1000000000) leftbytes = 1000000000;
-    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
-    if (sentbytes == -1) {
-      if (errno == EINTR) {
-        continue;
-      }
-      else {
-        return -1;
-      }
-    } else if (sentbytes == 0) {
-      return (ssize_t)(size - leftbytes);
-    }
-
-    leftbytes -= sentbytes;
-  }
-
-  return size;
-}
-
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
-void taosBlockSIGPIPE() {
-  sigset_t signal_mask;
-  sigemptyset(&signal_mask);
-  sigaddset(&signal_mask, SIGPIPE);
-  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
-  if (rc != 0) {
-    uError("failed to block SIGPIPE");
-  }
-}
-
-int tSystem(const char * cmd) 
-{ 
-  FILE * fp; 
-  int res; 
-  char buf[1024]; 
-  if (cmd == NULL) { 
-    uError("tSystem cmd is NULL!\n");
-    return -1;
-  } 
-  
-  if ((fp = popen(cmd, "r") ) == NULL) { 
-    uError("popen cmd:%s error: %s/n", cmd, strerror(errno)); 
-    return -1; 
-  } else {
-    while(fgets(buf, sizeof(buf), fp))  { 
-      uDebug("popen result:%s", buf); 
-    } 
-
-    if ((res = pclose(fp)) == -1) { 
-      uError("close popen file pointer fp error!\n");
-    } else { 
-      uDebug("popen res is :%d\n", res);
-    } 
-
-    return res;
-  }
-}
-
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-#define RANDOM_NETWORK_FAIL_FACTOR  20
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return send(sockfd, buf, len, flags);
-}
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
-}
-
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return read(fd, buf, count);
-}
-
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = EINTR;
-    return -1;
-  }
-
-  return write(fd, buf, count);
-}
-
-#endif /* TAOS_RANDOM_NETWORK_FAIL */
diff --git a/src/os/src/alpine/CMakeLists.txt b/src/os/src/alpine/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..79cbf917e2cf4cfa41492379452c7210da0d5e5e
--- /dev/null
+++ b/src/os/src/alpine/CMakeLists.txt
@@ -0,0 +1,12 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+
diff --git a/src/os/linux/src/linuxEnv.c b/src/os/src/alpine/alpineEnv.c
similarity index 100%
rename from src/os/linux/src/linuxEnv.c
rename to src/os/src/alpine/alpineEnv.c
diff --git a/src/os/linux/CMakeLists.txt b/src/os/src/darwin64/CMakeLists.txt
similarity index 61%
rename from src/os/linux/CMakeLists.txt
rename to src/os/src/darwin64/CMakeLists.txt
index 7be4a13f255d0e72b3ad7462edb5424cdf3762ec..71029d9291096ff6ba786114631324469f3a2cb3 100644
--- a/src/os/linux/CMakeLists.txt
+++ b/src/os/src/darwin64/CMakeLists.txt
@@ -1,16 +1,13 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+IF (TD_DARWIN_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
-
   ADD_LIBRARY(os ${SRC})
-  TARGET_LINK_LIBRARIES(os m rt)
 ENDIF ()
 
-SET_SOURCE_FILES_PROPERTIES(src/linuxSysPara.c PROPERTIES COMPILE_FLAGS -w)
diff --git a/src/os/darwin/src/darwinEnv.c b/src/os/src/darwin64/darwinEnv.c
similarity index 100%
rename from src/os/darwin/src/darwinEnv.c
rename to src/os/src/darwin64/darwinEnv.c
diff --git a/src/os/darwin/src/darwinPlatform.c b/src/os/src/darwin64/darwinPlatform.c
similarity index 85%
rename from src/os/darwin/src/darwinPlatform.c
rename to src/os/src/darwin64/darwinPlatform.c
index 44540cf69e51f9515d404e41fc58c0e7d46113d1..045e4a7099507e4c993b69b7f7db4a7690c33899 100644
--- a/src/os/darwin/src/darwinPlatform.c
+++ b/src/os/src/darwin64/darwinPlatform.c
@@ -15,12 +15,15 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
+#include "os.h"
 #include "taosdef.h"
-#include "tutil.h"
+#include "tglobal.h"
+#include "tconfig.h"
+#include "ttimer.h"
 #include "tulog.h"
+#include "tutil.h"
 
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
@@ -159,12 +162,14 @@ int taosInitTimer(void (*callback)(int), int ms) {
   tv.it_interval.tv_usec = 1000 * ms;  // resolution is in msecond
   tv.it_value = tv.it_interval;
 
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
+
+  return 0;
 }
 
 void taosUninitTimer() {
   struct itimerval tv = { 0 };
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
 }
 
 void taosGetSystemTimezone() {
@@ -207,7 +212,7 @@ void taosGetSystemLocale() {
 void taosPrintOsInfo() {}
 
 void taosKillSystem() {
-  tError("function taosKillSystem, exit!");
+  uError("function taosKillSystem, exit!");
   exit(0);
 }
 
@@ -221,95 +226,75 @@ void taosGetSystemInfo() {
 }
 
 void *taosInitTcpClient(char *ip, uint16_t port, char *flabel, int num, void *fp, void *shandle) {
-  tError("function taosInitTcpClient is not implemented in darwin system, exit!");
+  uError("function taosInitTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpClientConnection(void *chandle) {
-  tError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
-  tError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpClientData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpClientData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpClientData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpClient(void *chandle) {
-  tError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpServerConnection(void *chandle) {
-  tError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpServer(void *handle) {
-  tError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
-  tError("function taosInitTcpServer is not implemented in darwin system, exit!");
+  uError("function taosInitTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpServerData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpServerData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpServerData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosFreeMsgHdr(void *hdr) {
-  tError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosMsgHdrSize(void *hdr) {
-  tError("function taosMsgHdrSize is not implemented in darwin system, exit!");
+  uError("function taosMsgHdrSize is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSendMsgHdr(void *hdr, int fd) {
-  tError("function taosSendMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosSendMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) {
-  tError("function taosInitMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosInitMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSetMsgHdrData(void *hdr, char *data, int dataLen) {
-  tError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
+  uError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
   exit(0);
 }
 
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
 bool taosSkipSocketCheck() {
   return true;
 }
diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5902a9356b834b8122fa2eee24f2ad126ce999c1
--- /dev/null
+++ b/src/os/src/detail/CMakeLists.txt
@@ -0,0 +1,14 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(.)
+AUX_SOURCE_DIRECTORY(. SRC)
+SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w)
+SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w)
+
+ADD_LIBRARY(osdetail ${SRC})
+TARGET_LINK_LIBRARIES(osdetail os)
diff --git a/src/os/src/detail/osCoredump.c b/src/os/src/detail/osCoredump.c
new file mode 100644
index 0000000000000000000000000000000000000000..1ead88ab9ec42401e3ddb55f563486352e10d66e
--- /dev/null
+++ b/src/os/src/detail/osCoredump.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "tsystem.h"
+
+#ifndef TAOS_OS_FUNC_CORE
+
+int _sysctl(struct __sysctl_args *args );
+
+void taosSetCoreDump() {
+  if (0 == tsEnableCoreFile) {
+    return;
+  }
+  
+  // 1. set ulimit -c unlimited
+  struct rlimit rlim;
+  struct rlimit rlim_new;
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+    rlim_new.rlim_cur = RLIM_INFINITY;
+    rlim_new.rlim_max = RLIM_INFINITY;
+    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
+      uInfo("set unlimited fail, error: %s", strerror(errno));
+      rlim_new.rlim_cur = rlim.rlim_max;
+      rlim_new.rlim_max = rlim.rlim_max;
+      (void)setrlimit(RLIMIT_CORE, &rlim_new);
+    }
+  }
+
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+  }
+
+#ifndef _TD_ARM_
+  // 2. set the path for saving core file
+  struct __sysctl_args args;
+  int     old_usespid = 0;
+  size_t  old_len     = 0;
+  int     new_usespid = 1;
+  size_t  new_len     = sizeof(new_usespid);
+  
+  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
+  
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  args.newval  = &new_usespid;
+  args.newlen  = new_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
+  }
+  
+  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+
+
+  old_usespid = 0;
+  old_len     = 0;
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
+  }
+  
+  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+#endif
+
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c
new file mode 100644
index 0000000000000000000000000000000000000000..27555938c2bcd9855def598c9fbded0f6024c5fe
--- /dev/null
+++ b/src/os/src/detail/osDir.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_DIR
+
+void taosRemoveDir(char *rootDir) {
+  DIR *dir = opendir(rootDir);
+  if (dir == NULL) return;
+
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+     
+    char filename[1024];
+    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
+    if (de->d_type & DT_DIR) {
+      taosRemoveDir(filename);
+    } else {
+      (void)remove(filename);
+      uInfo("file:%s is removed", filename);
+    }
+  }
+
+  closedir(dir);
+  rmdir(rootDir);
+
+  uInfo("dir:%s is removed", rootDir);
+}
+
+int taosMkDir(const char *path, mode_t mode) {
+  int code = mkdir(path, 0755);
+  if (code < 0 && errno == EEXIST) code = 0;
+  return code;
+}
+
+void taosMvDir(char* destDir, char *srcDir) {
+  if (0 == tsEnableVnodeBak) {
+    uInfo("vnode backup not enabled");
+    return;
+  }
+
+  char shellCmd[1024+1] = {0}; 
+  
+  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
+  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
+  tSystem(shellCmd);
+  uInfo("shell cmd:%s is executed", shellCmd);
+}
+
+
+#endif
\ No newline at end of file
diff --git a/src/util/src/tfile.c b/src/os/src/detail/osFail.c
similarity index 55%
rename from src/util/src/tfile.c
rename to src/os/src/detail/osFail.c
index d1a9c2461518a787920eac6de591b320a265d20d..e0eb200851f976e1c9dad47afbc152bfb021e4e5 100644
--- a/src/util/src/tfile.c
+++ b/src/os/src/detail/osFail.c
@@ -13,31 +13,62 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <error.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <unistd.h>
+#define _DEFAULT_SOURCE
+#include "os.h"
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
+#define RANDOM_NETWORK_FAIL_FACTOR  20
 
-#include "os.h"
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+
+ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return send(sockfd, buf, len, flags);
+}
+
+ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
+                           socklen_t addrlen) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
+}
+
+ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return read(fd, buf, count);
+}
+
+ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = EINTR;
+    return -1;
+  }
+
+  return write(fd, buf, count);
+}
+
+#endif //TAOS_RANDOM_NETWORK_FAIL
 
 #ifdef TAOS_RANDOM_FILE_FAIL
 
 static int random_file_fail_factor = 20;
 static FILE *fpRandomFileFailOutput = NULL;
 
-void taosSetRandomFileFailFactor(int factor)
-{
+void taosSetRandomFileFailFactor(int factor) { 
   random_file_fail_factor = factor;
 }
 
-static void close_random_file_fail_output()
-{
+static void close_random_file_fail_output() {
   if (fpRandomFileFailOutput != NULL) {
     if (fpRandomFileFailOutput != stdout) {
       fclose(fpRandomFileFailOutput);
@@ -46,8 +77,7 @@ static void close_random_file_fail_output()
   }
 }
 
-static void random_file_fail_output_sig(int sig)
-{
+static void random_file_fail_output_sig(int sig) {
   fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
 
   struct sigaction act = {0};
@@ -58,8 +88,7 @@ static void random_file_fail_output_sig(int sig)
   exit(EXIT_FAILURE);
 }
 
-void taosSetRandomFileFailOutput(const char *path)
-{
+void taosSetRandomFileFailOutput(const char *path) {
   if (path == NULL) {
     fpRandomFileFailOutput = stdout;
   } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) {
@@ -75,43 +104,38 @@ void taosSetRandomFileFailOutput(const char *path)
   sigaction(SIGSEGV, &act, NULL);
   sigaction(SIGILL, &act, NULL);
 }
-#endif
 
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return tread(fd, buf, count);
+
+  return taosTReadImp(fd, buf, count);
 }
 
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return twrite(fd, buf, count);
+
+  return taosTWriteImp(fd, buf, count);
 }
 
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
+
   return lseek(fd, offset, whence);
 }
+
+#endif //TAOS_RANDOM_FILE_FAIL
diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c
new file mode 100644
index 0000000000000000000000000000000000000000..5502dfb4a6c20bc593c81892ebb0d863c2de258d
--- /dev/null
+++ b/src/os/src/detail/osFile.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttime.h"
+
+#ifndef TAOS_OS_FUNC_FILE
+
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+  
+  char tmpPath[PATH_MAX];
+  char *tmpDir = "/tmp/";
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}
+
+// rename file name
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+  int32_t ts = taosGetTimestampSec();
+
+  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
+
+  char *delimiterPos = strrchr(fullPath, delimiter);
+  if (delimiterPos == NULL) return -1;
+
+  int32_t fileNameLen = 0;
+  if (suffix)
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
+  else
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
+
+  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+  if (*dstPath == NULL) {
+    *dstPath = calloc(1, len + 1);
+    if (*dstPath == NULL) return -1;
+  }
+
+  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
+  strncat(*dstPath, fname, (size_t)fileNameLen);
+  (*dstPath)[len] = 0;
+
+  return rename(fullPath, *dstPath);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osFileOp.c b/src/os/src/detail/osFileOp.c
new file mode 100644
index 0000000000000000000000000000000000000000..897b6c3f034b0a4f79ed606e85dc534d746ee075
--- /dev/null
+++ b/src/os/src/detail/osFileOp.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count) {
+  size_t  leftbytes = count;
+  ssize_t readbytes;
+  char *  tbuf = (char *)buf;
+
+  while (leftbytes > 0) {
+    readbytes = read(fd, (void *)tbuf, leftbytes);
+    if (readbytes < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (readbytes == 0) {
+      return (ssize_t)(count - leftbytes);
+    }
+
+    leftbytes -= readbytes;
+    tbuf += readbytes;
+  }
+
+  return (ssize_t)count;
+}
+
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
+  size_t  leftbytes = size;
+  ssize_t sentbytes;
+
+  while (leftbytes > 0) {
+    /*
+     * TODO : Think to check if file is larger than 1GB
+     */
+    //if (leftbytes > 1000000000) leftbytes = 1000000000;
+    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
+    if (sentbytes == -1) {
+      if (errno == EINTR) {
+        continue;
+      }
+      else {
+        return -1;
+      }
+    } else if (sentbytes == 0) {
+      return (ssize_t)(size - leftbytes);
+    }
+
+    leftbytes -= sentbytes;
+  }
+
+  return size;
+}
+
+ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
+  size_t nleft = n; 
+  ssize_t nwritten = 0;
+  char *tbuf = (char *)buf;
+
+  while (nleft > 0) {
+    nwritten = write(fd, (void *)tbuf, nleft);
+    if (nwritten < 0) {
+      if (errno == EINTR) {
+        continue;
+      }
+      return -1;
+    }
+    nleft -= nwritten;
+    tbuf += nwritten;
+  }
+
+  return n;
+}
diff --git a/src/os/src/detail/osMalloc.c b/src/os/src/detail/osMalloc.c
new file mode 100644
index 0000000000000000000000000000000000000000..66377827daff8d748d0d8017539f51b589f9f961
--- /dev/null
+++ b/src/os/src/detail/osMalloc.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_MALLOC
+
+void *tmalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = malloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+void *tcalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = tmalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  tmemset(ret, 0);
+  return ret;
+}
+
+size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
+
+void * trealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return tmalloc(size);
+
+  if (size <= tsizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  tptr = realloc(tptr, tsize);
+  if (tptr == NULL) return NULL;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+void tzfree(void *ptr) {
+  if (ptr) {
+    free((void *)((char *)ptr - sizeof(size_t)));
+  }
+}
+
+#endif
\ No newline at end of file
diff --git a/src/util/src/tmem.c b/src/os/src/detail/osMem.c
similarity index 99%
rename from src/util/src/tmem.c
rename to src/os/src/detail/osMem.c
index 9c512ad8dc5938626d983d8263271f060abdc78c..93943c6543c9b24ff58b566e12ef8358d61e3a63 100644
--- a/src/util/src/tmem.c
+++ b/src/os/src/detail/osMem.c
@@ -13,13 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tulog.h"
 
-#define TAOS_MEM_CHECK_IMPL
-#include "tutil.h"
-
-
 #ifdef TAOS_MEM_CHECK
 
 static int allocMode = TAOS_ALLOC_MODE_DEFAULT;
diff --git a/src/os/src/detail/osPThread.c b/src/os/src/detail/osPThread.c
new file mode 100644
index 0000000000000000000000000000000000000000..18207dc12e937dc6f6a49d8778465a395c8d9078
--- /dev/null
+++ b/src/os/src/detail/osPThread.c
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_PTHREAD
+
+bool    taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
+int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c
new file mode 100644
index 0000000000000000000000000000000000000000..7e5f58563455ce9271a1bac0f41fe722f0896a1f
--- /dev/null
+++ b/src/os/src/detail/osRand.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_RAND
+
+uint32_t taosRand(void) { return rand(); }
+
+uint32_t trand(void) {
+  int fd;
+  int seed;
+
+  fd = open("/dev/urandom", 0);
+  if (fd < 0) {
+    seed = time(0);
+  } else {
+    int len = read(fd, &seed, sizeof(seed));
+    if (len < 0) {
+      seed = time(0);
+    }
+    close(fd);
+  }
+
+  return (uint32_t)seed;
+}
+
+#endif
+
+void taosRandStr(char* str, int32_t size) {
+  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
+  int32_t     len = 39;
+
+  for (int32_t i = 0; i < size; ++i) {
+    str[i] = set[taosRand() % len];
+  }
+}
\ No newline at end of file
diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c
new file mode 100644
index 0000000000000000000000000000000000000000..ea0b92de5fe9d04eea34e47427342742cffa4ae7
--- /dev/null
+++ b/src/os/src/detail/osSocket.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_SOCKET
+
+int taosSetNonblocking(int sock, int on) {
+  int flags = 0;
+  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
+    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  if (on)
+    flags |= O_NONBLOCK;
+  else
+    flags &= ~O_NONBLOCK;
+
+  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
+    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  return 0;
+}
+
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
+  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
+}
+
+void taosBlockSIGPIPE() {
+  sigset_t signal_mask;
+  sigemptyset(&signal_mask);
+  sigaddset(&signal_mask, SIGPIPE);
+  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+  if (rc != 0) {
+    uError("failed to block SIGPIPE");
+  }
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/src/detail/osSysinfo.c
similarity index 79%
rename from src/os/linux/src/linuxSysPara.c
rename to src/os/src/detail/osSysinfo.c
index 1331503619cd6295221bb9107334829d41e96fe3..7a395285af5ac7bc883c4dae0130f3e838e4ab7c 100644
--- a/src/os/linux/src/linuxSysPara.c
+++ b/src/os/src/detail/osSysinfo.c
@@ -15,12 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tsysctl.h"
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tutil.h"
 #include "tsystem.h"
+#include "taosdef.h"
+
+#ifndef TAOS_OS_FUNC_SYSINFO
 
 #define PROCESS_ITEM 12
 
@@ -200,7 +201,7 @@ static void taosGetSystemTimezone() {
    * Asia/Shanghai   (CST, +0800)
    * Europe/London   (BST, +0100)
    */
-  sprintf(tsTimezone, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
+  snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
 
   // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
   uInfo("timezone not configured, set to system default:%s", tsTimezone);
@@ -234,7 +235,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       uError("can't get locale from system, set it to en_US.UTF-8");
       strcpy(tsLocale, "en_US.UTF-8");
     } else {
-      tstrncpy(tsLocale, locale, sizeof(tsLocale));
+      tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
       uError("locale not configured, set to system default:%s", tsLocale);
     }
   }
@@ -247,7 +248,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       str++;
 
       char *revisedCharset = taosCharsetReplace(str);
-      tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
+      tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
 
       free(revisedCharset);
       uWarn("charset not configured, set to system default:%s", tsCharset);
@@ -542,126 +543,31 @@ void taosKillSystem() {
   kill(tsProcId, 2);
 }
 
-int _sysctl(struct __sysctl_args *args );
-void taosSetCoreDump() {
-  if (0 == tsEnableCoreFile) {
-    return;
-  }
-  
-  // 1. set ulimit -c unlimited
-  struct rlimit rlim;
-  struct rlimit rlim_new;
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
-    rlim_new.rlim_cur = RLIM_INFINITY;
-    rlim_new.rlim_max = RLIM_INFINITY;
-    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
-      uInfo("set unlimited fail, error: %s", strerror(errno));
-      rlim_new.rlim_cur = rlim.rlim_max;
-      rlim_new.rlim_max = rlim.rlim_max;
-      (void)setrlimit(RLIMIT_CORE, &rlim_new);
-    }
-  }
-
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+int tSystem(const char *cmd) {
+  FILE *fp;
+  int   res;
+  char  buf[1024];
+  if (cmd == NULL) {
+    uError("tSystem cmd is NULL!\n");
+    return -1;
   }
 
-#ifndef _TD_ARM_
-  // 2. set the path for saving core file
-  struct __sysctl_args args;
-  int     old_usespid = 0;
-  size_t  old_len     = 0;
-  int     new_usespid = 1;
-  size_t  new_len     = sizeof(new_usespid);
-  
-  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
-  
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  args.newval  = &new_usespid;
-  args.newlen  = new_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
-  }
-  
-  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-
+  if ((fp = popen(cmd, "r")) == NULL) {
+    uError("popen cmd:%s error: %s/n", cmd, strerror(errno));
+    return -1;
+  } else {
+    while (fgets(buf, sizeof(buf), fp)) {
+      uDebug("popen result:%s", buf);
+    }
 
-  old_usespid = 0;
-  old_len     = 0;
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
-  }
-  
-  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-#endif
-  
-#if 0
-  // 3. create the path for saving core file
-  int status; 
-  char coredump_dir[32] = "/var/log/taosdump";
-  if (opendir(coredump_dir) == NULL) {
-    status = mkdir(coredump_dir, S_IRWXU | S_IRWXG | S_IRWXO); 
-    if (status) {
-      uInfo("mkdir fail, error: %s\n", strerror(errno));
+    if ((res = pclose(fp)) == -1) {
+      uError("close popen file pointer fp error!\n");
+    } else {
+      uDebug("popen res is :%d\n", res);
     }
-  }
 
-  // 4. set kernel.core_pattern
-   struct __sysctl_args args;
-   char    old_corefile[128];
-   size_t  old_len;
-   char    new_corefile[128] = "/var/log/taosdump/core-%e-%p";
-   size_t  new_len = sizeof(new_corefile);
-   
-   int name[] = {CTL_KERN, KERN_CORE_PATTERN};
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   args.newval  = new_corefile;
-   args.newlen  = new_len;
-
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) set fail: %s", strerror(errno));
-   }
-   
-   uInfo("The old kern_core_pattern: %*s\n", old_len, old_corefile);
-
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) get fail: %s", strerror(errno));
-   }
-   
-   uInfo("The new kern_core_pattern: %*s\n", old_len, old_corefile);
-#endif
-  
+    return res;
+  }
 }
 
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c
new file mode 100644
index 0000000000000000000000000000000000000000..e0a2e903146927402ae9bda22eaf85990e899e2e
--- /dev/null
+++ b/src/os/src/detail/osTimer.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosdef.h"
+#include "tglobal.h"
+#include "ttimer.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_TIMER
+
+/*
+  to make taosMsleep work,
+   signal SIGALRM shall be blocked in the calling thread,
+
+  sigset_t set;
+  sigemptyset(&set);
+  sigaddset(&set, SIGALRM);
+  pthread_sigmask(SIG_BLOCK, &set, NULL);
+*/
+void taosMsleep(int mseconds) {
+  struct timeval timeout;
+  int            seconds, useconds;
+
+  seconds = mseconds / 1000;
+  useconds = (mseconds % 1000) * 1000;
+  timeout.tv_sec = seconds;
+  timeout.tv_usec = useconds;
+
+  /* sigset_t set; */
+  /* sigemptyset(&set); */
+  /* sigaddset(&set, SIGALRM); */
+  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
+
+  select(0, NULL, NULL, NULL, &timeout);
+
+  /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
+}
+
+
+static void taosDeleteTimer(void *tharg) {
+  timer_t *pTimer = tharg;
+  timer_delete(*pTimer);
+}
+
+static pthread_t timerThread;
+static timer_t         timerId;
+static volatile bool stopTimer = false;
+static void *taosProcessAlarmSignal(void *tharg) {
+  // Block the signal
+  sigset_t sigset;
+  sigemptyset(&sigset);
+  sigaddset(&sigset, SIGALRM);
+  sigprocmask(SIG_BLOCK, &sigset, NULL);
+  void (*callback)(int) = tharg;
+
+  struct sigevent sevent = {{0}};
+
+  #ifdef _ALPINE
+    sevent.sigev_notify = SIGEV_THREAD;
+    sevent.sigev_value.sival_int = syscall(__NR_gettid);
+  #else
+    sevent.sigev_notify = SIGEV_THREAD_ID;
+    sevent._sigev_un._tid = syscall(__NR_gettid);
+  #endif
+  
+  sevent.sigev_signo = SIGALRM;
+
+  if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) {
+    uError("Failed to create timer");
+  }
+
+  pthread_cleanup_push(taosDeleteTimer, &timerId);
+
+  struct itimerspec ts;
+  ts.it_value.tv_sec = 0;
+  ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK;
+  ts.it_interval.tv_sec = 0;
+  ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK;
+
+  if (timer_settime(timerId, 0, &ts, NULL)) {
+    uError("Failed to init timer");
+    return NULL;
+  }
+
+  int signo;
+  while (!stopTimer) {
+    if (sigwait(&sigset, &signo)) {
+      uError("Failed to wait signal: number %d", signo);
+      continue;
+    }
+    /* printf("Signal handling: number %d ......\n", signo); */
+
+    callback(0);
+  }
+  
+  pthread_cleanup_pop(1);
+
+  return NULL;
+}
+
+int taosInitTimer(void (*callback)(int), int ms) {
+  pthread_attr_t tattr;
+  pthread_attr_init(&tattr);
+  int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback);
+  pthread_attr_destroy(&tattr);
+  if (code != 0) {
+    uError("failed to create timer thread");
+    return -1;
+  }
+  return 0;
+}
+
+void taosUninitTimer() {
+  stopTimer = true;
+  pthread_join(timerThread, NULL);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osUtil.c b/src/os/src/detail/osUtil.c
new file mode 100644
index 0000000000000000000000000000000000000000..10576c9a0a01a0704fc8b54564364fba9424d6da
--- /dev/null
+++ b/src/os/src/detail/osUtil.c
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_UTIL
+
+int64_t tsosStr2int64(char *str) {
+  char *endptr = NULL;
+  return strtoll(str, &endptr, 10);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osWchar.c b/src/os/src/detail/osWchar.c
new file mode 100644
index 0000000000000000000000000000000000000000..4e02973e500076780f0a745c10eecb5fe824bb82
--- /dev/null
+++ b/src/os/src/detail/osWchar.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "taosdef.h"
+
+#ifndef TAOS_OS_FUNC_WCHAR
+
+int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
+  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
+}
+
+#endif
+
+#ifdef USE_LIBICONV
+#include "iconv.h"
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
+  size_t  ucs4_input_len = ucs4_max_len;
+  size_t  outLen = ucs4_max_len;
+  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
+    iconv_close(cd);
+    return -1;
+  }
+
+  iconv_close(cd);
+  return (int32_t)(ucs4_max_len - outLen);
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
+  size_t  ucs4_input_len = mbsLength;
+  size_t  outLeft = ucs4_max_len;
+  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
+    iconv_close(cd);
+    return false;
+  }
+
+  iconv_close(cd);
+  if (len != NULL) {
+    *len = ucs4_max_len - outLeft;
+  }
+
+  return true;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
+  if (cd == (iconv_t)(-1)) {
+    return false;
+  }
+
+  iconv_close(cd);
+  return true;
+}
+
+#else
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  mbstate_t state = {0};
+  int32_t   len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  memset(&state, 0, sizeof(state));
+  len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  return len;
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  mbstate_t state = {0};
+  int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
+  *len = retlen;
+
+  return retlen >= 0;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  return true;
+}
+
+#endif
+
+typedef struct CharsetPair {
+  char *oldCharset;
+  char *newCharset;
+} CharsetPair;
+
+char *taosCharsetReplace(char *charsetstr) {
+  CharsetPair charsetRep[] = {
+      { "utf8", "UTF-8" }, { "936", "CP936" },
+  };
+
+  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
+    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
+      return strdup(charsetRep[i].newCharset);
+    }
+  }
+
+  return strdup(charsetstr);
+}
\ No newline at end of file
diff --git a/src/os/src/linux64/CMakeLists.txt b/src/os/src/linux64/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..79cbf917e2cf4cfa41492379452c7210da0d5e5e
--- /dev/null
+++ b/src/os/src/linux64/CMakeLists.txt
@@ -0,0 +1,12 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+
diff --git a/src/os/src/linux64/linuxEnv.c b/src/os/src/linux64/linuxEnv.c
new file mode 100644
index 0000000000000000000000000000000000000000..0a9d81311a3f86947e75cd9190fbf306dc1871f7
--- /dev/null
+++ b/src/os/src/linux64/linuxEnv.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void osInit() {
+  strcpy(configDir, "/etc/taos");
+  strcpy(tsVnodeDir, "");
+  strcpy(tsDnodeDir, "");
+  strcpy(tsMnodeDir, "");
+  strcpy(tsDataDir, "/var/lib/taos");
+  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsScriptDir, "/etc/taos");
+  strcpy(tsOsName, "Linux");
+}
\ No newline at end of file
diff --git a/src/os/windows/CMakeLists.txt b/src/os/src/windows64/CMakeLists.txt
similarity index 100%
rename from src/os/windows/CMakeLists.txt
rename to src/os/src/windows64/CMakeLists.txt
diff --git a/src/os/windows/src/twindows.c b/src/os/src/windows64/twindows.c
similarity index 99%
rename from src/os/windows/src/twindows.c
rename to src/os/src/windows64/twindows.c
index 295824846e518aa6a03078e8f0d7d5a44d0da621..4284dcb14edac80568e9930d0517f2369b67cec7 100644
--- a/src/os/windows/src/twindows.c
+++ b/src/os/src/windows64/twindows.c
@@ -398,7 +398,7 @@ char *strndup(const char *s, size_t n) {
 void taosSetCoreDump() {}
 
 #ifdef _TD_GO_DLL_
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
diff --git a/src/os/windows/src/twinenv.c b/src/os/src/windows64/twinenv.c
similarity index 100%
rename from src/os/windows/src/twinenv.c
rename to src/os/src/windows64/twinenv.c
diff --git a/src/os/windows/src/twingetline.c b/src/os/src/windows64/twingetline.c
similarity index 100%
rename from src/os/windows/src/twingetline.c
rename to src/os/src/windows64/twingetline.c
diff --git a/src/os/windows/src/twinmsghdr.c b/src/os/src/windows64/twinmsghdr.c
similarity index 100%
rename from src/os/windows/src/twinmsghdr.c
rename to src/os/src/windows64/twinmsghdr.c
diff --git a/src/os/windows/src/twinsocket.c b/src/os/src/windows64/twinsocket.c
similarity index 100%
rename from src/os/windows/src/twinsocket.c
rename to src/os/src/windows64/twinsocket.c
diff --git a/src/os/windows/src/twinstrptime.c b/src/os/src/windows64/twinstrptime.c
similarity index 100%
rename from src/os/windows/src/twinstrptime.c
rename to src/os/src/windows64/twinstrptime.c
diff --git a/src/os/windows/src/twintcpclient.c b/src/os/src/windows64/twintcpclient.c
similarity index 100%
rename from src/os/windows/src/twintcpclient.c
rename to src/os/src/windows64/twintcpclient.c
diff --git a/src/os/windows/src/twintcpserver.c b/src/os/src/windows64/twintcpserver.c
similarity index 100%
rename from src/os/windows/src/twintcpserver.c
rename to src/os/src/windows64/twintcpserver.c
diff --git a/src/os/windows/src/twintime.c b/src/os/src/windows64/twintime.c
similarity index 100%
rename from src/os/windows/src/twintime.c
rename to src/os/src/windows64/twintime.c
diff --git a/src/os/windows/src/twintimer.c b/src/os/src/windows64/twintimer.c
similarity index 100%
rename from src/os/windows/src/twintimer.c
rename to src/os/src/windows64/twintimer.c
diff --git a/src/os/src/windows64/w64File.c b/src/os/src/windows64/w64File.c
new file mode 100644
index 0000000000000000000000000000000000000000..140277f990293f186c198fe752b9620a0f044305
--- /dev/null
+++ b/src/os/src/windows64/w64File.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+    char tmpPath[PATH_MAX];
+
+  char *tmpDir = getenv("tmp");
+  if (tmpDir == NULL) {
+    tmpDir = "";
+  }
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}
diff --git a/src/os/src/windows64/w64Wchar.c b/src/os/src/windows64/w64Wchar.c
new file mode 100644
index 0000000000000000000000000000000000000000..d5930d1de9856860af154776f78bd6f1d347f366
--- /dev/null
+++ b/src/os/src/windows64/w64Wchar.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+size_t twcslen(const wchar_t *wcs) {
+  int *wstr = (int *)wcs;
+  if (NULL == wstr) {
+    return 0;
+  }
+
+  size_t n = 0;
+  while (1) {
+    if (0 == *wstr++) {
+      break;
+    }
+    n++;
+  }
+
+  return n;
+}
+
+int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) {
+  for (int i = 0; i < bytes; ++i) {
+    int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4);
+    int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4);
+
+    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
+      return f1 - f2;
+    } else if (f1 == 0 && f2 == 0) {
+      return 0;
+    }
+
+    if (f1 != f2) {
+      return f1 - f2;
+    }
+  }
+
+  return 0;
+
+#if 0
+  int32_t ucs4_max_len = bytes + 4;
+  char *f1_mbs = calloc(bytes, 1);
+  char *f2_mbs = calloc(bytes, 1);
+  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
+    return -1;
+  }
+  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
+    return -1;
+  }
+  int32_t ret = strcmp(f1_mbs, f2_mbs);
+  free(f1_mbs);
+  free(f2_mbs);
+  return ret;
+#endif
+}
diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt
index 27bcae943c53cc6a43206a7a4d1875d7aed06a65..3280a37c9490b32d1b1931172c32c28d320f574b 100644
--- a/src/plugins/http/CMakeLists.txt
+++ b/src/plugins/http/CMakeLists.txt
@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(http ${SRC})
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index 225977abae698ccfeb12c36562d5588a3f166027..f46d3fb4270d83fb7e9166e1b4a2442e2c7a51df 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -46,19 +46,20 @@ static void httpDestroyContext(void *data) {
   httpReleaseSession(pContext);
   atomic_sub_fetch_32(&pThread->numOfContexts, 1);
   
+  httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount,
+            data, pContext->pThread->label, pContext->pThread->numOfContexts);
   pContext->pThread = 0;
   pContext->state = HTTP_CONTEXT_STATE_CLOSED;
 
   // avoid double free
   httpFreeJsonBuf(pContext);
   httpFreeMultiCmds(pContext);
-  
-  httpDebug("context:%p, is destroyed, refCount:%d data:%p", pContext, pContext->refCount, data);
+
   tfree(pContext);
 }
 
 bool httpInitContexts() {
-  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, false, httpDestroyContext, "restc");
+  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc");
   if (tsHttpServer.contextCache == NULL) {
     httpError("failed to init context cache");
     return false;
@@ -134,7 +135,10 @@ HttpContext *httpGetContext(void *ptr) {
 
 void httpReleaseContext(HttpContext *pContext) {
   int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1);
-  assert(refCount >= 0);
+  if (refCount < 0) {
+    httpError("context:%p, is already released, refCount:%d", pContext, refCount);
+    return;
+  }
 
   HttpContext **ppContext = pContext->ppContext;
   httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c
index 82666826bcbdace2c1d5a58c1cda82c89553fe4e..4748f03b662479db9f5dcdbc64b5294e9d6a9094 100644
--- a/src/plugins/http/src/httpJson.c
+++ b/src/plugins/http/src/httpJson.c
@@ -45,7 +45,7 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) {
 
   do {
     if (pContext->fd > 2){
-      len = (int)send(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
+      len = (int)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
     }
     else {
       return sz;
diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index 177d447f10f27b2ba334402b4ca7ee219099999d..ec3d2c0d44146e732a5e7ac78218844b87bf7b53 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -299,12 +299,14 @@ static void *httpAcceptHttpConnection(void *arg) {
       totalFds += pServer->pThreads[i].numOfContexts;
     }
 
+#if 0
     if (totalFds > tsHttpCacheSessions * 100) {
       httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd,
                 inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
       taosCloseSocket(connFd);
       continue;
     }
+#endif    
 
     taosKeepTcpAlive(connFd);
     taosSetNonblocking(connFd, 1);
@@ -336,7 +338,7 @@ static void *httpAcceptHttpConnection(void *arg) {
 
     // notify the data process, add into the FdObj list
     atomic_add_fetch_32(&pThread->numOfContexts, 1);
-    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalFds:%d, accept a new connection", pContext,
+    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalContext:%d, accept a new connection", pContext,
               connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds);
 
     // pick up next thread for next connection
diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt
index 0ea5507501258d4161bd9ef22973098e979d3742..9a202867972821a70c93d8b62ca6dd7671832ba3 100644
--- a/src/plugins/monitor/CMakeLists.txt
+++ b/src/plugins/monitor/CMakeLists.txt
@@ -3,11 +3,11 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(./src SRC)
   ADD_LIBRARY(monitor ${SRC})
diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt
index cc88bd95a45cf0f09ac8024f256685837335935d..5fc8f9039fce7a46539a0b63300a6ab4692889c0 100644
--- a/src/plugins/mqtt/CMakeLists.txt
+++ b/src/plugins/mqtt/CMakeLists.txt
@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(mqtt ${SRC})
diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt
index 620fd217de4b29c6e4a80ae841b75295b06fcb98..625b85d908537c361b446bee1bb6b94d5e2140e5 100644
--- a/src/query/CMakeLists.txt
+++ b/src/query/CMakeLists.txt
@@ -1,9 +1,9 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h
index 00049b486d765f341a97ce45250ec764f67502db..01b4c16ac1649e090e9acbe267dd83f83f141fe0 100644
--- a/src/query/inc/qAst.h
+++ b/src/query/inc/qAst.h
@@ -42,7 +42,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *);
 
 /**
  * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
- *
  */
 typedef struct tQueryInfo {
   uint8_t       optr;     // expression operator
diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index 42dfce42a6f893af32c5175bc52a094658c9a3ab..bd2e0a4470a4c067a4f1da1af9642351e6391d9a 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -20,8 +20,8 @@
 #include "hash.h"
 #include "qFill.h"
 #include "qResultbuf.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "query.h"
 #include "taosdef.h"
 #include "tarray.h"
@@ -43,7 +43,7 @@ typedef struct SSqlGroupbyExpr {
 
 typedef struct SPosInfo {
   int32_t pageId;
-  int16_t rowId;
+  int32_t rowId;
 } SPosInfo;
 
 typedef struct SWindowStatus {
@@ -177,13 +177,18 @@ typedef struct SQueryRuntimeEnv {
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
 } SQueryRuntimeEnv;
 
+enum {
+  QUERY_RESULT_NOT_READY = 1,
+  QUERY_RESULT_READY     = 2,
+};
+
 typedef struct SQInfo {
   void*            signature;
   int32_t          pointsInterpo;
   int32_t          code;  // error code to returned to client
-  sem_t            dataReady;
+//  sem_t            dataReady;
+
   void*            tsdb;
-  void*            param;
   int32_t          vgId;
   STableGroupInfo  tableGroupInfo;       // table id list < only includes the STable list>
   STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
@@ -200,8 +205,11 @@ typedef struct SQInfo {
    */
   int32_t          tableIndex;
   int32_t          numOfGroupResultPages;
-  void*            pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
+  void*            pBuf;        // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
 
+  pthread_mutex_t  lock;        // used to synchronize the rsp/query threads
+  int32_t          dataReady;   // denote if query result is ready or not
+  void*            rspContext;  // response context
 } SQInfo;
 
 #endif  // TDENGINE_QUERYEXECUTOR_H
diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h
index b57c48933f0878ca86384219b1c7f66446db28eb..36fc0c9820a8338973e718c473e7619da1990ad8 100644
--- a/src/query/inc/qExtbuffer.h
+++ b/src/query/inc/qExtbuffer.h
@@ -19,7 +19,6 @@
 extern "C" {
 #endif
 
-
 #include "os.h"
 #include "taosmsg.h"
 
@@ -28,9 +27,9 @@ extern "C" {
 #include "tdataformat.h"
 #include "talgo.h"
 
-#define DEFAULT_PAGE_SIZE (1024L*4)  // 16k larger than the SHistoInfo
-#define MAX_TMPFILE_PATH_LENGTH PATH_MAX
+#define MAX_TMPFILE_PATH_LENGTH        PATH_MAX
 #define INITIAL_ALLOCATION_BUFFER_SIZE 64
+#define DEFAULT_PAGE_SIZE              (4096L)  // 16k larger than the SHistoInfo
 
 typedef enum EXT_BUFFER_FLUSH_MODEL {
   /*
@@ -126,7 +125,7 @@ typedef struct tExtMemBuffer {
  * @param pModel
  * @return
  */
-tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel);
+tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel);
 
 /**
  *
diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h
index 8c8afb0957c862042e1da99e211ed06e091dee4a..5303251d981f9e12bb92088b65cf6231d6e45f88 100644
--- a/src/query/inc/qResultbuf.h
+++ b/src/query/inc/qResultbuf.h
@@ -13,50 +13,85 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODEQUERYUTIL_H
-#define TDENGINE_VNODEQUERYUTIL_H
+#ifndef TDENGINE_QRESULTBUF_H
+#define TDENGINE_QRESULTBUF_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <tlist.h>
 #include "hash.h"
 #include "os.h"
 #include "qExtbuffer.h"
+#include "tlockfree.h"
 
 typedef struct SArray* SIDList;
 
+typedef struct SPageDiskInfo {
+  int32_t offset;
+  int32_t length;
+} SPageDiskInfo;
+
+typedef struct SPageInfo {
+  SListNode*    pn;       // point to list node
+  int32_t       pageId;
+  SPageDiskInfo info;
+  void*         pData;
+  bool          used;     // set current page is in used
+} SPageInfo;
+
+typedef struct SFreeListItem {
+  int32_t offset;
+  int32_t len;
+} SFreeListItem;
+
+typedef struct SResultBufStatis {
+  int32_t flushBytes;
+  int32_t loadBytes;
+  int32_t getPages;
+  int32_t releasePages;
+  int32_t flushPages;
+} SResultBufStatis;
+
 typedef struct SDiskbasedResultBuf {
   int32_t   numOfRowsPerPage;
   int32_t   numOfPages;
   int64_t   totalBufSize;
-  int32_t   fd;                  // data file fd
+  int64_t   fileSize;            // disk file size
+  FILE*     file;
   int32_t   allocateId;          // allocated page id
-  int32_t   incStep;             // minimum allocated pages
-  void*     pBuf;                // mmap buffer pointer
   char*     path;                // file path
   int32_t   pageSize;            // current used page size
   int32_t   inMemPages;          // numOfPages that are allocated in memory
-  SHashObj* idsTable;            // id hash table
-  SIDList   list;                // for each id, there is a page id list
-
-  void*     iBuf;                // inmemory buf
-  void*     handle;              // for debug purpose
+  SHashObj* groupSet;            // id hash table
+  SHashObj* all;
+  SList*    lruList;
   void*     emptyDummyIdList;    // dummy id list
+  void*     assistBuf;           // assistant buffer for compress/decompress data
+  SArray*   pFree;               // free area in file
+  bool      comp;                // compressed before flushed to disk
+  int32_t   nextPos;             // next page flush position
+
+  const void*      handle;        // for debug purpose
+  SResultBufStatis statis;
 } SDiskbasedResultBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (4096L)
 #define DEFAULT_INMEM_BUF_PAGES       10
+#define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
 
 /**
  * create disk-based result buffer
  * @param pResultBuf
- * @param size
  * @param rowSize
+ * @param pagesize
+ * @param inMemPages
+ * @param handle
  * @return
  */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
-    int32_t inMemPages, void* handle);
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle);
 
 /**
  *
@@ -72,7 +107,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
@@ -88,42 +123,52 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
  * @param id
  * @return
  */
-static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
-  if (id < pResultBuf->inMemPages) {
-    return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize);
-  } else {
-    return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize);
-  }
-}
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
+
+/**
+ * release the referenced buf pages
+ * @param pResultBuf
+ * @param page
+ */
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
+
+/**
+ *
+ * @param pResultBuf
+ * @param pi
+ */
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
+
+
 /**
  * get the total buffer size in the format of disk file
  * @param pResultBuf
  * @return
  */
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf);
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * get the number of groups in the result buffer
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * destroy result buffer
  * @param pResultBuf
  */
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle);
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
  * @param pList
  * @return
  */
-int32_t getLastPageId(SIDList pList);
+SPageInfo* getLastPageInfo(SIDList pList);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_VNODEQUERYUTIL_H
+#endif  // TDENGINE_QRESULTBUF_H
diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qSqlparser.h
similarity index 99%
rename from src/query/inc/qsqlparser.h
rename to src/query/inc/qSqlparser.h
index 9e8706cf9220c2a0d0e55fb0a8f1028a876da9d4..cd9618b223752ca1c3eff3bbdac1856135c9c394 100644
--- a/src/query/inc/qsqlparser.h
+++ b/src/query/inc/qSqlparser.h
@@ -119,6 +119,7 @@ typedef struct SCreateDBInfo {
   int32_t   fsyncPeriod;
   int64_t   commitTime;
   int32_t   walLevel;
+  int32_t   quorum;
   int32_t   compressionLevel;
   SSQLToken precision;
   bool      ignoreExists;
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 7119cb75fe9f6604e42a96406ff631840477d5f7..ed7c7e8845317ac754758b8553e3f65ba8da4544 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -45,13 +45,14 @@ bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
 int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
 
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) {
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+    tFilePage* page) {
   assert(pResult != NULL && pRuntimeEnv != NULL);
 
   SQuery    *pQuery = pRuntimeEnv->pQuery;
-  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
-  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
+//  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
 
+  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
   return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
       pQuery->pSelectExpr[columnIndex].bytes * realRowId;
 }
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index d709b4f838280ade6f43e419a5927216de417be5..15cfeee6b24d9add13b71e22d21e53a800b65dc6 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -215,6 +215,7 @@ keep(Y)    ::= KEEP tagitemlist(X).           { Y = X; }
 tables(Y)  ::= MAXTABLES INTEGER(X).          { Y = X; }
 cache(Y)   ::= CACHE INTEGER(X).              { Y = X; }
 replica(Y) ::= REPLICA INTEGER(X).            { Y = X; }
+quorum(Y)  ::= QUORUM INTEGER(X).             { Y = X; }
 days(Y)    ::= DAYS INTEGER(X).               { Y = X; }
 minrows(Y) ::= MINROWS INTEGER(X).            { Y = X; }
 maxrows(Y) ::= MAXROWS INTEGER(X).            { Y = X; }
@@ -231,6 +232,7 @@ db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
 db_optr(Y) ::= db_optr(Z) tables(X).         { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) cache(X).          { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) replica(X).        { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) quorum(X).         { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) days(X).           { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) minrows(X).        { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); }
 db_optr(Y) ::= db_optr(Z) maxrows(X).        { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); }
@@ -246,6 +248,7 @@ db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
 alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
 
 alter_db_optr(Y) ::= alter_db_optr(Z) replica(X).     { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X).      { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) tables(X).      { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) keep(X).        { Y = Z; Y.keep = X; }
 alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X).      { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index 5ce9121cf1c4f5e88f3b68823431ad61923e70ec..58d2ff260ec79077fc83e1fa4a444041d8e338dc 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -22,7 +22,7 @@ extern "C" {
 
 #include "os.h"
 
-#include "../../common/inc/tname.h"
+#include "tname.h"
 #include "taosdef.h"
 #include "trpc.h"
 #include "tvariant.h"
diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c
index c2578c15c0536ea222f376e833a8fe11fa269229..44051c7e3b26f8dad68ae7af17eaf9748dbf26ce 100644
--- a/src/query/src/qAst.c
+++ b/src/query/src/qAst.c
@@ -18,8 +18,8 @@
 
 #include "exception.h"
 #include "qAst.h"
+#include "qSqlparser.h"
 #include "qSyntaxtreefunction.h"
-#include "qsqlparser.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tarray.h"
@@ -878,28 +878,6 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
   free(pRightOutput);
 }
 
-//void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) {
-//  if (pExprs == NULL) {
-//    return;
-//  }
-//
-//  tExprNode *pLeft = pExprs->_node.pLeft;
-//  tExprNode *pRight = pExprs->_node.pRight;
-//
-//  // recursive traverse left child branch
-//  if (pLeft->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pLeft, res);
-//  } else if (pLeft->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pLeft->pSchema->colId);
-//  }
-//
-//  if (pRight->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pRight, res);
-//  } else if (pRight->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pRight->pSchema->colId);
-//  }
-//}
-
 static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
   tbufWriteUint8(bw, expr->nodeType);
   
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 8afc2fd87a988d32f4311af876bb57e627eecf30..6efc8a827e4337a122f15be211fb06b03e7b1fda 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -221,7 +221,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
 }
 
 static int32_t getGroupResultId(int32_t groupIndex) {
-  int32_t base = 200000;
+  int32_t base = 20000000;
   return base + (groupIndex * 10000);
 }
 
@@ -478,10 +478,14 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (taosArrayGetSize(list) == 0) {
     pData = getNewDataBuf(pResultBuf, sid, &pageId);
   } else {
-    pageId = getLastPageId(list);
-    pData = getResBufPage(pResultBuf, pageId);
+    SPageInfo* pi = getLastPageInfo(list);
+    pData = getResBufPage(pResultBuf, pi->pageId);
+    pageId = pi->pageId;
 
     if (pData->num >= numOfRowsPerPage) {
+      // release current page first, and prepare the next one
+      releaseResBufPageInfo(pResultBuf, pi);
+
       pData = getNewDataBuf(pResultBuf, sid, &pageId);
       if (pData != NULL) {
         assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
@@ -497,6 +501,8 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (pWindowRes->pos.pageId == -1) {  // not allocated yet, allocate new buffer
     pWindowRes->pos.pageId = pageId;
     pWindowRes->pos.rowId = pData->num++;
+
+    assert(pWindowRes->pos.pageId >= 0);
   }
 
   return 0;
@@ -1490,8 +1496,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     goto _clean;
   }
 
-  qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv));
-
   pRuntimeEnv->offset[0] = 0;
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base;
@@ -1536,8 +1540,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
       }
     }
 
-    qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv));
-
     // set the order information for top/bottom query
     int32_t functionId = pCtx->functionId;
 
@@ -1558,25 +1560,19 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     }
   }
 
-  qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv));
-
   char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
 
   // set the intermediate result output buffer
   setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
 
-  qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv));
-
   // if it is group by normal column, do not set output buffer, the output buffer is pResult
   if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) {
     resetCtxOutputBuf(pRuntimeEnv);
   }
 
-  qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv));
-
   setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx);
 
-  qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv));
+  qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv));
   return TSDB_CODE_SUCCESS;
 
 _clean:
@@ -1615,7 +1611,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
   pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
 
-  destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
+  destroyResultBuf(pRuntimeEnv->pResultBuf);
   tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
   tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
 
@@ -2111,9 +2107,6 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
     }
     
     if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) {
-#if defined(_DEBUG_VIEW)
-      qDebug("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv));
-#endif
       // current block has been discard due to filter applied
       pRuntimeEnv->summary.discardBlocks += 1;
       qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
@@ -2446,6 +2439,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (!mergeFlag) {
@@ -2458,7 +2453,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
 
     pCtx[i].hasNull = true;
     pCtx[i].nStartQueryTimestamp = timestamp;
-    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
 
     // in case of tag column, the tag information should be extracted from input buffer
     if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) {
@@ -2615,14 +2610,16 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
 
   SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
   SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos);
+  tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId);
 
-  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1);
+  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
   TSKEY leftTimestamp = GET_INT64_VAL(b1);
 
   SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
   SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos);
+  tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId);
 
-  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2);
+  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
   TSKEY rightTimestamp = GET_INT64_VAL(b2);
 
   if (leftTimestamp == rightTimestamp) {
@@ -2685,35 +2682,26 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   int32_t id = getGroupResultId(pQInfo->groupIndex - 1);
   SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id);
 
-  int32_t total = 0;
   int32_t size = taosArrayGetSize(list);
-  for (int32_t i = 0; i < size; ++i) {
-    int32_t* pgId = taosArrayGet(list, i);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
-    total += pData->num;
-  }
-
-  int32_t rows = total;
 
   int32_t offset = 0;
   for (int32_t j = 0; j < size; ++j) {
-    int32_t* pgId = taosArrayGet(list, j);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
+    SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
+    tFilePage *pData = getResBufPage(pResultBuf, pi->pageId);
 
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
       char *  pDest = pQuery->sdata[i]->data;
-
-      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num,
-             bytes * pData->num);
+      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num);
     }
 
+//    rows += pData->num;
     offset += pData->num;
   }
 
   assert(pQuery->rec.rows == 0);
 
-  pQuery->rec.rows += rows;
+  pQuery->rec.rows += offset;
   pQInfo->offset += 1;
 }
 
@@ -2777,7 +2765,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     assert(pQInfo->numOfGroupResultPages == 0);
     return 0;
   } else if (numOfTables == 1) { // no need to merge results since only one table in each group
-
   }
 
   SCompSupporter cs = {pTableList, posList, pQInfo};
@@ -2802,8 +2789,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
 
     SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
     SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
 
-    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes);
+    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
     TSKEY ts = GET_INT64_VAL(b);
 
     assert(ts == pWindowRes->window.skey);
@@ -3517,9 +3505,11 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page);
 
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
@@ -3542,6 +3532,8 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
 
@@ -3550,7 +3542,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
       continue;
     }
 
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage);
     pCtx->currentStage = 0;
 
     int32_t functionId = pCtx->functionId;
@@ -3713,11 +3705,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
       pQInfo->groupIndex += 1;
     }
 
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId);
+
     for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
       int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
 
       char *out = pQuery->sdata[j]->data + numOfResult * size;
-      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]);
+      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page);
       memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
     }
 
@@ -4238,10 +4232,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   int32_t ps = DEFAULT_PAGE_SIZE;
   int32_t rowsize = 0;
   getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
+  int32_t TWOMB = 1024*1024*2;
 
   if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) {
-    int32_t numOfPages = getInitialPageNum(pQInfo);
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -4269,8 +4263,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     int32_t numOfResultRows = getInitialPageNum(pQInfo);
     getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
-
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -5894,16 +5887,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   }
 
   pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+  pthread_mutex_init(&pQInfo->lock, NULL);
 
   pQuery->pos = -1;
   pQuery->window = pQueryMsg->window;
-
-  if (sem_init(&pQInfo->dataReady, 0, 0) != 0) {
-    int32_t code = TAOS_SYSTEM_ERROR(errno);
-    qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, tstrerror(code));
-    goto _cleanup;
-  }
-
   colIdCheck(pQuery);
 
   qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo);
@@ -5945,7 +5933,7 @@ static bool isValidQInfo(void *param) {
   return (sig == (uint64_t)pQInfo);
 }
 
-static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) {
+static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) {
   int32_t code = TSDB_CODE_SUCCESS;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
@@ -5965,18 +5953,12 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
            pQuery->window.ekey, pQuery->order.order);
     setQueryStatus(pQuery, QUERY_COMPLETED);
     pQInfo->tableqinfoGroupInfo.numOfTables = 0;
-
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
-  pQInfo->param = param;
-
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
     setQueryStatus(pQuery, QUERY_COMPLETED);
-  
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -6018,7 +6000,6 @@ static void freeQInfo(SQInfo *pQInfo) {
     tfree(pQuery->sdata[col]);
   }
 
-  sem_destroy(&(pQInfo->dataReady));
   teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
 
   for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
@@ -6170,7 +6151,7 @@ typedef struct SQueryMgmt {
   pthread_mutex_t lock;
 } SQueryMgmt;
 
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) {
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) {
   assert(pQueryMsg != NULL && tsdb != NULL);
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -6266,7 +6247,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo
     goto _over;
   }
 
-  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param);
+  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery);
 
 _over:
   free(tagCond);
@@ -6306,24 +6287,36 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
   freeQInfo(pQInfo);
 }
 
-void qTableQuery(qinfo_t qinfo) {
+static bool doBuildResCheck(SQInfo* pQInfo) {
+  bool buildRes = false;
+
+  pthread_mutex_lock(&pQInfo->lock);
+
+  pQInfo->dataReady = QUERY_RESULT_READY;
+  buildRes = (pQInfo->rspContext != NULL);
+
+  pthread_mutex_unlock(&pQInfo->lock);
+
+  return buildRes;
+}
+
+bool qTableQuery(qinfo_t qinfo) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || pQInfo->signature != pQInfo) {
     qDebug("QInfo:%p has been freed, no need to execute", pQInfo);
-    return;
+    return false;
   }
 
   if (IS_QUERY_KILLED(pQInfo)) {
     qDebug("QInfo:%p it is already killed, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    return doBuildResCheck(pQInfo);
   }
 
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table exists for query, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED);
+    return doBuildResCheck(pQInfo);
   }
 
   // error occurs, record the error code and return to client
@@ -6331,8 +6324,7 @@ void qTableQuery(qinfo_t qinfo) {
   if (ret != TSDB_CODE_SUCCESS) {
     pQInfo->code = ret;
     qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code));
-    sem_post(&pQInfo->dataReady);
-    return;
+    return doBuildResCheck(pQInfo);
   }
 
   qDebug("QInfo:%p query task is launched", pQInfo);
@@ -6357,10 +6349,10 @@ void qTableQuery(qinfo_t qinfo) {
            pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
   }
 
-  sem_post(&pQInfo->dataReady);
+  return doBuildResCheck(pQInfo);
 }
 
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6373,11 +6365,21 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
     return pQInfo->code;
   }
 
-  sem_wait(&pQInfo->dataReady);
-  qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
-         pQInfo->code);
+  int32_t code = TSDB_CODE_SUCCESS;
+  pthread_mutex_lock(&pQInfo->lock);
+  if (pQInfo->dataReady == QUERY_RESULT_READY) {
+    *buildRes = true;
+    qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
+           pQInfo->code);
+  } else {
+    *buildRes = false;
+    qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo);
+    pQInfo->rspContext = pRspContext;
+  }
 
-  return pQInfo->code;
+  code = pQInfo->code;
+  pthread_mutex_unlock(&pQInfo->lock);
+  return code;
 }
 
 bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
@@ -6389,6 +6391,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   }
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
   bool ret = false;
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     ret = false;
@@ -6407,7 +6410,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   return ret;
 }
 
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) {
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6417,8 +6420,10 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
+
   size += sizeof(int32_t);
   size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo);
+
   *contLen = size + sizeof(SRetrieveTableRsp);
 
   // todo proper handle failed to allocate memory,
@@ -6427,6 +6432,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   if (*pRsp == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
+
   (*pRsp)->numOfRows = htonl(pQuery->rec.rows);
 
   int32_t code = pQInfo->code;
@@ -6434,8 +6440,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->offset = htobe64(pQuery->limit.offset);
     (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime);
   } else {
-    (*pRsp)->offset = 0;
     (*pRsp)->useconds = 0;
+    (*pRsp)->offset = 0;
   }
   
   (*pRsp)->precision = htons(pQuery->precision);
@@ -6446,10 +6452,20 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     code = pQInfo->code;
   }
 
+  pQInfo->rspContext = NULL;
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     (*pRsp)->completed = 1;  // notify no more result to client
   }
 
+  if (qHasMoreResultsToRetrieve(pQInfo)) {
+    *continueExec = true;
+  } else { // failed to dump result, free qhandle immediately
+    *continueExec = false;
+    qKillQuery(pQInfo);
+  }
+
   return code;
 }
 
@@ -6460,7 +6476,6 @@ int32_t qKillQuery(qinfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  sem_post(&pQInfo->dataReady);
   setQueryKilled(pQInfo);
   return TSDB_CODE_SUCCESS;
 }
@@ -6607,6 +6622,13 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
   setQueryStatus(pQuery, QUERY_COMPLETED);
 }
 
+void* qGetResultRetrieveMsg(qinfo_t qinfo) {
+  SQInfo* pQInfo = (SQInfo*) qinfo;
+  assert(pQInfo != NULL);
+
+  return pQInfo->rspContext;
+}
+
 void freeqinfoFn(void *qhandle) {
   void** handle = qhandle;
   if (handle == NULL || *handle == NULL) {
@@ -6618,19 +6640,21 @@ void freeqinfoFn(void *qhandle) {
 }
 
 void* qOpenQueryMgmt(int32_t vgId) {
-  const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, refresh handle pool
+  const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool
 
   char cacheName[128] = {0};
   sprintf(cacheName, "qhandle_%d", vgId);
 
-  SQueryMgmt* pQueryHandle = calloc(1, sizeof(SQueryMgmt));
+  SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt));
+
+  pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
+  pQueryMgmt->closed    = false;
+  pQueryMgmt->vgId      = vgId;
 
-  pQueryHandle->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
-  pQueryHandle->closed    = false;
-  pthread_mutex_init(&pQueryHandle->lock, NULL);
+  pthread_mutex_init(&pQueryMgmt->lock, NULL);
 
   qDebug("vgId:%d, open querymgmt success", vgId);
-  return pQueryHandle;
+  return pQueryMgmt;
 }
 
 static void queryMgmtKillQueryFn(void* handle) {
@@ -6670,7 +6694,7 @@ void qCleanupQueryMgmt(void* pQMgmt) {
   pthread_mutex_destroy(&pQueryMgmt->lock);
   tfree(pQueryMgmt);
 
-  qDebug("vgId:%d querymgmt cleanup completed", vgId);
+  qDebug("vgId:%d queryMgmt cleanup completed", vgId);
 }
 
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
@@ -6716,14 +6740,15 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) {
   }
 }
 
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) {
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) {
   SQueryMgmt *pQueryMgmt = pMgmt;
 
   if (pQueryMgmt->qinfoPool == NULL) {
     return NULL;
   }
 
-  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, needFree);
+  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
   return 0;
 }
 
+
diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c
index 69c5f0e24fe6361d41953c35fce1380b97d4e752..fb57f71199d8688fa8503790dab5551e7105ca3a 100644
--- a/src/query/src/qExtbuffer.c
+++ b/src/query/src/qExtbuffer.c
@@ -28,10 +28,10 @@
 /*
  * SColumnModel is deeply copy
  */
-tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel) {
+tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) {
   tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer));
 
-  pMemBuffer->pageSize = DEFAULT_PAGE_SIZE;
+  pMemBuffer->pageSize = pagesize;
   pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize;
   pMemBuffer->nElemSize = elemSize;
 
diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index ecc11f8f4d76769d0d52b03f44813349aed29adc..7cf995193bc4d3afd96b22f7802cd07e83ee9139 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -14,7 +14,7 @@
  */
 
 #include "os.h"
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "queryLog.h"
 #include "taosdef.h"
 #include "taosmsg.h"
@@ -907,6 +907,7 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
   pDBInfo->daysPerFile = -1;
 
   pDBInfo->replica = -1;
+  pDBInfo->quorum = -1;
   pDBInfo->keep = NULL;
 
   memset(&pDBInfo->precision, 0, sizeof(SSQLToken));
diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index c4490a01e79408ec2bf09527049082f9c05566c8..85e45e46b3eee0beef2babc893234aa0a5a0818f 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -535,7 +535,7 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) {
 
     if (pSeg->pBuffer[slotIdx] == NULL) {
       pSeg->pBuffer[slotIdx] = createExtMemBuffer(pBucket->numOfTotalPages * pBucket->pageSize, pBucket->nElemSize,
-                                                  pBucket->pOrderDesc->pColumnModel);
+                                                  pBucket->pageSize, pBucket->pOrderDesc->pColumnModel);
       pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL;
       pBucket->pOrderDesc->pColumnModel->capacity = pSeg->pBuffer[slotIdx]->numOfElemsPerPage;
     }
diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c
index a3d3386a878cf69c9dae7ea089f36c28df57ed4f..93ccad8e27b8ec3fc16af70f3015f429c70a8af2 100644
--- a/src/query/src/qResultbuf.c
+++ b/src/query/src/qResultbuf.c
@@ -1,221 +1,439 @@
 #include "qResultbuf.h"
+#include "stddef.h"
+#include "tscompression.h"
 #include "hash.h"
 #include "qExtbuffer.h"
 #include "queryLog.h"
 #include "taoserror.h"
 
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize,
-    int32_t pagesize, int32_t inMemPages, void* handle) {
+#define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES)
 
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle) {
   *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+
   SDiskbasedResultBuf* pResBuf = *pResultBuf;
   if (pResBuf == NULL) {
     return TSDB_CODE_COM_OUT_OF_MEMORY;  
   }
 
-  pResBuf->pageSize   = pagesize;
-  pResBuf->numOfPages = inMemPages;  // all pages are in buffer in the first place
-  pResBuf->inMemPages = inMemPages;
-  assert(inMemPages <= numOfPages);
-
-  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
+  pResBuf->pageSize     = pagesize;
+  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
+  pResBuf->totalBufSize = 0;
+  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
+  pResBuf->allocateId   = -1;
+  pResBuf->comp         = true;
+  pResBuf->file         = NULL;
+  pResBuf->handle       = handle;
+  pResBuf->fileSize = 0;
 
-  pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
-  pResBuf->incStep = 4;
-  pResBuf->allocateId = -1;
+  // at least more than 2 pages must be in memory
+  assert(inMemBufSize >= pagesize * 2);
 
-  pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize);
+  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
+  pResBuf->lruList = tdListNew(POINTER_BYTES);
 
   // init id hash table
-  pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
-  pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES);
+  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
+  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
 
   char path[PATH_MAX] = {0};
-  getTmpfilePath("tsdb_qbuf", path);
+  getTmpfilePath("qbuf", path);
   pResBuf->path = strdup(path);
 
-  pResBuf->fd = FD_INITIALIZER;
-  pResBuf->pBuf = NULL;
   pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
 
-  qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
-      pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize);
-  
+  qDebug("QInfo:%p create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", handle, pResBuf->pageSize,
+         pResBuf->inMemPages, pResBuf->path);
+
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); }
-
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
-
-#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages)
-#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
-
-static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
-  pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR | O_TRUNC, 0666);
-  if (!FD_VALID(pResultBuf->fd)) {
+static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
+  pResultBuf->file = fopen(pResultBuf->path, "wb+");
+  if (pResultBuf->file == NULL) {
     qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  assert(pResultBuf->numOfPages == pResultBuf->inMemPages);
-  pResultBuf->numOfPages += pResultBuf->incStep;
+  return TSDB_CODE_SUCCESS;
+}
 
-  int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-  if (ret != TSDB_CODE_SUCCESS) {
-    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
+static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
   }
 
-  pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
-  if (pResultBuf->pBuf == MAP_FAILED) {
-    qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
+  *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
+}
+
+static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
   }
 
-  pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
-  return TSDB_CODE_SUCCESS;
+  *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
 }
 
-static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) {
-  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize);
-  int32_t ret = TSDB_CODE_SUCCESS;
+static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
+  if (pResultBuf->pFree == NULL) {
+    return pResultBuf->nextPos;
+  } else {
+    int32_t offset = -1;
+
+    size_t num = taosArrayGetSize(pResultBuf->pFree);
+    for(int32_t i = 0; i < num; ++i) {
+      SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
+      if (pi->len >= size) {
+        offset = pi->offset;
+        pi->offset += size;
+        pi->len -= size;
+
+        return offset;
+      }
+    }
+
+    // no available recycle space, allocate new area in file
+    return pResultBuf->nextPos;
+  }
+}
+
+static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  assert(!pg->used && pg->pData != NULL);
+
+  int32_t size = -1;
+  char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
+
+  // this page is flushed to disk for the first time
+  if (pg->info.offset == -1) {
+    pg->info.offset = allocatePositionInFile(pResultBuf, size);
+    pResultBuf->nextPos += size;
 
-  if (pResultBuf->pBuf == NULL) {
-    assert(pResultBuf->fd == FD_INITIALIZER);
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file);
 
-    if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
-      return ret;
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   } else {
-    ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->numOfPages += incNumOfPages;
-
-    /*
-     * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
-     * be insufficient
-     */
-    ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-    if (ret != TSDB_CODE_SUCCESS) {
-      //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
-      //           strerror(errno));
-      return TSDB_CODE_QRY_NO_DISKSPACE;
+    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
+    if (pg->info.length < size) {
+      // 1. add current space to free list
+      taosArrayPush(pResultBuf->pFree, &pg->info);
+
+      // 2. allocate new position, and update the info
+      pg->info.offset = allocatePositionInFile(pResultBuf, size);
+      pResultBuf->nextPos += size;
     }
 
-    pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
-    pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
+    //3. write to disk.
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    fwrite(t, size, 1, pResultBuf->file);
 
-    if (pResultBuf->pBuf == MAP_FAILED) {
-      //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
-      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  char* ret = pg->pData;
+  memset(ret, 0, pResultBuf->pageSize);
+
+  pg->pData = NULL;
+  pg->info.length = size;
+
+  pResultBuf->statis.flushBytes += pg->info.length;
+
+  return ret;
 }
 
-#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1)
+static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
 
-static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  assert(pResultBuf != NULL);
+  if (pResultBuf->file == NULL) {
+    if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
+      terrno = ret;
+      return NULL;
+    }
+  }
+
+  return doFlushPageToDisk(pResultBuf, pg);
+}
+
+// load file block data in disk
+static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+  ret = fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
+  if (ret != pg->info.length) {
+    terrno = errno;
+    return NULL;
+  }
+
+  pResultBuf->statis.loadBytes += pg->info.length;
 
-  char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
+  int32_t fullSize = 0;
+  doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
+
+  return GET_DATA_PAYLOAD(pg);
+}
+
+#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages)
+
+static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
+
+  SArray* pa = taosArrayInit(1, POINTER_BYTES);
+  int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
+  assert(ret == 0);
+
+  return pa;
+}
+
+static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
+  SIDList list = NULL;
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
   if (p == NULL) {  // it is a new group id
-    return -1;
+    list = addNewGroup(pResultBuf, groupId);
+  } else {
+    list = (SIDList) (*p);
   }
 
-  int32_t slot = GET_INT32_VAL(p);
-  assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable));
+  pResultBuf->numOfPages += 1;
 
-  return slot;
+  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
+  ppi->info     = PAGE_INFO_INITIALIZER;
+  ppi->pageId   = pageId;
+  ppi->pData    = NULL;
+  ppi->pn       = NULL;
+  ppi->used     = true;
+
+  return *(SPageInfo**) taosArrayPush(list, &ppi);
 }
 
-static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
-  taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t));
+static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
+  SListIter iter = {0};
+  tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
+
+  SListNode* pn = NULL;
+  while((pn = tdListNext(&iter)) != NULL) {
+    assert(pn != NULL);
+
+    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
+    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
 
-  SArray* pa = taosArrayInit(1, sizeof(int32_t));
-  taosArrayPush(pResultBuf->list, &pa);
+    if (!pageInfo->used) {
+      break;
+    }
+  }
 
-  assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable));
-  return num;
+  return pn;
 }
 
-static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
-    slot = addNewGroupId(pResultBuf, groupId);
+static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
+  char* bufPage = NULL;
+  SListNode* pn = getEldestUnrefedPage(pResultBuf);
+
+  // all pages are referenced by user, try to allocate new space
+  if (pn == NULL) {
+    int32_t prev = pResultBuf->inMemPages;
+    pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5;
+
+    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
+          pResultBuf->inMemPages, pResultBuf->pageSize);
+  } else {
+    pResultBuf->statis.flushPages += 1;
+    tdListPopNode(pResultBuf->lruList, pn);
+
+    SPageInfo* d = *(SPageInfo**) pn->data;
+    assert(d->pn == pn);
+
+    d->pn = NULL;
+    tfree(pn);
+
+    bufPage = flushPageToDisk(pResultBuf, d);
   }
 
-  SIDList pList = taosArrayGetP(pResultBuf->list, slot);
-  taosArrayPush(pList, &pageId);
+  return bufPage;
+}
+
+static void lruListPushFront(SList *pList, SPageInfo* pi) {
+  tdListPrepend(pList, &pi);
+  SListNode* front = tdListGetHead(pList);
+  pi->pn = front;
+}
+
+static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
+  tdListPopNode(pList, pi->pn);
+  tdListPrependNode(pList, pi->pn);
 }
 
 tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
+  pResultBuf->statis.getPages += 1;
+
+  char* availablePage = NULL;
   if (NO_AVAILABLE_PAGES(pResultBuf)) {
-    if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
-      return NULL;
-    }
+    availablePage = evicOneDataPage(pResultBuf);
   }
 
   // register new id in this group
   *pageId = (++pResultBuf->allocateId);
-  registerPageId(pResultBuf, groupId, *pageId);
 
-  // clear memory for the new page
-  tFilePage* page = getResBufPage(pResultBuf, *pageId);
-  memset(page, 0, pResultBuf->pageSize);
-  
-  return page;
+  // register page id info
+  SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
+
+  // add to LRU list
+  assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
+
+  lruListPushFront(pResultBuf->lruList, pi);
+
+  // add to hash map
+  taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
+
+  // allocate buf
+  if (availablePage == NULL) {
+    pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+  } else {
+    pi->pData = availablePage;
+  }
+
+  pResultBuf->totalBufSize += pResultBuf->pageSize;
+
+  ((void**)pi->pData)[0] = pi;
+  pi->used = true;
+
+  return GET_DATA_PAYLOAD(pi);
+}
+
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
+  assert(pResultBuf != NULL && id >= 0);
+  pResultBuf->statis.getPages += 1;
+
+  SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
+  assert(pi != NULL && *pi != NULL);
+
+  if ((*pi)->pData != NULL) { // it is in memory
+    // no need to update the LRU list if only one page exists
+    if (pResultBuf->numOfPages == 1) {
+      (*pi)->used = true;
+      return GET_DATA_PAYLOAD(*pi);
+    }
+
+    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
+    assert(*pInfo == *pi);
+
+    lruListMoveToFront(pResultBuf->lruList, (*pi));
+    (*pi)->used = true;
+
+    return GET_DATA_PAYLOAD(*pi);
+
+  } else { // not in memory
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
+
+    char* availablePage = NULL;
+    if (NO_AVAILABLE_PAGES(pResultBuf)) {
+      availablePage = evicOneDataPage(pResultBuf);
+    }
+
+    if (availablePage == NULL) {
+      (*pi)->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+    } else {
+      (*pi)->pData = availablePage;
+    }
+
+    ((void**)((*pi)->pData))[0] = (*pi);
+
+    lruListPushFront(pResultBuf->lruList, *pi);
+    loadPageFromDisk(pResultBuf, *pi);
+    return GET_DATA_PAYLOAD(*pi);
+  }
 }
 
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
+  assert(pResultBuf != NULL && page != NULL);
+  char* p = (char*) page - POINTER_BYTES;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  releaseResBufPageInfo(pResultBuf, ppi);
+}
+
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
+  assert(pi->pData != NULL && pi->used);
+
+  pi->used = false;
+  pResultBuf->statis.releasePages += 1;
+}
+
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
+
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
 
 SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
+  assert(pResultBuf != NULL);
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
     return pResultBuf->emptyDummyIdList;
   } else {
-    return taosArrayGetP(pResultBuf->list, slot);
+    return (SArray*) (*p);
   }
 }
 
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   if (pResultBuf == NULL) {
     return;
   }
 
-  if (FD_VALID(pResultBuf->fd)) {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
-        pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
+  if (pResultBuf->file != NULL) {
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes",
+        pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize);
 
-    close(pResultBuf->fd);
-    munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->pBuf = NULL;
+    fclose(pResultBuf->file);
   } else {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
            pResultBuf->totalBufSize);
   }
 
   unlink(pResultBuf->path);
   tfree(pResultBuf->path);
 
-  size_t size = taosArrayGetSize(pResultBuf->list);
-  for (int32_t i = 0; i < size; ++i) {
-    SArray* pa = taosArrayGetP(pResultBuf->list, i);
-    taosArrayDestroy(pa);
+  SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
+  while(taosHashIterNext(iter)) {
+    SArray** p = (SArray**) taosHashIterGet(iter);
+    size_t n = taosArrayGetSize(*p);
+    for(int32_t i = 0; i < n; ++i) {
+      SPageInfo* pi = taosArrayGetP(*p, i);
+      tfree(pi->pData);
+      tfree(pi);
+    }
+
+    taosArrayDestroy(*p);
   }
 
-  taosArrayDestroy(pResultBuf->list);
+  taosHashDestroyIter(iter);
+
+  tdListFree(pResultBuf->lruList);
   taosArrayDestroy(pResultBuf->emptyDummyIdList);
-  taosHashCleanup(pResultBuf->idsTable);
+  taosHashCleanup(pResultBuf->groupSet);
+  taosHashCleanup(pResultBuf->all);
 
-  tfree(pResultBuf->iBuf);
+  tfree(pResultBuf->assistBuf);
   tfree(pResultBuf);
 }
 
-int32_t getLastPageId(SIDList pList) {
+SPageInfo* getLastPageInfo(SIDList pList) {
   size_t size = taosArrayGetSize(pList);
-  return *(int32_t*) taosArrayGet(pList, size - 1);
+  return (SPageInfo*) taosArrayGetP(pList, size - 1);
 }
 
diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c
index 0ea0ff7bf3c30f4fe8d12b1ed426f118b2d684c7..b54dc32fb39d5e0f194a7d2e333a8e682a303c19 100644
--- a/src/query/src/qTokenizer.c
+++ b/src/query/src/qTokenizer.c
@@ -116,6 +116,7 @@ static SKeyword keywordTable[] = {
     {"STATE",        TK_STATE},
     {"KEEP",         TK_KEEP},
     {"REPLICA",      TK_REPLICA},
+    {"QUORUM",       TK_QUORUM},
     {"DAYS",         TK_DAYS},
     {"MINROWS",      TK_MINROWS},
     {"MAXROWS",      TK_MAXROWS},
diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c
index 20b29107f5ebaffb6807223f6e9763496ca668aa..90cf394035aa6a2752c9ca6054adc3482b2af260 100644
--- a/src/query/src/qTsbuf.c
+++ b/src/query/src/qTsbuf.c
@@ -740,7 +740,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
   int32_t size = pSrcBuf->fileSize - offset;
 
 #ifdef LINUX
-  ssize_t rc = tsendfile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
+  ssize_t rc = taosTSendFile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
 #else
   ssize_t rc = fsendfile(pDestBuf->f, pSrcBuf->f, &offset, size);
 #endif
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 7b03dd9d941668119b93d3fb19dd1be3af9f3289..c9143b3a5323fa2419e3a543cc3d4d94d404cffd 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -236,11 +236,13 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
   if (pWindowRes == NULL) {
     return;
   }
-  
+
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
     SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
     
-    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
     size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     memset(s, 0, size);
     
@@ -277,8 +279,11 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
     memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
     
     // copy the output buffer data from src to dst, the position info keep unchanged
-    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst);
-    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src);
+    tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId);
+    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);
+
+    tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId);
+    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
     size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     
     memcpy(dstBuf, srcBuf, s);
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index ac9952bb97d9c333a5e4209318cf1b26d3fe4b9c..fe26f61725778d373d8a25f3895f7854714bb020 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -30,7 +30,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stdbool.h>
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "tcmdtype.h"
 #include "tstoken.h"
 #include "ttokendef.h"
@@ -97,26 +97,26 @@
 #endif
 /************* Begin control #defines *****************************************/
 #define YYCODETYPE unsigned short int
-#define YYNOCODE 272
+#define YYNOCODE 274
 #define YYACTIONTYPE unsigned short int
 #define ParseTOKENTYPE SSQLToken
 typedef union {
   int yyinit;
   ParseTOKENTYPE yy0;
-  SSubclauseInfo* yy25;
-  tSQLExpr* yy66;
-  SCreateAcctSQL yy73;
-  int yy82;
-  SQuerySQL* yy150;
-  SCreateDBInfo yy158;
-  TAOS_FIELD yy181;
-  SLimitVal yy188;
-  tSQLExprList* yy224;
-  int64_t yy271;
-  tVariant yy312;
-  SCreateTableSQL* yy374;
-  tFieldList* yy449;
-  tVariantList* yy494;
+  int yy46;
+  tSQLExpr* yy64;
+  tVariant yy134;
+  SCreateAcctSQL yy149;
+  int64_t yy207;
+  SLimitVal yy216;
+  TAOS_FIELD yy223;
+  SSubclauseInfo* yy231;
+  SCreateDBInfo yy268;
+  tSQLExprList* yy290;
+  SQuerySQL* yy414;
+  SCreateTableSQL* yy470;
+  tVariantList* yy498;
+  tFieldList* yy523;
 } YYMINORTYPE;
 #ifndef YYSTACKDEPTH
 #define YYSTACKDEPTH 100
@@ -126,17 +126,17 @@ typedef union {
 #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo
 #define ParseARG_STORE yypParser->pInfo = pInfo
 #define YYFALLBACK 1
-#define YYNSTATE             242
-#define YYNRULE              223
-#define YYNTOKEN             206
-#define YY_MAX_SHIFT         241
-#define YY_MIN_SHIFTREDUCE   401
-#define YY_MAX_SHIFTREDUCE   623
-#define YY_ERROR_ACTION      624
-#define YY_ACCEPT_ACTION     625
-#define YY_NO_ACTION         626
-#define YY_MIN_REDUCE        627
-#define YY_MAX_REDUCE        849
+#define YYNSTATE             243
+#define YYNRULE              226
+#define YYNTOKEN             207
+#define YY_MAX_SHIFT         242
+#define YY_MIN_SHIFTREDUCE   405
+#define YY_MAX_SHIFTREDUCE   630
+#define YY_ERROR_ACTION      631
+#define YY_ACCEPT_ACTION     632
+#define YY_NO_ACTION         633
+#define YY_MIN_REDUCE        634
+#define YY_MAX_REDUCE        859
 /************* End control #defines *******************************************/
 
 /* Define the yytestcase() macro to be a no-op if is not already defined
@@ -202,217 +202,218 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (547)
+#define YY_ACTTAB_COUNT (552)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */   727,  442,  726,   11,  725,  625,  241,  508,  728,  443,
- /*    10 */   730,  731,  729,   41,   43,  524,   35,   36,  521,  134,
- /*    20 */   522,   29,  523,  442,  197,   39,   37,   40,   38,  153,
- /*    30 */   239,  443,  219,   34,   33,  217,  216,   32,   31,   30,
- /*    40 */    41,   43,  757,   35,   36,  139,  170,  171,   29,  135,
- /*    50 */    21,  197,   39,   37,   40,   38,  182,  833,  158,  837,
- /*    60 */    34,   33,  744,  768,   32,   31,   30,  402,  403,  404,
- /*    70 */   405,  406,  407,  408,  409,  410,  411,  412,  413,  240,
- /*    80 */    41,   43,  846,   35,   36,  742,   60,  135,   29,  135,
- /*    90 */    21,  197,   39,   37,   40,   38,  157,  837,   27,  836,
- /*   100 */    34,   33,   56,  228,   32,   31,   30,  103,   43,    8,
- /*   110 */    35,   36,   61,  113,  765,   29,  757,  525,  197,   39,
- /*   120 */    37,   40,   38,  166,  537,  743,  579,   34,   33,   18,
- /*   130 */   154,   32,   31,   30,   16,  234,  208,  233,  207,  206,
- /*   140 */   205,  232,  204,  231,  230,  229,  203,  723,  168,  711,
- /*   150 */   712,  713,  714,  715,  716,  717,  718,  719,  720,  721,
- /*   160 */   722,   35,   36,  792,  103,  192,   29,  175,  155,  197,
- /*   170 */    39,   37,   40,   38,  179,  178,   21,  581,   34,   33,
- /*   180 */   442,   12,   32,   31,   30,  162,  592,  746,  443,  583,
- /*   190 */    17,  586,   76,  589,  103,  162,  592,   26,  228,  583,
- /*   200 */   148,  586,   99,  589,  103,   21,   88,   87,  142,  167,
- /*   210 */   165,  743,  169,  582,  147,  214,  213,  159,  160,   50,
- /*   220 */   791,  196,   74,   78,   83,   86,   77,  159,  160,  746,
- /*   230 */   235,  540,   80,  162,  592,   17,   51,  583,  215,  586,
- /*   240 */   743,  589,   26,   39,   37,   40,   38,  832,  194,  746,
- /*   250 */    58,   34,   33,   47,  185,   32,   31,   30,  666,  831,
- /*   260 */    59,  126,  181,  560,  561,  159,  160,   16,  234,  150,
- /*   270 */   233,   21,   48,  585,  232,  588,  231,  230,  229,   34,
- /*   280 */    33,  745,   42,   32,   31,   30,  116,  117,   68,   64,
- /*   290 */    67,  151,   42,  591,   32,   31,   30,  130,  128,   91,
- /*   300 */    90,   89,  675,  591,  220,  126,  743,   98,  590,  238,
- /*   310 */   237,   95,  667,  152,   26,  126,  532,  584,  590,  587,
- /*   320 */   551,  161,  552,  184,   46,  609,   14,  593,  514,   13,
- /*   330 */    42,   13,   46,  513,  201,   73,   72,   22,   22,   10,
- /*   340 */     9,  591,  528,  526,  529,  527,   85,   84,  802,  140,
- /*   350 */   141,  143,  801,  144,  163,  145,  590,  146,  798,  137,
- /*   360 */     3,  133,  138,  136,  797,  164,  759,  737,  218,  767,
- /*   370 */   100,  784,  783,  114,  112,  115,  677,  202,  131,   24,
- /*   380 */   211,  674,  212,  845,   70,  844,   26,  842,  118,   93,
- /*   390 */   695,   25,  547,   23,  186,  132,  664,   79,  662,   81,
- /*   400 */    82,  660,  659,  172,  190,  127,  657,  656,  655,  654,
- /*   410 */   653,  645,  129,  651,  649,  647,   52,   49,  756,  771,
- /*   420 */   104,  772,   44,  785,  195,  193,  191,  189,  187,   28,
- /*   430 */   210,   75,  221,  222,  223,  224,  225,  226,  199,  227,
- /*   440 */   236,   53,  623,  174,  173,  622,   62,  177,  149,  621,
- /*   450 */   183,   65,  176,  614,  658,  180,   92,   94,  652,  121,
- /*   460 */   125,  534,  120,  696,  741,  119,  110,  107,  105,  106,
- /*   470 */   122,  108,  123,    2,  111,  109,  124,    1,  184,   57,
- /*   480 */    55,  548,  156,  101,  188,    5,  553,  102,    6,   63,
- /*   490 */   594,   19,    4,   20,   15,  198,    7,  200,  483,  479,
- /*   500 */   477,  476,  475,  473,  446,  209,   66,   45,   69,   71,
- /*   510 */    22,  510,  509,  507,   54,  467,  465,  457,  463,  459,
- /*   520 */   461,  455,  453,  482,  481,  480,  478,  474,  472,   46,
- /*   530 */   444,  417,  415,  627,  626,  626,  626,  626,  626,  626,
- /*   540 */   626,  626,  626,  626,  626,   96,   97,
+ /*     0 */   103,  446,  135,  673,  632,  242,  126,  515,  135,  447,
+ /*    10 */   135,  158,  847,   41,   43,   11,   35,   36,  846,  157,
+ /*    20 */   847,   29,  134,  446,  197,   39,   37,   40,   38,  155,
+ /*    30 */   103,  447,  139,   34,   33,  217,  216,   32,   31,   30,
+ /*    40 */    41,   43,  767,   35,   36,   32,   31,   30,   29,  756,
+ /*    50 */   446,  197,   39,   37,   40,   38,  182,  802,  447,  192,
+ /*    60 */    34,   33,   21,   21,   32,   31,   30,  406,  407,  408,
+ /*    70 */   409,  410,  411,  412,  413,  414,  415,  416,  417,  241,
+ /*    80 */    41,   43,  228,   35,   36,  194,  843,   58,   29,   21,
+ /*    90 */   842,  197,   39,   37,   40,   38,  166,  167,  753,  753,
+ /*   100 */    34,   33,  168,   56,   32,   31,   30,  778,  841,   16,
+ /*   110 */   235,  208,  234,  233,  207,  206,  205,  232,  204,  231,
+ /*   120 */   230,  229,  203,  215,  151,  753,  732,  586,  719,  720,
+ /*   130 */   721,  722,  723,  724,  725,  726,  727,  728,  729,  730,
+ /*   140 */   731,   43,    8,   35,   36,   61,  113,   21,   29,  153,
+ /*   150 */   240,  197,   39,   37,   40,   38,  239,  238,   95,  775,
+ /*   160 */    34,   33,  165,   99,   32,   31,   30,  169,   35,   36,
+ /*   170 */   214,  213,  592,   29,  595,  103,  197,   39,   37,   40,
+ /*   180 */    38,  220,  756,  753,  236,   34,   33,  175,   12,   32,
+ /*   190 */    31,   30,  162,  599,  179,  178,  590,  767,  593,  103,
+ /*   200 */   596,  161,  162,  599,  756,   17,  590,  148,  593,  152,
+ /*   210 */   596,  154,   26,   88,   87,  142,  185,  567,  568,   16,
+ /*   220 */   235,  147,  234,  233,  159,  160,  219,  232,  196,  231,
+ /*   230 */   230,  229,  801,   76,  159,  160,  162,  599,  547,  228,
+ /*   240 */   590,    3,  593,   17,  596,   74,   78,   83,   86,   77,
+ /*   250 */    26,   39,   37,   40,   38,   80,   59,  754,   21,   34,
+ /*   260 */    33,  544,   60,   32,   31,   30,   18,  140,  159,  160,
+ /*   270 */   181,  737,  539,  736,   27,  734,  735,  150,  682,  184,
+ /*   280 */   738,  126,  740,  741,  739,  674,  531,  141,  126,  528,
+ /*   290 */    42,  529,  558,  530,  752,  591,   46,  594,   34,   33,
+ /*   300 */    42,  598,   32,   31,   30,  116,  117,   68,   64,   67,
+ /*   310 */   588,  598,  143,   50,   73,   72,  597,  170,  171,  130,
+ /*   320 */   128,   91,   90,   89,   98,   47,  597,  144,  559,  616,
+ /*   330 */    51,   26,   14,   13,   42,  145,  600,  521,  520,  201,
+ /*   340 */    13,   46,   22,   22,   48,  598,  589,   10,    9,  535,
+ /*   350 */   533,  536,  534,   85,   84,  146,  137,  133,  856,  138,
+ /*   360 */   597,  136,  755,  812,  811,  163,  808,  807,  164,  777,
+ /*   370 */   747,  218,  794,  100,  793,  769,  114,  115,   26,  684,
+ /*   380 */   112,  202,  131,  183,   24,  211,  681,  212,  855,  532,
+ /*   390 */    70,  854,   93,  852,  118,  702,  554,   25,   23,  132,
+ /*   400 */   671,   79,   52,  186,  669,   81,   82,  667,  666,  172,
+ /*   410 */   127,  664,  190,  663,  662,  661,  660,  652,  129,  658,
+ /*   420 */    49,  656,  654,  766,  781,  782,  795,  104,  195,   44,
+ /*   430 */   193,  191,  189,  187,  210,  105,   75,   28,  221,  199,
+ /*   440 */   222,  223,   53,  225,  224,  149,  226,   62,   65,  703,
+ /*   450 */   227,  237,  630,  173,  174,  629,  177,  665,  628,  119,
+ /*   460 */   176,   92,  121,  125,  120,  751,  122,  123,  659,  124,
+ /*   470 */   108,  106,  107,  109,  110,  111,   94,    1,    2,  621,
+ /*   480 */   180,  184,  541,   55,   57,  555,  101,  156,  188,  198,
+ /*   490 */    19,   63,    5,  560,  102,    4,    6,  601,   20,   15,
+ /*   500 */     7,  488,  484,  200,  482,  481,  480,  477,  450,  209,
+ /*   510 */    66,   45,   22,   69,   71,  517,  516,  514,  471,   54,
+ /*   520 */   469,  461,  467,  463,  465,  459,  457,  487,  486,  485,
+ /*   530 */   483,  479,  478,  476,   46,  448,  421,  419,  634,  633,
+ /*   540 */   633,  633,  633,  633,  633,  633,  633,  633,  633,  633,
+ /*   550 */    96,   97,
 };
 static const YYCODETYPE yy_lookahead[] = {
- /*     0 */   226,    1,  228,  260,  230,  207,  208,    5,  234,    9,
- /*    10 */   236,  237,  238,   13,   14,    2,   16,   17,    5,  260,
- /*    20 */     7,   21,    9,    1,   24,   25,   26,   27,   28,  209,
- /*    30 */   210,    9,  210,   33,   34,   33,   34,   37,   38,   39,
- /*    40 */    13,   14,  244,   16,   17,  260,   33,   34,   21,  260,
- /*    50 */   210,   24,   25,   26,   27,   28,  258,  260,  269,  270,
- /*    60 */    33,   34,  240,  210,   37,   38,   39,   45,   46,   47,
+ /*     0 */   211,    1,  262,  215,  208,  209,  218,    5,  262,    9,
+ /*    10 */   262,  271,  272,   13,   14,  262,   16,   17,  272,  271,
+ /*    20 */   272,   21,  262,    1,   24,   25,   26,   27,   28,  228,
+ /*    30 */   211,    9,  262,   33,   34,   33,   34,   37,   38,   39,
+ /*    40 */    13,   14,  246,   16,   17,   37,   38,   39,   21,  248,
+ /*    50 */     1,   24,   25,   26,   27,   28,  260,  268,    9,  270,
+ /*    60 */    33,   34,  211,  211,   37,   38,   39,   45,   46,   47,
  /*    70 */    48,   49,   50,   51,   52,   53,   54,   55,   56,   57,
- /*    80 */    13,   14,  246,   16,   17,  245,  247,  260,   21,  260,
- /*    90 */   210,   24,   25,   26,   27,   28,  269,  270,  259,  270,
- /*   100 */    33,   34,  102,   78,   37,   38,   39,  210,   14,   98,
- /*   110 */    16,   17,  101,  102,  261,   21,  244,  104,   24,   25,
- /*   120 */    26,   27,   28,  243,  103,  245,   99,   33,   34,  108,
- /*   130 */   258,   37,   38,   39,   85,   86,   87,   88,   89,   90,
- /*   140 */    91,   92,   93,   94,   95,   96,   97,  226,   63,  228,
- /*   150 */   229,  230,  231,  232,  233,  234,  235,  236,  237,  238,
- /*   160 */   239,   16,   17,  266,  210,  268,   21,  126,  227,   24,
- /*   170 */    25,   26,   27,   28,  133,  134,  210,    1,   33,   34,
- /*   180 */     1,   44,   37,   38,   39,    1,    2,  246,    9,    5,
- /*   190 */    98,    7,   72,    9,  210,    1,    2,  105,   78,    5,
- /*   200 */    63,    7,  210,    9,  210,  210,   69,   70,   71,  243,
- /*   210 */   227,  245,  127,   37,   77,  130,  131,   33,   34,  103,
- /*   220 */   266,   37,   64,   65,   66,   67,   68,   33,   34,  246,
- /*   230 */   227,   37,   74,    1,    2,   98,  120,    5,  243,    7,
- /*   240 */   245,    9,  105,   25,   26,   27,   28,  260,  264,  246,
- /*   250 */   266,   33,   34,  103,  262,   37,   38,   39,  214,  260,
- /*   260 */   266,  217,  125,  115,  116,   33,   34,   85,   86,  132,
- /*   270 */    88,  210,  122,    5,   92,    7,   94,   95,   96,   33,
- /*   280 */    34,  246,   98,   37,   38,   39,   64,   65,   66,   67,
- /*   290 */    68,  260,   98,  109,   37,   38,   39,   64,   65,   66,
- /*   300 */    67,   68,  214,  109,  243,  217,  245,   98,  124,   60,
- /*   310 */    61,   62,  214,  260,  105,  217,   99,    5,  124,    7,
- /*   320 */    99,   59,   99,  106,  103,   99,  103,   99,   99,  103,
- /*   330 */    98,  103,  103,   99,   99,  128,  129,  103,  103,  128,
- /*   340 */   129,  109,    5,    5,    7,    7,   72,   73,  241,  260,
- /*   350 */   260,  260,  241,  260,  241,  260,  124,  260,  241,  260,
- /*   360 */    98,  260,  260,  260,  241,  241,  244,  242,  241,  210,
- /*   370 */   210,  267,  267,  210,  248,  210,  210,  210,  210,  210,
- /*   380 */   210,  210,  210,  210,  210,  210,  105,  210,  210,   59,
- /*   390 */   210,  210,  109,  210,  263,  210,  210,  210,  210,  210,
- /*   400 */   210,  210,  210,  210,  263,  210,  210,  210,  210,  210,
- /*   410 */   210,  210,  210,  210,  210,  210,  119,  121,  257,  211,
- /*   420 */   256,  211,  118,  211,  113,  117,  112,  111,  110,  123,
- /*   430 */    75,   84,   83,   49,   80,   82,   53,   81,  211,   79,
- /*   440 */    75,  211,    5,    5,  135,    5,  215,    5,  211,    5,
- /*   450 */   244,  215,  135,   87,  211,  126,  212,  212,  211,  219,
- /*   460 */   218,   99,  223,  225,  244,  224,  250,  253,  255,  254,
- /*   470 */   222,  252,  220,  213,  249,  251,  221,  216,  106,  103,
- /*   480 */   107,   99,    1,   98,   98,  114,   99,   98,  114,   72,
- /*   490 */    99,  103,   98,  103,   98,  100,   98,  100,    9,    5,
- /*   500 */     5,    5,    5,    5,   76,   15,   72,   16,  129,  129,
- /*   510 */   103,    5,    5,   99,   98,    5,    5,    5,    5,    5,
- /*   520 */     5,    5,    5,    5,    5,    5,    5,    5,    5,  103,
- /*   530 */    76,   59,   58,    0,  271,  271,  271,  271,  271,  271,
- /*   540 */   271,  271,  271,  271,  271,   21,   21,  271,  271,  271,
- /*   550 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   560 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   570 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   580 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   590 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   600 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   610 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   620 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   630 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   640 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   650 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   660 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   670 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   680 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   690 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   700 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   710 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   720 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   730 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   740 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   750 */   271,  271,  271,
+ /*    80 */    13,   14,   78,   16,   17,  266,  262,  268,   21,  211,
+ /*    90 */   262,   24,   25,   26,   27,   28,  245,  245,  247,  247,
+ /*   100 */    33,   34,   63,  103,   37,   38,   39,  211,  262,   85,
+ /*   110 */    86,   87,   88,   89,   90,   91,   92,   93,   94,   95,
+ /*   120 */    96,   97,   98,  245,  262,  247,  227,  100,  229,  230,
+ /*   130 */   231,  232,  233,  234,  235,  236,  237,  238,  239,  240,
+ /*   140 */   241,   14,   99,   16,   17,  102,  103,  211,   21,  210,
+ /*   150 */   211,   24,   25,   26,   27,   28,   60,   61,   62,  263,
+ /*   160 */    33,   34,  228,  211,   37,   38,   39,  128,   16,   17,
+ /*   170 */   131,  132,    5,   21,    7,  211,   24,   25,   26,   27,
+ /*   180 */    28,  245,  248,  247,  228,   33,   34,  127,   44,   37,
+ /*   190 */    38,   39,    1,    2,  134,  135,    5,  246,    7,  211,
+ /*   200 */     9,   59,    1,    2,  248,   99,    5,   63,    7,  262,
+ /*   210 */     9,  260,  106,   69,   70,   71,  264,  116,  117,   85,
+ /*   220 */    86,   77,   88,   89,   33,   34,  211,   93,   37,   95,
+ /*   230 */    96,   97,  268,   72,   33,   34,    1,    2,   37,   78,
+ /*   240 */     5,   99,    7,   99,    9,   64,   65,   66,   67,   68,
+ /*   250 */   106,   25,   26,   27,   28,   74,  268,  242,  211,   33,
+ /*   260 */    34,  104,  249,   37,   38,   39,  109,  262,   33,   34,
+ /*   270 */   126,  227,  100,  229,  261,  231,  232,  133,  215,  107,
+ /*   280 */   236,  218,  238,  239,  240,  215,    2,  262,  218,    5,
+ /*   290 */    99,    7,  100,    9,  247,    5,  104,    7,   33,   34,
+ /*   300 */    99,  110,   37,   38,   39,   64,   65,   66,   67,   68,
+ /*   310 */     1,  110,  262,  104,  129,  130,  125,   33,   34,   64,
+ /*   320 */    65,   66,   67,   68,   99,  104,  125,  262,  100,  100,
+ /*   330 */   121,  106,  104,  104,   99,  262,  100,  100,  100,  100,
+ /*   340 */   104,  104,  104,  104,  123,  110,   37,  129,  130,    5,
+ /*   350 */     5,    7,    7,   72,   73,  262,  262,  262,  248,  262,
+ /*   360 */   125,  262,  248,  243,  243,  243,  243,  243,  243,  211,
+ /*   370 */   244,  243,  269,  211,  269,  246,  211,  211,  106,  211,
+ /*   380 */   250,  211,  211,  246,  211,  211,  211,  211,  211,  105,
+ /*   390 */   211,  211,   59,  211,  211,  211,  110,  211,  211,  211,
+ /*   400 */   211,  211,  120,  265,  211,  211,  211,  211,  211,  211,
+ /*   410 */   211,  211,  265,  211,  211,  211,  211,  211,  211,  211,
+ /*   420 */   122,  211,  211,  259,  212,  212,  212,  258,  114,  119,
+ /*   430 */   118,  113,  112,  111,   75,  257,   84,  124,   83,  212,
+ /*   440 */    49,   80,  212,   53,   82,  212,   81,  216,  216,  226,
+ /*   450 */    79,   75,    5,  136,    5,    5,    5,  212,    5,  225,
+ /*   460 */   136,  213,  220,  219,  224,  246,  223,  221,  212,  222,
+ /*   470 */   254,  256,  255,  253,  252,  251,  213,  217,  214,   87,
+ /*   480 */   127,  107,  100,  108,  104,  100,   99,    1,   99,  101,
+ /*   490 */   104,   72,  115,  100,   99,   99,  115,  100,  104,   99,
+ /*   500 */    99,    9,    5,  101,    5,    5,    5,    5,   76,   15,
+ /*   510 */    72,   16,  104,  130,  130,    5,    5,  100,    5,   99,
+ /*   520 */     5,    5,    5,    5,    5,    5,    5,    5,    5,    5,
+ /*   530 */     5,    5,    5,    5,  104,   76,   59,   58,    0,  273,
+ /*   540 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   550 */    21,   21,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   560 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   570 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   580 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   590 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   600 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   610 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   620 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   630 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   640 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   650 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   660 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   670 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   680 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   690 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   700 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   710 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   720 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   730 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   740 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   750 */   273,  273,  273,  273,  273,  273,  273,  273,  273,
 };
-#define YY_SHIFT_COUNT    (241)
+#define YY_SHIFT_COUNT    (242)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (533)
+#define YY_SHIFT_MAX      (538)
 static const unsigned short int yy_shift_ofst[] = {
- /*     0 */   137,   49,  182,  184,  232,  179,  179,  179,  179,  179,
- /*    10 */   179,    0,   22,  232,   13,   13,   13,   92,  179,  179,
- /*    20 */   179,  179,  179,  120,   25,   25,  547,  194,  232,  232,
- /*    30 */   232,  232,  232,  232,  232,  232,  232,  232,  232,  232,
- /*    40 */   232,  232,  232,  232,  232,   13,   13,    2,    2,    2,
- /*    50 */     2,    2,    2,   11,    2,  209,  179,  179,  148,  148,
- /*    60 */    21,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    70 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    80 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    90 */   179,  179,  179,  179,  179,  179,  179,  179,  281,  330,
- /*   100 */   330,  283,  283,  330,  297,  296,  304,  311,  308,  314,
- /*   110 */   316,  318,  306,  281,  330,  330,  355,  355,  330,  347,
- /*   120 */   349,  384,  354,  353,  383,  356,  360,  330,  365,  330,
- /*   130 */   365,  547,  547,   27,   67,   67,   67,   94,  145,  218,
- /*   140 */   218,  218,  158,  246,  246,  246,  246,  222,  233,   85,
- /*   150 */    41,  257,  257,  249,  217,  221,  223,  226,  228,  268,
- /*   160 */   312,  176,  262,  150,  116,  229,  234,  235,  207,  211,
- /*   170 */   337,  338,  274,  437,  309,  438,  440,  317,  442,  444,
- /*   180 */   366,  329,  372,  362,  373,  376,  382,  385,  481,  386,
- /*   190 */   387,  389,  388,  371,  390,  374,  391,  394,  396,  395,
- /*   200 */   398,  397,  417,  489,  494,  495,  496,  497,  498,  428,
- /*   210 */   490,  434,  491,  379,  380,  407,  506,  507,  414,  416,
- /*   220 */   407,  510,  511,  512,  513,  514,  515,  516,  517,  518,
- /*   230 */   519,  520,  521,  522,  523,  426,  454,  524,  525,  472,
- /*   240 */   474,  533,
+ /*     0 */   144,   24,  134,  191,  235,   49,   49,   49,   49,   49,
+ /*    10 */    49,    0,   22,  235,  284,  284,  284,  106,   49,   49,
+ /*    20 */    49,   49,   49,  161,    4,    4,  552,  201,  235,  235,
+ /*    30 */   235,  235,  235,  235,  235,  235,  235,  235,  235,  235,
+ /*    40 */   235,  235,  235,  235,  235,  284,  284,    2,    2,    2,
+ /*    50 */     2,    2,    2,   43,    2,  225,   49,   49,  101,  101,
+ /*    60 */   157,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    70 */    49,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    80 */    49,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    90 */    49,   49,   49,   49,   49,   49,   49,   49,  272,  333,
+ /*   100 */   333,  286,  286,  333,  282,  298,  310,  314,  312,  318,
+ /*   110 */   320,  322,  313,  272,  333,  333,  359,  359,  333,  352,
+ /*   120 */   355,  391,  361,  362,  390,  365,  371,  333,  376,  333,
+ /*   130 */   376,  552,  552,   27,   67,   67,   67,  127,  152,  226,
+ /*   140 */   226,  226,  181,  265,  265,  265,  265,  241,  255,   39,
+ /*   150 */    60,    8,    8,   96,  172,  192,  228,  229,  236,  167,
+ /*   160 */   290,  309,  142,  221,  209,  237,  238,  239,  185,  218,
+ /*   170 */   344,  345,  281,  447,  317,  449,  450,  324,  451,  453,
+ /*   180 */   392,  353,  374,  382,  375,  380,  385,  387,  486,  389,
+ /*   190 */   393,  395,  386,  377,  394,  381,  397,  396,  400,  388,
+ /*   200 */   401,  402,  419,  492,  497,  499,  500,  501,  502,  432,
+ /*   210 */   494,  438,  495,  383,  384,  408,  510,  511,  417,  420,
+ /*   220 */   408,  513,  515,  516,  517,  518,  519,  520,  521,  522,
+ /*   230 */   523,  524,  525,  526,  527,  528,  430,  459,  529,  530,
+ /*   240 */   477,  479,  538,
 };
 #define YY_REDUCE_COUNT (132)
-#define YY_REDUCE_MIN   (-257)
-#define YY_REDUCE_MAX   (261)
+#define YY_REDUCE_MIN   (-260)
+#define YY_REDUCE_MAX   (264)
 static const short yy_reduce_ofst[] = {
- /*     0 */  -202,  -79, -226, -211, -173, -103,  -16, -120,  -34,   -5,
- /*    10 */    61, -147, -180, -171,  -59,  -17,    3, -128,   -8,  -46,
- /*    20 */    -6, -178, -160,   44,   88,   98, -161, -257, -241, -215,
- /*    30 */  -203,  -13,   -1,   31,   53,   89,   90,   91,   93,   95,
- /*    40 */    97,   99,  101,  102,  103, -164,   35,  107,  111,  113,
- /*    50 */   117,  123,  124,  125,  127,  122,  159,  160,  104,  105,
- /*    60 */   126,  163,  165,  166,  167,  168,  169,  170,  171,  172,
- /*    70 */   173,  174,  175,  177,  178,  180,  181,  183,  185,  186,
- /*    80 */   187,  188,  189,  190,  191,  192,  193,  195,  196,  197,
- /*    90 */   198,  199,  200,  201,  202,  203,  204,  205,  206,  208,
- /*   100 */   210,  131,  141,  212,  161,  164,  213,  215,  214,  219,
- /*   110 */   224,  216,  225,  220,  227,  230,  231,  236,  237,  238,
- /*   120 */   241,  239,  240,  248,  252,  255,  242,  243,  244,  247,
- /*   130 */   245,  261,  260,
+ /*     0 */  -204, -101,   44, -260, -252, -211, -181, -149, -148, -122,
+ /*    10 */   -64, -104,  -61, -254, -199,  -66,  -44,  -49,  -48,  -36,
+ /*    20 */   -12,   15,   47, -212,   63,   70,   13, -247, -240, -230,
+ /*    30 */  -176, -172, -154, -138,  -53,    5,   25,   50,   65,   73,
+ /*    40 */    93,   94,   95,   97,   99,  110,  114,  120,  121,  122,
+ /*    50 */   123,  124,  125,  126,  128,  129,  158,  162,  103,  105,
+ /*    60 */   130,  165,  166,  168,  170,  171,  173,  174,  175,  176,
+ /*    70 */   177,  179,  180,  182,  183,  184,  186,  187,  188,  189,
+ /*    80 */   190,  193,  194,  195,  196,  197,  198,  199,  200,  202,
+ /*    90 */   203,  204,  205,  206,  207,  208,  210,  211,  137,  212,
+ /*   100 */   213,  138,  147,  214,  164,  169,  178,  215,  217,  216,
+ /*   110 */   220,  222,  224,  219,  227,  230,  231,  232,  233,  223,
+ /*   120 */   234,  240,  242,  243,  246,  247,  244,  245,  248,  256,
+ /*   130 */   263,  260,  264,
 };
 static const YYACTIONTYPE yy_default[] = {
- /*     0 */   624,  676,  665,  839,  839,  624,  624,  624,  624,  624,
- /*    10 */   624,  769,  642,  839,  624,  624,  624,  624,  624,  624,
- /*    20 */   624,  624,  624,  678,  678,  678,  764,  624,  624,  624,
- /*    30 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    40 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    50 */   624,  624,  624,  624,  624,  624,  624,  624,  788,  788,
- /*    60 */   762,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    70 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  663,
- /*    80 */   624,  661,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    90 */   624,  624,  624,  624,  624,  650,  624,  624,  624,  644,
- /*   100 */   644,  624,  624,  644,  795,  799,  793,  781,  789,  780,
- /*   110 */   776,  775,  803,  624,  644,  644,  673,  673,  644,  694,
- /*   120 */   692,  690,  682,  688,  684,  686,  680,  644,  671,  644,
- /*   130 */   671,  710,  724,  624,  804,  838,  794,  822,  821,  834,
- /*   140 */   828,  827,  624,  826,  825,  824,  823,  624,  624,  624,
- /*   150 */   624,  830,  829,  624,  624,  624,  624,  624,  624,  624,
- /*   160 */   624,  624,  806,  800,  796,  624,  624,  624,  624,  624,
- /*   170 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   180 */   624,  624,  761,  624,  624,  770,  624,  624,  624,  624,
- /*   190 */   624,  624,  790,  624,  782,  624,  624,  624,  624,  624,
- /*   200 */   624,  738,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   210 */   624,  624,  624,  624,  624,  843,  624,  624,  624,  732,
- /*   220 */   841,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   230 */   624,  624,  624,  624,  624,  697,  624,  648,  646,  624,
- /*   240 */   640,  624,
+ /*     0 */   631,  683,  672,  849,  849,  631,  631,  631,  631,  631,
+ /*    10 */   631,  779,  649,  849,  631,  631,  631,  631,  631,  631,
+ /*    20 */   631,  631,  631,  685,  685,  685,  774,  631,  631,  631,
+ /*    30 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    40 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    50 */   631,  631,  631,  631,  631,  631,  631,  631,  798,  798,
+ /*    60 */   772,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    70 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  670,
+ /*    80 */   631,  668,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    90 */   631,  631,  631,  631,  631,  657,  631,  631,  631,  651,
+ /*   100 */   651,  631,  631,  651,  805,  809,  803,  791,  799,  790,
+ /*   110 */   786,  785,  813,  631,  651,  651,  680,  680,  651,  701,
+ /*   120 */   699,  697,  689,  695,  691,  693,  687,  651,  678,  651,
+ /*   130 */   678,  718,  733,  631,  814,  848,  804,  832,  831,  844,
+ /*   140 */   838,  837,  631,  836,  835,  834,  833,  631,  631,  631,
+ /*   150 */   631,  840,  839,  631,  631,  631,  631,  631,  631,  631,
+ /*   160 */   631,  631,  816,  810,  806,  631,  631,  631,  631,  631,
+ /*   170 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   180 */   631,  631,  771,  631,  631,  780,  631,  631,  631,  631,
+ /*   190 */   631,  631,  800,  631,  792,  631,  631,  631,  631,  631,
+ /*   200 */   631,  748,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   210 */   631,  631,  631,  631,  631,  853,  631,  631,  631,  742,
+ /*   220 */   851,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   230 */   631,  631,  631,  631,  631,  631,  704,  631,  655,  653,
+ /*   240 */   631,  647,  631,
 };
 /********** End of lemon-generated parsing tables *****************************/
 
@@ -521,6 +522,7 @@ static const YYCODETYPE yyFallback[] = {
     0,  /*  MAXTABLES => nothing */
     0,  /*      CACHE => nothing */
     0,  /*    REPLICA => nothing */
+    0,  /*     QUORUM => nothing */
     0,  /*       DAYS => nothing */
     0,  /*    MINROWS => nothing */
     0,  /*    MAXROWS => nothing */
@@ -813,188 +815,190 @@ static const char *const yyTokenName[] = {
   /*   86 */ "MAXTABLES",
   /*   87 */ "CACHE",
   /*   88 */ "REPLICA",
-  /*   89 */ "DAYS",
-  /*   90 */ "MINROWS",
-  /*   91 */ "MAXROWS",
-  /*   92 */ "BLOCKS",
-  /*   93 */ "CTIME",
-  /*   94 */ "WAL",
-  /*   95 */ "FSYNC",
-  /*   96 */ "COMP",
-  /*   97 */ "PRECISION",
-  /*   98 */ "LP",
-  /*   99 */ "RP",
-  /*  100 */ "TAGS",
-  /*  101 */ "USING",
-  /*  102 */ "AS",
-  /*  103 */ "COMMA",
-  /*  104 */ "NULL",
-  /*  105 */ "SELECT",
-  /*  106 */ "UNION",
-  /*  107 */ "ALL",
-  /*  108 */ "FROM",
-  /*  109 */ "VARIABLE",
-  /*  110 */ "INTERVAL",
-  /*  111 */ "FILL",
-  /*  112 */ "SLIDING",
-  /*  113 */ "ORDER",
-  /*  114 */ "BY",
-  /*  115 */ "ASC",
-  /*  116 */ "DESC",
-  /*  117 */ "GROUP",
-  /*  118 */ "HAVING",
-  /*  119 */ "LIMIT",
-  /*  120 */ "OFFSET",
-  /*  121 */ "SLIMIT",
-  /*  122 */ "SOFFSET",
-  /*  123 */ "WHERE",
-  /*  124 */ "NOW",
-  /*  125 */ "RESET",
-  /*  126 */ "QUERY",
-  /*  127 */ "ADD",
-  /*  128 */ "COLUMN",
-  /*  129 */ "TAG",
-  /*  130 */ "CHANGE",
-  /*  131 */ "SET",
-  /*  132 */ "KILL",
-  /*  133 */ "CONNECTION",
-  /*  134 */ "STREAM",
-  /*  135 */ "COLON",
-  /*  136 */ "ABORT",
-  /*  137 */ "AFTER",
-  /*  138 */ "ATTACH",
-  /*  139 */ "BEFORE",
-  /*  140 */ "BEGIN",
-  /*  141 */ "CASCADE",
-  /*  142 */ "CLUSTER",
-  /*  143 */ "CONFLICT",
-  /*  144 */ "COPY",
-  /*  145 */ "DEFERRED",
-  /*  146 */ "DELIMITERS",
-  /*  147 */ "DETACH",
-  /*  148 */ "EACH",
-  /*  149 */ "END",
-  /*  150 */ "EXPLAIN",
-  /*  151 */ "FAIL",
-  /*  152 */ "FOR",
-  /*  153 */ "IGNORE",
-  /*  154 */ "IMMEDIATE",
-  /*  155 */ "INITIALLY",
-  /*  156 */ "INSTEAD",
-  /*  157 */ "MATCH",
-  /*  158 */ "KEY",
-  /*  159 */ "OF",
-  /*  160 */ "RAISE",
-  /*  161 */ "REPLACE",
-  /*  162 */ "RESTRICT",
-  /*  163 */ "ROW",
-  /*  164 */ "STATEMENT",
-  /*  165 */ "TRIGGER",
-  /*  166 */ "VIEW",
-  /*  167 */ "COUNT",
-  /*  168 */ "SUM",
-  /*  169 */ "AVG",
-  /*  170 */ "MIN",
-  /*  171 */ "MAX",
-  /*  172 */ "FIRST",
-  /*  173 */ "LAST",
-  /*  174 */ "TOP",
-  /*  175 */ "BOTTOM",
-  /*  176 */ "STDDEV",
-  /*  177 */ "PERCENTILE",
-  /*  178 */ "APERCENTILE",
-  /*  179 */ "LEASTSQUARES",
-  /*  180 */ "HISTOGRAM",
-  /*  181 */ "DIFF",
-  /*  182 */ "SPREAD",
-  /*  183 */ "TWA",
-  /*  184 */ "INTERP",
-  /*  185 */ "LAST_ROW",
-  /*  186 */ "RATE",
-  /*  187 */ "IRATE",
-  /*  188 */ "SUM_RATE",
-  /*  189 */ "SUM_IRATE",
-  /*  190 */ "AVG_RATE",
-  /*  191 */ "AVG_IRATE",
-  /*  192 */ "TBID",
-  /*  193 */ "SEMI",
-  /*  194 */ "NONE",
-  /*  195 */ "PREV",
-  /*  196 */ "LINEAR",
-  /*  197 */ "IMPORT",
-  /*  198 */ "METRIC",
-  /*  199 */ "TBNAME",
-  /*  200 */ "JOIN",
-  /*  201 */ "METRICS",
-  /*  202 */ "STABLE",
-  /*  203 */ "INSERT",
-  /*  204 */ "INTO",
-  /*  205 */ "VALUES",
-  /*  206 */ "error",
-  /*  207 */ "program",
-  /*  208 */ "cmd",
-  /*  209 */ "dbPrefix",
-  /*  210 */ "ids",
-  /*  211 */ "cpxName",
-  /*  212 */ "ifexists",
-  /*  213 */ "alter_db_optr",
-  /*  214 */ "acct_optr",
-  /*  215 */ "ifnotexists",
-  /*  216 */ "db_optr",
-  /*  217 */ "pps",
-  /*  218 */ "tseries",
-  /*  219 */ "dbs",
-  /*  220 */ "streams",
-  /*  221 */ "storage",
-  /*  222 */ "qtime",
-  /*  223 */ "users",
-  /*  224 */ "conns",
-  /*  225 */ "state",
-  /*  226 */ "keep",
-  /*  227 */ "tagitemlist",
-  /*  228 */ "tables",
-  /*  229 */ "cache",
-  /*  230 */ "replica",
-  /*  231 */ "days",
-  /*  232 */ "minrows",
-  /*  233 */ "maxrows",
-  /*  234 */ "blocks",
-  /*  235 */ "ctime",
-  /*  236 */ "wal",
-  /*  237 */ "fsync",
-  /*  238 */ "comp",
-  /*  239 */ "prec",
-  /*  240 */ "typename",
-  /*  241 */ "signed",
-  /*  242 */ "create_table_args",
-  /*  243 */ "columnlist",
-  /*  244 */ "select",
-  /*  245 */ "column",
-  /*  246 */ "tagitem",
-  /*  247 */ "selcollist",
-  /*  248 */ "from",
-  /*  249 */ "where_opt",
-  /*  250 */ "interval_opt",
-  /*  251 */ "fill_opt",
-  /*  252 */ "sliding_opt",
-  /*  253 */ "groupby_opt",
-  /*  254 */ "orderby_opt",
-  /*  255 */ "having_opt",
-  /*  256 */ "slimit_opt",
-  /*  257 */ "limit_opt",
-  /*  258 */ "union",
-  /*  259 */ "sclp",
-  /*  260 */ "expr",
-  /*  261 */ "as",
-  /*  262 */ "tablelist",
-  /*  263 */ "tmvar",
-  /*  264 */ "sortlist",
-  /*  265 */ "sortitem",
-  /*  266 */ "item",
-  /*  267 */ "sortorder",
-  /*  268 */ "grouplist",
-  /*  269 */ "exprlist",
-  /*  270 */ "expritem",
+  /*   89 */ "QUORUM",
+  /*   90 */ "DAYS",
+  /*   91 */ "MINROWS",
+  /*   92 */ "MAXROWS",
+  /*   93 */ "BLOCKS",
+  /*   94 */ "CTIME",
+  /*   95 */ "WAL",
+  /*   96 */ "FSYNC",
+  /*   97 */ "COMP",
+  /*   98 */ "PRECISION",
+  /*   99 */ "LP",
+  /*  100 */ "RP",
+  /*  101 */ "TAGS",
+  /*  102 */ "USING",
+  /*  103 */ "AS",
+  /*  104 */ "COMMA",
+  /*  105 */ "NULL",
+  /*  106 */ "SELECT",
+  /*  107 */ "UNION",
+  /*  108 */ "ALL",
+  /*  109 */ "FROM",
+  /*  110 */ "VARIABLE",
+  /*  111 */ "INTERVAL",
+  /*  112 */ "FILL",
+  /*  113 */ "SLIDING",
+  /*  114 */ "ORDER",
+  /*  115 */ "BY",
+  /*  116 */ "ASC",
+  /*  117 */ "DESC",
+  /*  118 */ "GROUP",
+  /*  119 */ "HAVING",
+  /*  120 */ "LIMIT",
+  /*  121 */ "OFFSET",
+  /*  122 */ "SLIMIT",
+  /*  123 */ "SOFFSET",
+  /*  124 */ "WHERE",
+  /*  125 */ "NOW",
+  /*  126 */ "RESET",
+  /*  127 */ "QUERY",
+  /*  128 */ "ADD",
+  /*  129 */ "COLUMN",
+  /*  130 */ "TAG",
+  /*  131 */ "CHANGE",
+  /*  132 */ "SET",
+  /*  133 */ "KILL",
+  /*  134 */ "CONNECTION",
+  /*  135 */ "STREAM",
+  /*  136 */ "COLON",
+  /*  137 */ "ABORT",
+  /*  138 */ "AFTER",
+  /*  139 */ "ATTACH",
+  /*  140 */ "BEFORE",
+  /*  141 */ "BEGIN",
+  /*  142 */ "CASCADE",
+  /*  143 */ "CLUSTER",
+  /*  144 */ "CONFLICT",
+  /*  145 */ "COPY",
+  /*  146 */ "DEFERRED",
+  /*  147 */ "DELIMITERS",
+  /*  148 */ "DETACH",
+  /*  149 */ "EACH",
+  /*  150 */ "END",
+  /*  151 */ "EXPLAIN",
+  /*  152 */ "FAIL",
+  /*  153 */ "FOR",
+  /*  154 */ "IGNORE",
+  /*  155 */ "IMMEDIATE",
+  /*  156 */ "INITIALLY",
+  /*  157 */ "INSTEAD",
+  /*  158 */ "MATCH",
+  /*  159 */ "KEY",
+  /*  160 */ "OF",
+  /*  161 */ "RAISE",
+  /*  162 */ "REPLACE",
+  /*  163 */ "RESTRICT",
+  /*  164 */ "ROW",
+  /*  165 */ "STATEMENT",
+  /*  166 */ "TRIGGER",
+  /*  167 */ "VIEW",
+  /*  168 */ "COUNT",
+  /*  169 */ "SUM",
+  /*  170 */ "AVG",
+  /*  171 */ "MIN",
+  /*  172 */ "MAX",
+  /*  173 */ "FIRST",
+  /*  174 */ "LAST",
+  /*  175 */ "TOP",
+  /*  176 */ "BOTTOM",
+  /*  177 */ "STDDEV",
+  /*  178 */ "PERCENTILE",
+  /*  179 */ "APERCENTILE",
+  /*  180 */ "LEASTSQUARES",
+  /*  181 */ "HISTOGRAM",
+  /*  182 */ "DIFF",
+  /*  183 */ "SPREAD",
+  /*  184 */ "TWA",
+  /*  185 */ "INTERP",
+  /*  186 */ "LAST_ROW",
+  /*  187 */ "RATE",
+  /*  188 */ "IRATE",
+  /*  189 */ "SUM_RATE",
+  /*  190 */ "SUM_IRATE",
+  /*  191 */ "AVG_RATE",
+  /*  192 */ "AVG_IRATE",
+  /*  193 */ "TBID",
+  /*  194 */ "SEMI",
+  /*  195 */ "NONE",
+  /*  196 */ "PREV",
+  /*  197 */ "LINEAR",
+  /*  198 */ "IMPORT",
+  /*  199 */ "METRIC",
+  /*  200 */ "TBNAME",
+  /*  201 */ "JOIN",
+  /*  202 */ "METRICS",
+  /*  203 */ "STABLE",
+  /*  204 */ "INSERT",
+  /*  205 */ "INTO",
+  /*  206 */ "VALUES",
+  /*  207 */ "error",
+  /*  208 */ "program",
+  /*  209 */ "cmd",
+  /*  210 */ "dbPrefix",
+  /*  211 */ "ids",
+  /*  212 */ "cpxName",
+  /*  213 */ "ifexists",
+  /*  214 */ "alter_db_optr",
+  /*  215 */ "acct_optr",
+  /*  216 */ "ifnotexists",
+  /*  217 */ "db_optr",
+  /*  218 */ "pps",
+  /*  219 */ "tseries",
+  /*  220 */ "dbs",
+  /*  221 */ "streams",
+  /*  222 */ "storage",
+  /*  223 */ "qtime",
+  /*  224 */ "users",
+  /*  225 */ "conns",
+  /*  226 */ "state",
+  /*  227 */ "keep",
+  /*  228 */ "tagitemlist",
+  /*  229 */ "tables",
+  /*  230 */ "cache",
+  /*  231 */ "replica",
+  /*  232 */ "quorum",
+  /*  233 */ "days",
+  /*  234 */ "minrows",
+  /*  235 */ "maxrows",
+  /*  236 */ "blocks",
+  /*  237 */ "ctime",
+  /*  238 */ "wal",
+  /*  239 */ "fsync",
+  /*  240 */ "comp",
+  /*  241 */ "prec",
+  /*  242 */ "typename",
+  /*  243 */ "signed",
+  /*  244 */ "create_table_args",
+  /*  245 */ "columnlist",
+  /*  246 */ "select",
+  /*  247 */ "column",
+  /*  248 */ "tagitem",
+  /*  249 */ "selcollist",
+  /*  250 */ "from",
+  /*  251 */ "where_opt",
+  /*  252 */ "interval_opt",
+  /*  253 */ "fill_opt",
+  /*  254 */ "sliding_opt",
+  /*  255 */ "groupby_opt",
+  /*  256 */ "orderby_opt",
+  /*  257 */ "having_opt",
+  /*  258 */ "slimit_opt",
+  /*  259 */ "limit_opt",
+  /*  260 */ "union",
+  /*  261 */ "sclp",
+  /*  262 */ "expr",
+  /*  263 */ "as",
+  /*  264 */ "tablelist",
+  /*  265 */ "tmvar",
+  /*  266 */ "sortlist",
+  /*  267 */ "sortitem",
+  /*  268 */ "item",
+  /*  269 */ "sortorder",
+  /*  270 */ "grouplist",
+  /*  271 */ "exprlist",
+  /*  272 */ "expritem",
 };
 #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
 
@@ -1076,155 +1080,158 @@ static const char *const yyRuleName[] = {
  /*  71 */ "tables ::= MAXTABLES INTEGER",
  /*  72 */ "cache ::= CACHE INTEGER",
  /*  73 */ "replica ::= REPLICA INTEGER",
- /*  74 */ "days ::= DAYS INTEGER",
- /*  75 */ "minrows ::= MINROWS INTEGER",
- /*  76 */ "maxrows ::= MAXROWS INTEGER",
- /*  77 */ "blocks ::= BLOCKS INTEGER",
- /*  78 */ "ctime ::= CTIME INTEGER",
- /*  79 */ "wal ::= WAL INTEGER",
- /*  80 */ "fsync ::= FSYNC INTEGER",
- /*  81 */ "comp ::= COMP INTEGER",
- /*  82 */ "prec ::= PRECISION STRING",
- /*  83 */ "db_optr ::=",
- /*  84 */ "db_optr ::= db_optr tables",
- /*  85 */ "db_optr ::= db_optr cache",
- /*  86 */ "db_optr ::= db_optr replica",
- /*  87 */ "db_optr ::= db_optr days",
- /*  88 */ "db_optr ::= db_optr minrows",
- /*  89 */ "db_optr ::= db_optr maxrows",
- /*  90 */ "db_optr ::= db_optr blocks",
- /*  91 */ "db_optr ::= db_optr ctime",
- /*  92 */ "db_optr ::= db_optr wal",
- /*  93 */ "db_optr ::= db_optr fsync",
- /*  94 */ "db_optr ::= db_optr comp",
- /*  95 */ "db_optr ::= db_optr prec",
- /*  96 */ "db_optr ::= db_optr keep",
- /*  97 */ "alter_db_optr ::=",
- /*  98 */ "alter_db_optr ::= alter_db_optr replica",
- /*  99 */ "alter_db_optr ::= alter_db_optr tables",
- /* 100 */ "alter_db_optr ::= alter_db_optr keep",
- /* 101 */ "alter_db_optr ::= alter_db_optr blocks",
- /* 102 */ "alter_db_optr ::= alter_db_optr comp",
- /* 103 */ "alter_db_optr ::= alter_db_optr wal",
- /* 104 */ "alter_db_optr ::= alter_db_optr fsync",
- /* 105 */ "typename ::= ids",
- /* 106 */ "typename ::= ids LP signed RP",
- /* 107 */ "signed ::= INTEGER",
- /* 108 */ "signed ::= PLUS INTEGER",
- /* 109 */ "signed ::= MINUS INTEGER",
- /* 110 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
- /* 111 */ "create_table_args ::= LP columnlist RP",
- /* 112 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
- /* 113 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
- /* 114 */ "create_table_args ::= AS select",
- /* 115 */ "columnlist ::= columnlist COMMA column",
- /* 116 */ "columnlist ::= column",
- /* 117 */ "column ::= ids typename",
- /* 118 */ "tagitemlist ::= tagitemlist COMMA tagitem",
- /* 119 */ "tagitemlist ::= tagitem",
- /* 120 */ "tagitem ::= INTEGER",
- /* 121 */ "tagitem ::= FLOAT",
- /* 122 */ "tagitem ::= STRING",
- /* 123 */ "tagitem ::= BOOL",
- /* 124 */ "tagitem ::= NULL",
- /* 125 */ "tagitem ::= MINUS INTEGER",
- /* 126 */ "tagitem ::= MINUS FLOAT",
- /* 127 */ "tagitem ::= PLUS INTEGER",
- /* 128 */ "tagitem ::= PLUS FLOAT",
- /* 129 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
- /* 130 */ "union ::= select",
- /* 131 */ "union ::= LP union RP",
- /* 132 */ "union ::= union UNION ALL select",
- /* 133 */ "union ::= union UNION ALL LP select RP",
- /* 134 */ "cmd ::= union",
- /* 135 */ "select ::= SELECT selcollist",
- /* 136 */ "sclp ::= selcollist COMMA",
- /* 137 */ "sclp ::=",
- /* 138 */ "selcollist ::= sclp expr as",
- /* 139 */ "selcollist ::= sclp STAR",
- /* 140 */ "as ::= AS ids",
- /* 141 */ "as ::= ids",
- /* 142 */ "as ::=",
- /* 143 */ "from ::= FROM tablelist",
- /* 144 */ "tablelist ::= ids cpxName",
- /* 145 */ "tablelist ::= tablelist COMMA ids cpxName",
- /* 146 */ "tmvar ::= VARIABLE",
- /* 147 */ "interval_opt ::= INTERVAL LP tmvar RP",
- /* 148 */ "interval_opt ::=",
- /* 149 */ "fill_opt ::=",
- /* 150 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
- /* 151 */ "fill_opt ::= FILL LP ID RP",
- /* 152 */ "sliding_opt ::= SLIDING LP tmvar RP",
- /* 153 */ "sliding_opt ::=",
- /* 154 */ "orderby_opt ::=",
- /* 155 */ "orderby_opt ::= ORDER BY sortlist",
- /* 156 */ "sortlist ::= sortlist COMMA item sortorder",
- /* 157 */ "sortlist ::= item sortorder",
- /* 158 */ "item ::= ids cpxName",
- /* 159 */ "sortorder ::= ASC",
- /* 160 */ "sortorder ::= DESC",
- /* 161 */ "sortorder ::=",
- /* 162 */ "groupby_opt ::=",
- /* 163 */ "groupby_opt ::= GROUP BY grouplist",
- /* 164 */ "grouplist ::= grouplist COMMA item",
- /* 165 */ "grouplist ::= item",
- /* 166 */ "having_opt ::=",
- /* 167 */ "having_opt ::= HAVING expr",
- /* 168 */ "limit_opt ::=",
- /* 169 */ "limit_opt ::= LIMIT signed",
- /* 170 */ "limit_opt ::= LIMIT signed OFFSET signed",
- /* 171 */ "limit_opt ::= LIMIT signed COMMA signed",
- /* 172 */ "slimit_opt ::=",
- /* 173 */ "slimit_opt ::= SLIMIT signed",
- /* 174 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
- /* 175 */ "slimit_opt ::= SLIMIT signed COMMA signed",
- /* 176 */ "where_opt ::=",
- /* 177 */ "where_opt ::= WHERE expr",
- /* 178 */ "expr ::= LP expr RP",
- /* 179 */ "expr ::= ID",
- /* 180 */ "expr ::= ID DOT ID",
- /* 181 */ "expr ::= ID DOT STAR",
- /* 182 */ "expr ::= INTEGER",
- /* 183 */ "expr ::= MINUS INTEGER",
- /* 184 */ "expr ::= PLUS INTEGER",
- /* 185 */ "expr ::= FLOAT",
- /* 186 */ "expr ::= MINUS FLOAT",
- /* 187 */ "expr ::= PLUS FLOAT",
- /* 188 */ "expr ::= STRING",
- /* 189 */ "expr ::= NOW",
- /* 190 */ "expr ::= VARIABLE",
- /* 191 */ "expr ::= BOOL",
- /* 192 */ "expr ::= ID LP exprlist RP",
- /* 193 */ "expr ::= ID LP STAR RP",
- /* 194 */ "expr ::= expr AND expr",
- /* 195 */ "expr ::= expr OR expr",
- /* 196 */ "expr ::= expr LT expr",
- /* 197 */ "expr ::= expr GT expr",
- /* 198 */ "expr ::= expr LE expr",
- /* 199 */ "expr ::= expr GE expr",
- /* 200 */ "expr ::= expr NE expr",
- /* 201 */ "expr ::= expr EQ expr",
- /* 202 */ "expr ::= expr PLUS expr",
- /* 203 */ "expr ::= expr MINUS expr",
- /* 204 */ "expr ::= expr STAR expr",
- /* 205 */ "expr ::= expr SLASH expr",
- /* 206 */ "expr ::= expr REM expr",
- /* 207 */ "expr ::= expr LIKE expr",
- /* 208 */ "expr ::= expr IN LP exprlist RP",
- /* 209 */ "exprlist ::= exprlist COMMA expritem",
- /* 210 */ "exprlist ::= expritem",
- /* 211 */ "expritem ::= expr",
- /* 212 */ "expritem ::=",
- /* 213 */ "cmd ::= RESET QUERY CACHE",
- /* 214 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
- /* 215 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
- /* 216 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
- /* 217 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
- /* 218 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
- /* 219 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
- /* 220 */ "cmd ::= KILL CONNECTION INTEGER",
- /* 221 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
- /* 222 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
+ /*  74 */ "quorum ::= QUORUM INTEGER",
+ /*  75 */ "days ::= DAYS INTEGER",
+ /*  76 */ "minrows ::= MINROWS INTEGER",
+ /*  77 */ "maxrows ::= MAXROWS INTEGER",
+ /*  78 */ "blocks ::= BLOCKS INTEGER",
+ /*  79 */ "ctime ::= CTIME INTEGER",
+ /*  80 */ "wal ::= WAL INTEGER",
+ /*  81 */ "fsync ::= FSYNC INTEGER",
+ /*  82 */ "comp ::= COMP INTEGER",
+ /*  83 */ "prec ::= PRECISION STRING",
+ /*  84 */ "db_optr ::=",
+ /*  85 */ "db_optr ::= db_optr tables",
+ /*  86 */ "db_optr ::= db_optr cache",
+ /*  87 */ "db_optr ::= db_optr replica",
+ /*  88 */ "db_optr ::= db_optr quorum",
+ /*  89 */ "db_optr ::= db_optr days",
+ /*  90 */ "db_optr ::= db_optr minrows",
+ /*  91 */ "db_optr ::= db_optr maxrows",
+ /*  92 */ "db_optr ::= db_optr blocks",
+ /*  93 */ "db_optr ::= db_optr ctime",
+ /*  94 */ "db_optr ::= db_optr wal",
+ /*  95 */ "db_optr ::= db_optr fsync",
+ /*  96 */ "db_optr ::= db_optr comp",
+ /*  97 */ "db_optr ::= db_optr prec",
+ /*  98 */ "db_optr ::= db_optr keep",
+ /*  99 */ "alter_db_optr ::=",
+ /* 100 */ "alter_db_optr ::= alter_db_optr replica",
+ /* 101 */ "alter_db_optr ::= alter_db_optr quorum",
+ /* 102 */ "alter_db_optr ::= alter_db_optr tables",
+ /* 103 */ "alter_db_optr ::= alter_db_optr keep",
+ /* 104 */ "alter_db_optr ::= alter_db_optr blocks",
+ /* 105 */ "alter_db_optr ::= alter_db_optr comp",
+ /* 106 */ "alter_db_optr ::= alter_db_optr wal",
+ /* 107 */ "alter_db_optr ::= alter_db_optr fsync",
+ /* 108 */ "typename ::= ids",
+ /* 109 */ "typename ::= ids LP signed RP",
+ /* 110 */ "signed ::= INTEGER",
+ /* 111 */ "signed ::= PLUS INTEGER",
+ /* 112 */ "signed ::= MINUS INTEGER",
+ /* 113 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
+ /* 114 */ "create_table_args ::= LP columnlist RP",
+ /* 115 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
+ /* 116 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
+ /* 117 */ "create_table_args ::= AS select",
+ /* 118 */ "columnlist ::= columnlist COMMA column",
+ /* 119 */ "columnlist ::= column",
+ /* 120 */ "column ::= ids typename",
+ /* 121 */ "tagitemlist ::= tagitemlist COMMA tagitem",
+ /* 122 */ "tagitemlist ::= tagitem",
+ /* 123 */ "tagitem ::= INTEGER",
+ /* 124 */ "tagitem ::= FLOAT",
+ /* 125 */ "tagitem ::= STRING",
+ /* 126 */ "tagitem ::= BOOL",
+ /* 127 */ "tagitem ::= NULL",
+ /* 128 */ "tagitem ::= MINUS INTEGER",
+ /* 129 */ "tagitem ::= MINUS FLOAT",
+ /* 130 */ "tagitem ::= PLUS INTEGER",
+ /* 131 */ "tagitem ::= PLUS FLOAT",
+ /* 132 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
+ /* 133 */ "union ::= select",
+ /* 134 */ "union ::= LP union RP",
+ /* 135 */ "union ::= union UNION ALL select",
+ /* 136 */ "union ::= union UNION ALL LP select RP",
+ /* 137 */ "cmd ::= union",
+ /* 138 */ "select ::= SELECT selcollist",
+ /* 139 */ "sclp ::= selcollist COMMA",
+ /* 140 */ "sclp ::=",
+ /* 141 */ "selcollist ::= sclp expr as",
+ /* 142 */ "selcollist ::= sclp STAR",
+ /* 143 */ "as ::= AS ids",
+ /* 144 */ "as ::= ids",
+ /* 145 */ "as ::=",
+ /* 146 */ "from ::= FROM tablelist",
+ /* 147 */ "tablelist ::= ids cpxName",
+ /* 148 */ "tablelist ::= tablelist COMMA ids cpxName",
+ /* 149 */ "tmvar ::= VARIABLE",
+ /* 150 */ "interval_opt ::= INTERVAL LP tmvar RP",
+ /* 151 */ "interval_opt ::=",
+ /* 152 */ "fill_opt ::=",
+ /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
+ /* 154 */ "fill_opt ::= FILL LP ID RP",
+ /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP",
+ /* 156 */ "sliding_opt ::=",
+ /* 157 */ "orderby_opt ::=",
+ /* 158 */ "orderby_opt ::= ORDER BY sortlist",
+ /* 159 */ "sortlist ::= sortlist COMMA item sortorder",
+ /* 160 */ "sortlist ::= item sortorder",
+ /* 161 */ "item ::= ids cpxName",
+ /* 162 */ "sortorder ::= ASC",
+ /* 163 */ "sortorder ::= DESC",
+ /* 164 */ "sortorder ::=",
+ /* 165 */ "groupby_opt ::=",
+ /* 166 */ "groupby_opt ::= GROUP BY grouplist",
+ /* 167 */ "grouplist ::= grouplist COMMA item",
+ /* 168 */ "grouplist ::= item",
+ /* 169 */ "having_opt ::=",
+ /* 170 */ "having_opt ::= HAVING expr",
+ /* 171 */ "limit_opt ::=",
+ /* 172 */ "limit_opt ::= LIMIT signed",
+ /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed",
+ /* 174 */ "limit_opt ::= LIMIT signed COMMA signed",
+ /* 175 */ "slimit_opt ::=",
+ /* 176 */ "slimit_opt ::= SLIMIT signed",
+ /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
+ /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed",
+ /* 179 */ "where_opt ::=",
+ /* 180 */ "where_opt ::= WHERE expr",
+ /* 181 */ "expr ::= LP expr RP",
+ /* 182 */ "expr ::= ID",
+ /* 183 */ "expr ::= ID DOT ID",
+ /* 184 */ "expr ::= ID DOT STAR",
+ /* 185 */ "expr ::= INTEGER",
+ /* 186 */ "expr ::= MINUS INTEGER",
+ /* 187 */ "expr ::= PLUS INTEGER",
+ /* 188 */ "expr ::= FLOAT",
+ /* 189 */ "expr ::= MINUS FLOAT",
+ /* 190 */ "expr ::= PLUS FLOAT",
+ /* 191 */ "expr ::= STRING",
+ /* 192 */ "expr ::= NOW",
+ /* 193 */ "expr ::= VARIABLE",
+ /* 194 */ "expr ::= BOOL",
+ /* 195 */ "expr ::= ID LP exprlist RP",
+ /* 196 */ "expr ::= ID LP STAR RP",
+ /* 197 */ "expr ::= expr AND expr",
+ /* 198 */ "expr ::= expr OR expr",
+ /* 199 */ "expr ::= expr LT expr",
+ /* 200 */ "expr ::= expr GT expr",
+ /* 201 */ "expr ::= expr LE expr",
+ /* 202 */ "expr ::= expr GE expr",
+ /* 203 */ "expr ::= expr NE expr",
+ /* 204 */ "expr ::= expr EQ expr",
+ /* 205 */ "expr ::= expr PLUS expr",
+ /* 206 */ "expr ::= expr MINUS expr",
+ /* 207 */ "expr ::= expr STAR expr",
+ /* 208 */ "expr ::= expr SLASH expr",
+ /* 209 */ "expr ::= expr REM expr",
+ /* 210 */ "expr ::= expr LIKE expr",
+ /* 211 */ "expr ::= expr IN LP exprlist RP",
+ /* 212 */ "exprlist ::= exprlist COMMA expritem",
+ /* 213 */ "exprlist ::= expritem",
+ /* 214 */ "expritem ::= expr",
+ /* 215 */ "expritem ::=",
+ /* 216 */ "cmd ::= RESET QUERY CACHE",
+ /* 217 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
+ /* 218 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
+ /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
+ /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
+ /* 221 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
+ /* 222 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
+ /* 223 */ "cmd ::= KILL CONNECTION INTEGER",
+ /* 224 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
+ /* 225 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
 };
 #endif /* NDEBUG */
 
@@ -1345,50 +1352,50 @@ static void yy_destructor(
     ** inside the C code.
     */
 /********* Begin destructor definitions ***************************************/
-    case 226: /* keep */
-    case 227: /* tagitemlist */
-    case 251: /* fill_opt */
-    case 253: /* groupby_opt */
-    case 254: /* orderby_opt */
-    case 264: /* sortlist */
-    case 268: /* grouplist */
+    case 227: /* keep */
+    case 228: /* tagitemlist */
+    case 253: /* fill_opt */
+    case 255: /* groupby_opt */
+    case 256: /* orderby_opt */
+    case 266: /* sortlist */
+    case 270: /* grouplist */
 {
-tVariantListDestroy((yypminor->yy494));
+tVariantListDestroy((yypminor->yy498));
 }
       break;
-    case 243: /* columnlist */
+    case 245: /* columnlist */
 {
-tFieldListDestroy((yypminor->yy449));
+tFieldListDestroy((yypminor->yy523));
 }
       break;
-    case 244: /* select */
+    case 246: /* select */
 {
-doDestroyQuerySql((yypminor->yy150));
+doDestroyQuerySql((yypminor->yy414));
 }
       break;
-    case 247: /* selcollist */
-    case 259: /* sclp */
-    case 269: /* exprlist */
+    case 249: /* selcollist */
+    case 261: /* sclp */
+    case 271: /* exprlist */
 {
-tSQLExprListDestroy((yypminor->yy224));
+tSQLExprListDestroy((yypminor->yy290));
 }
       break;
-    case 249: /* where_opt */
-    case 255: /* having_opt */
-    case 260: /* expr */
-    case 270: /* expritem */
+    case 251: /* where_opt */
+    case 257: /* having_opt */
+    case 262: /* expr */
+    case 272: /* expritem */
 {
-tSQLExprDestroy((yypminor->yy66));
+tSQLExprDestroy((yypminor->yy64));
 }
       break;
-    case 258: /* union */
+    case 260: /* union */
 {
-destroyAllSelectClause((yypminor->yy25));
+destroyAllSelectClause((yypminor->yy231));
 }
       break;
-    case 265: /* sortitem */
+    case 267: /* sortitem */
 {
-tVariantDestroy(&(yypminor->yy312));
+tVariantDestroy(&(yypminor->yy134));
 }
       break;
 /********* End destructor definitions *****************************************/
@@ -1682,229 +1689,232 @@ static const struct {
   YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
 } yyRuleInfo[] = {
-  {  207,   -1 }, /* (0) program ::= cmd */
-  {  208,   -2 }, /* (1) cmd ::= SHOW DATABASES */
-  {  208,   -2 }, /* (2) cmd ::= SHOW MNODES */
-  {  208,   -2 }, /* (3) cmd ::= SHOW DNODES */
-  {  208,   -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
-  {  208,   -2 }, /* (5) cmd ::= SHOW USERS */
-  {  208,   -2 }, /* (6) cmd ::= SHOW MODULES */
-  {  208,   -2 }, /* (7) cmd ::= SHOW QUERIES */
-  {  208,   -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
-  {  208,   -2 }, /* (9) cmd ::= SHOW STREAMS */
-  {  208,   -2 }, /* (10) cmd ::= SHOW CONFIGS */
-  {  208,   -2 }, /* (11) cmd ::= SHOW SCORES */
-  {  208,   -2 }, /* (12) cmd ::= SHOW GRANTS */
-  {  208,   -2 }, /* (13) cmd ::= SHOW VNODES */
-  {  208,   -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
-  {  209,    0 }, /* (15) dbPrefix ::= */
-  {  209,   -2 }, /* (16) dbPrefix ::= ids DOT */
-  {  211,    0 }, /* (17) cpxName ::= */
-  {  211,   -2 }, /* (18) cpxName ::= DOT ids */
-  {  208,   -3 }, /* (19) cmd ::= SHOW dbPrefix TABLES */
-  {  208,   -5 }, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */
-  {  208,   -3 }, /* (21) cmd ::= SHOW dbPrefix STABLES */
-  {  208,   -5 }, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */
-  {  208,   -3 }, /* (23) cmd ::= SHOW dbPrefix VGROUPS */
-  {  208,   -4 }, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */
-  {  208,   -5 }, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */
-  {  208,   -4 }, /* (26) cmd ::= DROP DATABASE ifexists ids */
-  {  208,   -3 }, /* (27) cmd ::= DROP DNODE ids */
-  {  208,   -3 }, /* (28) cmd ::= DROP USER ids */
-  {  208,   -3 }, /* (29) cmd ::= DROP ACCOUNT ids */
-  {  208,   -2 }, /* (30) cmd ::= USE ids */
-  {  208,   -3 }, /* (31) cmd ::= DESCRIBE ids cpxName */
-  {  208,   -5 }, /* (32) cmd ::= ALTER USER ids PASS ids */
-  {  208,   -5 }, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */
-  {  208,   -4 }, /* (34) cmd ::= ALTER DNODE ids ids */
-  {  208,   -5 }, /* (35) cmd ::= ALTER DNODE ids ids ids */
-  {  208,   -3 }, /* (36) cmd ::= ALTER LOCAL ids */
-  {  208,   -4 }, /* (37) cmd ::= ALTER LOCAL ids ids */
-  {  208,   -4 }, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */
-  {  208,   -4 }, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */
-  {  208,   -6 }, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-  {  210,   -1 }, /* (41) ids ::= ID */
-  {  210,   -1 }, /* (42) ids ::= STRING */
-  {  212,   -2 }, /* (43) ifexists ::= IF EXISTS */
-  {  212,    0 }, /* (44) ifexists ::= */
-  {  215,   -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */
-  {  215,    0 }, /* (46) ifnotexists ::= */
-  {  208,   -3 }, /* (47) cmd ::= CREATE DNODE ids */
-  {  208,   -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-  {  208,   -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-  {  208,   -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */
-  {  217,    0 }, /* (51) pps ::= */
-  {  217,   -2 }, /* (52) pps ::= PPS INTEGER */
-  {  218,    0 }, /* (53) tseries ::= */
-  {  218,   -2 }, /* (54) tseries ::= TSERIES INTEGER */
-  {  219,    0 }, /* (55) dbs ::= */
-  {  219,   -2 }, /* (56) dbs ::= DBS INTEGER */
-  {  220,    0 }, /* (57) streams ::= */
-  {  220,   -2 }, /* (58) streams ::= STREAMS INTEGER */
-  {  221,    0 }, /* (59) storage ::= */
-  {  221,   -2 }, /* (60) storage ::= STORAGE INTEGER */
-  {  222,    0 }, /* (61) qtime ::= */
-  {  222,   -2 }, /* (62) qtime ::= QTIME INTEGER */
-  {  223,    0 }, /* (63) users ::= */
-  {  223,   -2 }, /* (64) users ::= USERS INTEGER */
-  {  224,    0 }, /* (65) conns ::= */
-  {  224,   -2 }, /* (66) conns ::= CONNS INTEGER */
-  {  225,    0 }, /* (67) state ::= */
-  {  225,   -2 }, /* (68) state ::= STATE ids */
-  {  214,   -9 }, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
-  {  226,   -2 }, /* (70) keep ::= KEEP tagitemlist */
-  {  228,   -2 }, /* (71) tables ::= MAXTABLES INTEGER */
-  {  229,   -2 }, /* (72) cache ::= CACHE INTEGER */
-  {  230,   -2 }, /* (73) replica ::= REPLICA INTEGER */
-  {  231,   -2 }, /* (74) days ::= DAYS INTEGER */
-  {  232,   -2 }, /* (75) minrows ::= MINROWS INTEGER */
-  {  233,   -2 }, /* (76) maxrows ::= MAXROWS INTEGER */
-  {  234,   -2 }, /* (77) blocks ::= BLOCKS INTEGER */
-  {  235,   -2 }, /* (78) ctime ::= CTIME INTEGER */
-  {  236,   -2 }, /* (79) wal ::= WAL INTEGER */
-  {  237,   -2 }, /* (80) fsync ::= FSYNC INTEGER */
-  {  238,   -2 }, /* (81) comp ::= COMP INTEGER */
-  {  239,   -2 }, /* (82) prec ::= PRECISION STRING */
-  {  216,    0 }, /* (83) db_optr ::= */
-  {  216,   -2 }, /* (84) db_optr ::= db_optr tables */
-  {  216,   -2 }, /* (85) db_optr ::= db_optr cache */
-  {  216,   -2 }, /* (86) db_optr ::= db_optr replica */
-  {  216,   -2 }, /* (87) db_optr ::= db_optr days */
-  {  216,   -2 }, /* (88) db_optr ::= db_optr minrows */
-  {  216,   -2 }, /* (89) db_optr ::= db_optr maxrows */
-  {  216,   -2 }, /* (90) db_optr ::= db_optr blocks */
-  {  216,   -2 }, /* (91) db_optr ::= db_optr ctime */
-  {  216,   -2 }, /* (92) db_optr ::= db_optr wal */
-  {  216,   -2 }, /* (93) db_optr ::= db_optr fsync */
-  {  216,   -2 }, /* (94) db_optr ::= db_optr comp */
-  {  216,   -2 }, /* (95) db_optr ::= db_optr prec */
-  {  216,   -2 }, /* (96) db_optr ::= db_optr keep */
-  {  213,    0 }, /* (97) alter_db_optr ::= */
-  {  213,   -2 }, /* (98) alter_db_optr ::= alter_db_optr replica */
-  {  213,   -2 }, /* (99) alter_db_optr ::= alter_db_optr tables */
-  {  213,   -2 }, /* (100) alter_db_optr ::= alter_db_optr keep */
-  {  213,   -2 }, /* (101) alter_db_optr ::= alter_db_optr blocks */
-  {  213,   -2 }, /* (102) alter_db_optr ::= alter_db_optr comp */
-  {  213,   -2 }, /* (103) alter_db_optr ::= alter_db_optr wal */
-  {  213,   -2 }, /* (104) alter_db_optr ::= alter_db_optr fsync */
-  {  240,   -1 }, /* (105) typename ::= ids */
-  {  240,   -4 }, /* (106) typename ::= ids LP signed RP */
-  {  241,   -1 }, /* (107) signed ::= INTEGER */
-  {  241,   -2 }, /* (108) signed ::= PLUS INTEGER */
-  {  241,   -2 }, /* (109) signed ::= MINUS INTEGER */
-  {  208,   -6 }, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
-  {  242,   -3 }, /* (111) create_table_args ::= LP columnlist RP */
-  {  242,   -7 }, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
-  {  242,   -7 }, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
-  {  242,   -2 }, /* (114) create_table_args ::= AS select */
-  {  243,   -3 }, /* (115) columnlist ::= columnlist COMMA column */
-  {  243,   -1 }, /* (116) columnlist ::= column */
-  {  245,   -2 }, /* (117) column ::= ids typename */
-  {  227,   -3 }, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */
-  {  227,   -1 }, /* (119) tagitemlist ::= tagitem */
-  {  246,   -1 }, /* (120) tagitem ::= INTEGER */
-  {  246,   -1 }, /* (121) tagitem ::= FLOAT */
-  {  246,   -1 }, /* (122) tagitem ::= STRING */
-  {  246,   -1 }, /* (123) tagitem ::= BOOL */
-  {  246,   -1 }, /* (124) tagitem ::= NULL */
-  {  246,   -2 }, /* (125) tagitem ::= MINUS INTEGER */
-  {  246,   -2 }, /* (126) tagitem ::= MINUS FLOAT */
-  {  246,   -2 }, /* (127) tagitem ::= PLUS INTEGER */
-  {  246,   -2 }, /* (128) tagitem ::= PLUS FLOAT */
-  {  244,  -12 }, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
-  {  258,   -1 }, /* (130) union ::= select */
-  {  258,   -3 }, /* (131) union ::= LP union RP */
-  {  258,   -4 }, /* (132) union ::= union UNION ALL select */
-  {  258,   -6 }, /* (133) union ::= union UNION ALL LP select RP */
-  {  208,   -1 }, /* (134) cmd ::= union */
-  {  244,   -2 }, /* (135) select ::= SELECT selcollist */
-  {  259,   -2 }, /* (136) sclp ::= selcollist COMMA */
-  {  259,    0 }, /* (137) sclp ::= */
-  {  247,   -3 }, /* (138) selcollist ::= sclp expr as */
-  {  247,   -2 }, /* (139) selcollist ::= sclp STAR */
-  {  261,   -2 }, /* (140) as ::= AS ids */
-  {  261,   -1 }, /* (141) as ::= ids */
-  {  261,    0 }, /* (142) as ::= */
-  {  248,   -2 }, /* (143) from ::= FROM tablelist */
-  {  262,   -2 }, /* (144) tablelist ::= ids cpxName */
-  {  262,   -4 }, /* (145) tablelist ::= tablelist COMMA ids cpxName */
-  {  263,   -1 }, /* (146) tmvar ::= VARIABLE */
-  {  250,   -4 }, /* (147) interval_opt ::= INTERVAL LP tmvar RP */
-  {  250,    0 }, /* (148) interval_opt ::= */
-  {  251,    0 }, /* (149) fill_opt ::= */
-  {  251,   -6 }, /* (150) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
-  {  251,   -4 }, /* (151) fill_opt ::= FILL LP ID RP */
-  {  252,   -4 }, /* (152) sliding_opt ::= SLIDING LP tmvar RP */
-  {  252,    0 }, /* (153) sliding_opt ::= */
-  {  254,    0 }, /* (154) orderby_opt ::= */
-  {  254,   -3 }, /* (155) orderby_opt ::= ORDER BY sortlist */
-  {  264,   -4 }, /* (156) sortlist ::= sortlist COMMA item sortorder */
-  {  264,   -2 }, /* (157) sortlist ::= item sortorder */
-  {  266,   -2 }, /* (158) item ::= ids cpxName */
-  {  267,   -1 }, /* (159) sortorder ::= ASC */
-  {  267,   -1 }, /* (160) sortorder ::= DESC */
-  {  267,    0 }, /* (161) sortorder ::= */
-  {  253,    0 }, /* (162) groupby_opt ::= */
-  {  253,   -3 }, /* (163) groupby_opt ::= GROUP BY grouplist */
-  {  268,   -3 }, /* (164) grouplist ::= grouplist COMMA item */
-  {  268,   -1 }, /* (165) grouplist ::= item */
-  {  255,    0 }, /* (166) having_opt ::= */
-  {  255,   -2 }, /* (167) having_opt ::= HAVING expr */
-  {  257,    0 }, /* (168) limit_opt ::= */
-  {  257,   -2 }, /* (169) limit_opt ::= LIMIT signed */
-  {  257,   -4 }, /* (170) limit_opt ::= LIMIT signed OFFSET signed */
-  {  257,   -4 }, /* (171) limit_opt ::= LIMIT signed COMMA signed */
-  {  256,    0 }, /* (172) slimit_opt ::= */
-  {  256,   -2 }, /* (173) slimit_opt ::= SLIMIT signed */
-  {  256,   -4 }, /* (174) slimit_opt ::= SLIMIT signed SOFFSET signed */
-  {  256,   -4 }, /* (175) slimit_opt ::= SLIMIT signed COMMA signed */
-  {  249,    0 }, /* (176) where_opt ::= */
-  {  249,   -2 }, /* (177) where_opt ::= WHERE expr */
-  {  260,   -3 }, /* (178) expr ::= LP expr RP */
-  {  260,   -1 }, /* (179) expr ::= ID */
-  {  260,   -3 }, /* (180) expr ::= ID DOT ID */
-  {  260,   -3 }, /* (181) expr ::= ID DOT STAR */
-  {  260,   -1 }, /* (182) expr ::= INTEGER */
-  {  260,   -2 }, /* (183) expr ::= MINUS INTEGER */
-  {  260,   -2 }, /* (184) expr ::= PLUS INTEGER */
-  {  260,   -1 }, /* (185) expr ::= FLOAT */
-  {  260,   -2 }, /* (186) expr ::= MINUS FLOAT */
-  {  260,   -2 }, /* (187) expr ::= PLUS FLOAT */
-  {  260,   -1 }, /* (188) expr ::= STRING */
-  {  260,   -1 }, /* (189) expr ::= NOW */
-  {  260,   -1 }, /* (190) expr ::= VARIABLE */
-  {  260,   -1 }, /* (191) expr ::= BOOL */
-  {  260,   -4 }, /* (192) expr ::= ID LP exprlist RP */
-  {  260,   -4 }, /* (193) expr ::= ID LP STAR RP */
-  {  260,   -3 }, /* (194) expr ::= expr AND expr */
-  {  260,   -3 }, /* (195) expr ::= expr OR expr */
-  {  260,   -3 }, /* (196) expr ::= expr LT expr */
-  {  260,   -3 }, /* (197) expr ::= expr GT expr */
-  {  260,   -3 }, /* (198) expr ::= expr LE expr */
-  {  260,   -3 }, /* (199) expr ::= expr GE expr */
-  {  260,   -3 }, /* (200) expr ::= expr NE expr */
-  {  260,   -3 }, /* (201) expr ::= expr EQ expr */
-  {  260,   -3 }, /* (202) expr ::= expr PLUS expr */
-  {  260,   -3 }, /* (203) expr ::= expr MINUS expr */
-  {  260,   -3 }, /* (204) expr ::= expr STAR expr */
-  {  260,   -3 }, /* (205) expr ::= expr SLASH expr */
-  {  260,   -3 }, /* (206) expr ::= expr REM expr */
-  {  260,   -3 }, /* (207) expr ::= expr LIKE expr */
-  {  260,   -5 }, /* (208) expr ::= expr IN LP exprlist RP */
-  {  269,   -3 }, /* (209) exprlist ::= exprlist COMMA expritem */
-  {  269,   -1 }, /* (210) exprlist ::= expritem */
-  {  270,   -1 }, /* (211) expritem ::= expr */
-  {  270,    0 }, /* (212) expritem ::= */
-  {  208,   -3 }, /* (213) cmd ::= RESET QUERY CACHE */
-  {  208,   -7 }, /* (214) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
-  {  208,   -7 }, /* (215) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
-  {  208,   -7 }, /* (216) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
-  {  208,   -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
-  {  208,   -8 }, /* (218) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
-  {  208,   -9 }, /* (219) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
-  {  208,   -3 }, /* (220) cmd ::= KILL CONNECTION INTEGER */
-  {  208,   -5 }, /* (221) cmd ::= KILL STREAM INTEGER COLON INTEGER */
-  {  208,   -5 }, /* (222) cmd ::= KILL QUERY INTEGER COLON INTEGER */
+  {  208,   -1 }, /* (0) program ::= cmd */
+  {  209,   -2 }, /* (1) cmd ::= SHOW DATABASES */
+  {  209,   -2 }, /* (2) cmd ::= SHOW MNODES */
+  {  209,   -2 }, /* (3) cmd ::= SHOW DNODES */
+  {  209,   -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
+  {  209,   -2 }, /* (5) cmd ::= SHOW USERS */
+  {  209,   -2 }, /* (6) cmd ::= SHOW MODULES */
+  {  209,   -2 }, /* (7) cmd ::= SHOW QUERIES */
+  {  209,   -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
+  {  209,   -2 }, /* (9) cmd ::= SHOW STREAMS */
+  {  209,   -2 }, /* (10) cmd ::= SHOW CONFIGS */
+  {  209,   -2 }, /* (11) cmd ::= SHOW SCORES */
+  {  209,   -2 }, /* (12) cmd ::= SHOW GRANTS */
+  {  209,   -2 }, /* (13) cmd ::= SHOW VNODES */
+  {  209,   -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
+  {  210,    0 }, /* (15) dbPrefix ::= */
+  {  210,   -2 }, /* (16) dbPrefix ::= ids DOT */
+  {  212,    0 }, /* (17) cpxName ::= */
+  {  212,   -2 }, /* (18) cpxName ::= DOT ids */
+  {  209,   -3 }, /* (19) cmd ::= SHOW dbPrefix TABLES */
+  {  209,   -5 }, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */
+  {  209,   -3 }, /* (21) cmd ::= SHOW dbPrefix STABLES */
+  {  209,   -5 }, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */
+  {  209,   -3 }, /* (23) cmd ::= SHOW dbPrefix VGROUPS */
+  {  209,   -4 }, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */
+  {  209,   -5 }, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */
+  {  209,   -4 }, /* (26) cmd ::= DROP DATABASE ifexists ids */
+  {  209,   -3 }, /* (27) cmd ::= DROP DNODE ids */
+  {  209,   -3 }, /* (28) cmd ::= DROP USER ids */
+  {  209,   -3 }, /* (29) cmd ::= DROP ACCOUNT ids */
+  {  209,   -2 }, /* (30) cmd ::= USE ids */
+  {  209,   -3 }, /* (31) cmd ::= DESCRIBE ids cpxName */
+  {  209,   -5 }, /* (32) cmd ::= ALTER USER ids PASS ids */
+  {  209,   -5 }, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */
+  {  209,   -4 }, /* (34) cmd ::= ALTER DNODE ids ids */
+  {  209,   -5 }, /* (35) cmd ::= ALTER DNODE ids ids ids */
+  {  209,   -3 }, /* (36) cmd ::= ALTER LOCAL ids */
+  {  209,   -4 }, /* (37) cmd ::= ALTER LOCAL ids ids */
+  {  209,   -4 }, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */
+  {  209,   -4 }, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */
+  {  209,   -6 }, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
+  {  211,   -1 }, /* (41) ids ::= ID */
+  {  211,   -1 }, /* (42) ids ::= STRING */
+  {  213,   -2 }, /* (43) ifexists ::= IF EXISTS */
+  {  213,    0 }, /* (44) ifexists ::= */
+  {  216,   -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */
+  {  216,    0 }, /* (46) ifnotexists ::= */
+  {  209,   -3 }, /* (47) cmd ::= CREATE DNODE ids */
+  {  209,   -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
+  {  209,   -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
+  {  209,   -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */
+  {  218,    0 }, /* (51) pps ::= */
+  {  218,   -2 }, /* (52) pps ::= PPS INTEGER */
+  {  219,    0 }, /* (53) tseries ::= */
+  {  219,   -2 }, /* (54) tseries ::= TSERIES INTEGER */
+  {  220,    0 }, /* (55) dbs ::= */
+  {  220,   -2 }, /* (56) dbs ::= DBS INTEGER */
+  {  221,    0 }, /* (57) streams ::= */
+  {  221,   -2 }, /* (58) streams ::= STREAMS INTEGER */
+  {  222,    0 }, /* (59) storage ::= */
+  {  222,   -2 }, /* (60) storage ::= STORAGE INTEGER */
+  {  223,    0 }, /* (61) qtime ::= */
+  {  223,   -2 }, /* (62) qtime ::= QTIME INTEGER */
+  {  224,    0 }, /* (63) users ::= */
+  {  224,   -2 }, /* (64) users ::= USERS INTEGER */
+  {  225,    0 }, /* (65) conns ::= */
+  {  225,   -2 }, /* (66) conns ::= CONNS INTEGER */
+  {  226,    0 }, /* (67) state ::= */
+  {  226,   -2 }, /* (68) state ::= STATE ids */
+  {  215,   -9 }, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
+  {  227,   -2 }, /* (70) keep ::= KEEP tagitemlist */
+  {  229,   -2 }, /* (71) tables ::= MAXTABLES INTEGER */
+  {  230,   -2 }, /* (72) cache ::= CACHE INTEGER */
+  {  231,   -2 }, /* (73) replica ::= REPLICA INTEGER */
+  {  232,   -2 }, /* (74) quorum ::= QUORUM INTEGER */
+  {  233,   -2 }, /* (75) days ::= DAYS INTEGER */
+  {  234,   -2 }, /* (76) minrows ::= MINROWS INTEGER */
+  {  235,   -2 }, /* (77) maxrows ::= MAXROWS INTEGER */
+  {  236,   -2 }, /* (78) blocks ::= BLOCKS INTEGER */
+  {  237,   -2 }, /* (79) ctime ::= CTIME INTEGER */
+  {  238,   -2 }, /* (80) wal ::= WAL INTEGER */
+  {  239,   -2 }, /* (81) fsync ::= FSYNC INTEGER */
+  {  240,   -2 }, /* (82) comp ::= COMP INTEGER */
+  {  241,   -2 }, /* (83) prec ::= PRECISION STRING */
+  {  217,    0 }, /* (84) db_optr ::= */
+  {  217,   -2 }, /* (85) db_optr ::= db_optr tables */
+  {  217,   -2 }, /* (86) db_optr ::= db_optr cache */
+  {  217,   -2 }, /* (87) db_optr ::= db_optr replica */
+  {  217,   -2 }, /* (88) db_optr ::= db_optr quorum */
+  {  217,   -2 }, /* (89) db_optr ::= db_optr days */
+  {  217,   -2 }, /* (90) db_optr ::= db_optr minrows */
+  {  217,   -2 }, /* (91) db_optr ::= db_optr maxrows */
+  {  217,   -2 }, /* (92) db_optr ::= db_optr blocks */
+  {  217,   -2 }, /* (93) db_optr ::= db_optr ctime */
+  {  217,   -2 }, /* (94) db_optr ::= db_optr wal */
+  {  217,   -2 }, /* (95) db_optr ::= db_optr fsync */
+  {  217,   -2 }, /* (96) db_optr ::= db_optr comp */
+  {  217,   -2 }, /* (97) db_optr ::= db_optr prec */
+  {  217,   -2 }, /* (98) db_optr ::= db_optr keep */
+  {  214,    0 }, /* (99) alter_db_optr ::= */
+  {  214,   -2 }, /* (100) alter_db_optr ::= alter_db_optr replica */
+  {  214,   -2 }, /* (101) alter_db_optr ::= alter_db_optr quorum */
+  {  214,   -2 }, /* (102) alter_db_optr ::= alter_db_optr tables */
+  {  214,   -2 }, /* (103) alter_db_optr ::= alter_db_optr keep */
+  {  214,   -2 }, /* (104) alter_db_optr ::= alter_db_optr blocks */
+  {  214,   -2 }, /* (105) alter_db_optr ::= alter_db_optr comp */
+  {  214,   -2 }, /* (106) alter_db_optr ::= alter_db_optr wal */
+  {  214,   -2 }, /* (107) alter_db_optr ::= alter_db_optr fsync */
+  {  242,   -1 }, /* (108) typename ::= ids */
+  {  242,   -4 }, /* (109) typename ::= ids LP signed RP */
+  {  243,   -1 }, /* (110) signed ::= INTEGER */
+  {  243,   -2 }, /* (111) signed ::= PLUS INTEGER */
+  {  243,   -2 }, /* (112) signed ::= MINUS INTEGER */
+  {  209,   -6 }, /* (113) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+  {  244,   -3 }, /* (114) create_table_args ::= LP columnlist RP */
+  {  244,   -7 }, /* (115) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+  {  244,   -7 }, /* (116) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+  {  244,   -2 }, /* (117) create_table_args ::= AS select */
+  {  245,   -3 }, /* (118) columnlist ::= columnlist COMMA column */
+  {  245,   -1 }, /* (119) columnlist ::= column */
+  {  247,   -2 }, /* (120) column ::= ids typename */
+  {  228,   -3 }, /* (121) tagitemlist ::= tagitemlist COMMA tagitem */
+  {  228,   -1 }, /* (122) tagitemlist ::= tagitem */
+  {  248,   -1 }, /* (123) tagitem ::= INTEGER */
+  {  248,   -1 }, /* (124) tagitem ::= FLOAT */
+  {  248,   -1 }, /* (125) tagitem ::= STRING */
+  {  248,   -1 }, /* (126) tagitem ::= BOOL */
+  {  248,   -1 }, /* (127) tagitem ::= NULL */
+  {  248,   -2 }, /* (128) tagitem ::= MINUS INTEGER */
+  {  248,   -2 }, /* (129) tagitem ::= MINUS FLOAT */
+  {  248,   -2 }, /* (130) tagitem ::= PLUS INTEGER */
+  {  248,   -2 }, /* (131) tagitem ::= PLUS FLOAT */
+  {  246,  -12 }, /* (132) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+  {  260,   -1 }, /* (133) union ::= select */
+  {  260,   -3 }, /* (134) union ::= LP union RP */
+  {  260,   -4 }, /* (135) union ::= union UNION ALL select */
+  {  260,   -6 }, /* (136) union ::= union UNION ALL LP select RP */
+  {  209,   -1 }, /* (137) cmd ::= union */
+  {  246,   -2 }, /* (138) select ::= SELECT selcollist */
+  {  261,   -2 }, /* (139) sclp ::= selcollist COMMA */
+  {  261,    0 }, /* (140) sclp ::= */
+  {  249,   -3 }, /* (141) selcollist ::= sclp expr as */
+  {  249,   -2 }, /* (142) selcollist ::= sclp STAR */
+  {  263,   -2 }, /* (143) as ::= AS ids */
+  {  263,   -1 }, /* (144) as ::= ids */
+  {  263,    0 }, /* (145) as ::= */
+  {  250,   -2 }, /* (146) from ::= FROM tablelist */
+  {  264,   -2 }, /* (147) tablelist ::= ids cpxName */
+  {  264,   -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */
+  {  265,   -1 }, /* (149) tmvar ::= VARIABLE */
+  {  252,   -4 }, /* (150) interval_opt ::= INTERVAL LP tmvar RP */
+  {  252,    0 }, /* (151) interval_opt ::= */
+  {  253,    0 }, /* (152) fill_opt ::= */
+  {  253,   -6 }, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+  {  253,   -4 }, /* (154) fill_opt ::= FILL LP ID RP */
+  {  254,   -4 }, /* (155) sliding_opt ::= SLIDING LP tmvar RP */
+  {  254,    0 }, /* (156) sliding_opt ::= */
+  {  256,    0 }, /* (157) orderby_opt ::= */
+  {  256,   -3 }, /* (158) orderby_opt ::= ORDER BY sortlist */
+  {  266,   -4 }, /* (159) sortlist ::= sortlist COMMA item sortorder */
+  {  266,   -2 }, /* (160) sortlist ::= item sortorder */
+  {  268,   -2 }, /* (161) item ::= ids cpxName */
+  {  269,   -1 }, /* (162) sortorder ::= ASC */
+  {  269,   -1 }, /* (163) sortorder ::= DESC */
+  {  269,    0 }, /* (164) sortorder ::= */
+  {  255,    0 }, /* (165) groupby_opt ::= */
+  {  255,   -3 }, /* (166) groupby_opt ::= GROUP BY grouplist */
+  {  270,   -3 }, /* (167) grouplist ::= grouplist COMMA item */
+  {  270,   -1 }, /* (168) grouplist ::= item */
+  {  257,    0 }, /* (169) having_opt ::= */
+  {  257,   -2 }, /* (170) having_opt ::= HAVING expr */
+  {  259,    0 }, /* (171) limit_opt ::= */
+  {  259,   -2 }, /* (172) limit_opt ::= LIMIT signed */
+  {  259,   -4 }, /* (173) limit_opt ::= LIMIT signed OFFSET signed */
+  {  259,   -4 }, /* (174) limit_opt ::= LIMIT signed COMMA signed */
+  {  258,    0 }, /* (175) slimit_opt ::= */
+  {  258,   -2 }, /* (176) slimit_opt ::= SLIMIT signed */
+  {  258,   -4 }, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */
+  {  258,   -4 }, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */
+  {  251,    0 }, /* (179) where_opt ::= */
+  {  251,   -2 }, /* (180) where_opt ::= WHERE expr */
+  {  262,   -3 }, /* (181) expr ::= LP expr RP */
+  {  262,   -1 }, /* (182) expr ::= ID */
+  {  262,   -3 }, /* (183) expr ::= ID DOT ID */
+  {  262,   -3 }, /* (184) expr ::= ID DOT STAR */
+  {  262,   -1 }, /* (185) expr ::= INTEGER */
+  {  262,   -2 }, /* (186) expr ::= MINUS INTEGER */
+  {  262,   -2 }, /* (187) expr ::= PLUS INTEGER */
+  {  262,   -1 }, /* (188) expr ::= FLOAT */
+  {  262,   -2 }, /* (189) expr ::= MINUS FLOAT */
+  {  262,   -2 }, /* (190) expr ::= PLUS FLOAT */
+  {  262,   -1 }, /* (191) expr ::= STRING */
+  {  262,   -1 }, /* (192) expr ::= NOW */
+  {  262,   -1 }, /* (193) expr ::= VARIABLE */
+  {  262,   -1 }, /* (194) expr ::= BOOL */
+  {  262,   -4 }, /* (195) expr ::= ID LP exprlist RP */
+  {  262,   -4 }, /* (196) expr ::= ID LP STAR RP */
+  {  262,   -3 }, /* (197) expr ::= expr AND expr */
+  {  262,   -3 }, /* (198) expr ::= expr OR expr */
+  {  262,   -3 }, /* (199) expr ::= expr LT expr */
+  {  262,   -3 }, /* (200) expr ::= expr GT expr */
+  {  262,   -3 }, /* (201) expr ::= expr LE expr */
+  {  262,   -3 }, /* (202) expr ::= expr GE expr */
+  {  262,   -3 }, /* (203) expr ::= expr NE expr */
+  {  262,   -3 }, /* (204) expr ::= expr EQ expr */
+  {  262,   -3 }, /* (205) expr ::= expr PLUS expr */
+  {  262,   -3 }, /* (206) expr ::= expr MINUS expr */
+  {  262,   -3 }, /* (207) expr ::= expr STAR expr */
+  {  262,   -3 }, /* (208) expr ::= expr SLASH expr */
+  {  262,   -3 }, /* (209) expr ::= expr REM expr */
+  {  262,   -3 }, /* (210) expr ::= expr LIKE expr */
+  {  262,   -5 }, /* (211) expr ::= expr IN LP exprlist RP */
+  {  271,   -3 }, /* (212) exprlist ::= exprlist COMMA expritem */
+  {  271,   -1 }, /* (213) exprlist ::= expritem */
+  {  272,   -1 }, /* (214) expritem ::= expr */
+  {  272,    0 }, /* (215) expritem ::= */
+  {  209,   -3 }, /* (216) cmd ::= RESET QUERY CACHE */
+  {  209,   -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+  {  209,   -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+  {  209,   -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+  {  209,   -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+  {  209,   -8 }, /* (221) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+  {  209,   -9 }, /* (222) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+  {  209,   -3 }, /* (223) cmd ::= KILL CONNECTION INTEGER */
+  {  209,   -5 }, /* (224) cmd ::= KILL STREAM INTEGER COLON INTEGER */
+  {  209,   -5 }, /* (225) cmd ::= KILL QUERY INTEGER COLON INTEGER */
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
@@ -2124,13 +2134,13 @@ static void yy_reduce(
 { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);          }
         break;
       case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */
-{ SSQLToken t = {0};  setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);}
+{ SSQLToken t = {0};  setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);}
         break;
       case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);}
         break;
       case 40: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
         break;
       case 41: /* ids ::= ID */
       case 42: /* ids ::= STRING */ yytestcase(yyruleno==42);
@@ -2151,10 +2161,10 @@ static void yy_reduce(
 { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);}
         break;
       case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
         break;
       case 49: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);}
+{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);}
         break;
       case 50: /* cmd ::= CREATE USER ids PASS ids */
 { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);}
@@ -2183,529 +2193,535 @@ static void yy_reduce(
         break;
       case 69: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */
 {
-    yylhsminor.yy73.maxUsers   = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
-    yylhsminor.yy73.maxDbs     = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
-    yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
-    yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
-    yylhsminor.yy73.maxPointsPerSecond     = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
-    yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
-    yylhsminor.yy73.maxQueryTime   = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
-    yylhsminor.yy73.maxConnections   = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
-    yylhsminor.yy73.stat    = yymsp[0].minor.yy0;
-}
-  yymsp[-8].minor.yy73 = yylhsminor.yy73;
+    yylhsminor.yy149.maxUsers   = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
+    yylhsminor.yy149.maxDbs     = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
+    yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
+    yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
+    yylhsminor.yy149.maxPointsPerSecond     = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
+    yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
+    yylhsminor.yy149.maxQueryTime   = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
+    yylhsminor.yy149.maxConnections   = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
+    yylhsminor.yy149.stat    = yymsp[0].minor.yy0;
+}
+  yymsp[-8].minor.yy149 = yylhsminor.yy149;
         break;
       case 70: /* keep ::= KEEP tagitemlist */
-{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; }
+{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; }
         break;
       case 71: /* tables ::= MAXTABLES INTEGER */
       case 72: /* cache ::= CACHE INTEGER */ yytestcase(yyruleno==72);
       case 73: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==73);
-      case 74: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==74);
-      case 75: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==75);
-      case 76: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==76);
-      case 77: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==77);
-      case 78: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==78);
-      case 79: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==79);
-      case 80: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==80);
-      case 81: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==81);
-      case 82: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==82);
+      case 74: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==74);
+      case 75: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==75);
+      case 76: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==76);
+      case 77: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==77);
+      case 78: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==78);
+      case 79: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==79);
+      case 80: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==80);
+      case 81: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==81);
+      case 82: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==82);
+      case 83: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==83);
 { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
         break;
-      case 83: /* db_optr ::= */
-{setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
+      case 84: /* db_optr ::= */
+{setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
         break;
-      case 84: /* db_optr ::= db_optr tables */
-      case 99: /* alter_db_optr ::= alter_db_optr tables */ yytestcase(yyruleno==99);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxTablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 85: /* db_optr ::= db_optr tables */
+      case 102: /* alter_db_optr ::= alter_db_optr tables */ yytestcase(yyruleno==102);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxTablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 85: /* db_optr ::= db_optr cache */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 86: /* db_optr ::= db_optr cache */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 86: /* db_optr ::= db_optr replica */
-      case 98: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==98);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 87: /* db_optr ::= db_optr replica */
+      case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 87: /* db_optr ::= db_optr days */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 88: /* db_optr ::= db_optr quorum */
+      case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 88: /* db_optr ::= db_optr minrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 89: /* db_optr ::= db_optr days */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 89: /* db_optr ::= db_optr maxrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 90: /* db_optr ::= db_optr minrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 90: /* db_optr ::= db_optr blocks */
-      case 101: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==101);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 91: /* db_optr ::= db_optr maxrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 91: /* db_optr ::= db_optr ctime */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 92: /* db_optr ::= db_optr blocks */
+      case 104: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==104);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 92: /* db_optr ::= db_optr wal */
-      case 103: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==103);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 93: /* db_optr ::= db_optr ctime */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 93: /* db_optr ::= db_optr fsync */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 94: /* db_optr ::= db_optr wal */
+      case 106: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==106);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 94: /* db_optr ::= db_optr comp */
-      case 102: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==102);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 95: /* db_optr ::= db_optr fsync */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 95: /* db_optr ::= db_optr prec */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 96: /* db_optr ::= db_optr comp */
+      case 105: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==105);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 96: /* db_optr ::= db_optr keep */
-      case 100: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==100);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 97: /* db_optr ::= db_optr prec */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 97: /* alter_db_optr ::= */
-{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
+      case 98: /* db_optr ::= db_optr keep */
+      case 103: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==103);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 104: /* alter_db_optr ::= alter_db_optr fsync */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 99: /* alter_db_optr ::= */
+{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
         break;
-      case 105: /* typename ::= ids */
+      case 107: /* alter_db_optr ::= alter_db_optr fsync */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
+        break;
+      case 108: /* typename ::= ids */
 { 
   yymsp[0].minor.yy0.type = 0;
-  tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0); 
+  tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); 
 }
-  yymsp[0].minor.yy181 = yylhsminor.yy181;
+  yymsp[0].minor.yy223 = yylhsminor.yy223;
         break;
-      case 106: /* typename ::= ids LP signed RP */
+      case 109: /* typename ::= ids LP signed RP */
 {
-    if (yymsp[-1].minor.yy271 <= 0) {
+    if (yymsp[-1].minor.yy207 <= 0) {
       yymsp[-3].minor.yy0.type = 0;
-      tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+      tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
     } else {
-      yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271;          // negative value of name length
-      tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+      yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207;          // negative value of name length
+      tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
     }
 }
-  yymsp[-3].minor.yy181 = yylhsminor.yy181;
+  yymsp[-3].minor.yy223 = yylhsminor.yy223;
         break;
-      case 107: /* signed ::= INTEGER */
-{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[0].minor.yy271 = yylhsminor.yy271;
+      case 110: /* signed ::= INTEGER */
+{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[0].minor.yy207 = yylhsminor.yy207;
         break;
-      case 108: /* signed ::= PLUS INTEGER */
-{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+      case 111: /* signed ::= PLUS INTEGER */
+{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
         break;
-      case 109: /* signed ::= MINUS INTEGER */
-{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
+      case 112: /* signed ::= MINUS INTEGER */
+{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
         break;
-      case 110: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+      case 113: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
 {
     yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
     setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0);
 }
         break;
-      case 111: /* create_table_args ::= LP columnlist RP */
+      case 114: /* create_table_args ::= LP columnlist RP */
 {
-    yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
-    setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
+    setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 112: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+      case 115: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
 {
-    yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE);
-    setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE);
+    setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 113: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+      case 116: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
-    yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
-    setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
+    setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 114: /* create_table_args ::= AS select */
+      case 117: /* create_table_args ::= AS select */
 {
-    yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM);
-    setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM);
+    setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 115: /* columnlist ::= columnlist COMMA column */
-{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181);   }
-  yymsp[-2].minor.yy449 = yylhsminor.yy449;
+      case 118: /* columnlist ::= columnlist COMMA column */
+{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223);   }
+  yymsp[-2].minor.yy523 = yylhsminor.yy523;
         break;
-      case 116: /* columnlist ::= column */
-{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);}
-  yymsp[0].minor.yy449 = yylhsminor.yy449;
+      case 119: /* columnlist ::= column */
+{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);}
+  yymsp[0].minor.yy523 = yylhsminor.yy523;
         break;
-      case 117: /* column ::= ids typename */
+      case 120: /* column ::= ids typename */
 {
-    tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181);
+    tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223);
 }
-  yymsp[-1].minor.yy181 = yylhsminor.yy181;
+  yymsp[-1].minor.yy223 = yylhsminor.yy223;
         break;
-      case 118: /* tagitemlist ::= tagitemlist COMMA tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);    }
-  yymsp[-2].minor.yy494 = yylhsminor.yy494;
+      case 121: /* tagitemlist ::= tagitemlist COMMA tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);    }
+  yymsp[-2].minor.yy498 = yylhsminor.yy498;
         break;
-      case 119: /* tagitemlist ::= tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); }
-  yymsp[0].minor.yy494 = yylhsminor.yy494;
+      case 122: /* tagitemlist ::= tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
+  yymsp[0].minor.yy498 = yylhsminor.yy498;
         break;
-      case 120: /* tagitem ::= INTEGER */
-      case 121: /* tagitem ::= FLOAT */ yytestcase(yyruleno==121);
-      case 122: /* tagitem ::= STRING */ yytestcase(yyruleno==122);
-      case 123: /* tagitem ::= BOOL */ yytestcase(yyruleno==123);
-{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
-  yymsp[0].minor.yy312 = yylhsminor.yy312;
+      case 123: /* tagitem ::= INTEGER */
+      case 124: /* tagitem ::= FLOAT */ yytestcase(yyruleno==124);
+      case 125: /* tagitem ::= STRING */ yytestcase(yyruleno==125);
+      case 126: /* tagitem ::= BOOL */ yytestcase(yyruleno==126);
+{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+  yymsp[0].minor.yy134 = yylhsminor.yy134;
         break;
-      case 124: /* tagitem ::= NULL */
-{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
-  yymsp[0].minor.yy312 = yylhsminor.yy312;
+      case 127: /* tagitem ::= NULL */
+{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+  yymsp[0].minor.yy134 = yylhsminor.yy134;
         break;
-      case 125: /* tagitem ::= MINUS INTEGER */
-      case 126: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==126);
-      case 127: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==127);
-      case 128: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==128);
+      case 128: /* tagitem ::= MINUS INTEGER */
+      case 129: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==129);
+      case 130: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==130);
+      case 131: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==131);
 {
     yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
     yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type;
     toTSDBType(yymsp[-1].minor.yy0.type);
-    tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+    tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
 }
-  yymsp[-1].minor.yy312 = yylhsminor.yy312;
+  yymsp[-1].minor.yy134 = yylhsminor.yy134;
         break;
-      case 129: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+      case 132: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
 {
-  yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188);
+  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
 }
-  yymsp[-11].minor.yy150 = yylhsminor.yy150;
+  yymsp[-11].minor.yy414 = yylhsminor.yy414;
         break;
-      case 130: /* union ::= select */
-{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); }
-  yymsp[0].minor.yy25 = yylhsminor.yy25;
+      case 133: /* union ::= select */
+{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); }
+  yymsp[0].minor.yy231 = yylhsminor.yy231;
         break;
-      case 131: /* union ::= LP union RP */
-{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; }
+      case 134: /* union ::= LP union RP */
+{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; }
         break;
-      case 132: /* union ::= union UNION ALL select */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); }
-  yymsp[-3].minor.yy25 = yylhsminor.yy25;
+      case 135: /* union ::= union UNION ALL select */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); }
+  yymsp[-3].minor.yy231 = yylhsminor.yy231;
         break;
-      case 133: /* union ::= union UNION ALL LP select RP */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); }
-  yymsp[-5].minor.yy25 = yylhsminor.yy25;
+      case 136: /* union ::= union UNION ALL LP select RP */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); }
+  yymsp[-5].minor.yy231 = yylhsminor.yy231;
         break;
-      case 134: /* cmd ::= union */
-{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); }
+      case 137: /* cmd ::= union */
+{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); }
         break;
-      case 135: /* select ::= SELECT selcollist */
+      case 138: /* select ::= SELECT selcollist */
 {
-  yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 }
-  yymsp[-1].minor.yy150 = yylhsminor.yy150;
+  yymsp[-1].minor.yy414 = yylhsminor.yy414;
         break;
-      case 136: /* sclp ::= selcollist COMMA */
-{yylhsminor.yy224 = yymsp[-1].minor.yy224;}
-  yymsp[-1].minor.yy224 = yylhsminor.yy224;
+      case 139: /* sclp ::= selcollist COMMA */
+{yylhsminor.yy290 = yymsp[-1].minor.yy290;}
+  yymsp[-1].minor.yy290 = yylhsminor.yy290;
         break;
-      case 137: /* sclp ::= */
-{yymsp[1].minor.yy224 = 0;}
+      case 140: /* sclp ::= */
+{yymsp[1].minor.yy290 = 0;}
         break;
-      case 138: /* selcollist ::= sclp expr as */
+      case 141: /* selcollist ::= sclp expr as */
 {
-   yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
+   yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
 }
-  yymsp[-2].minor.yy224 = yylhsminor.yy224;
+  yymsp[-2].minor.yy290 = yylhsminor.yy290;
         break;
-      case 139: /* selcollist ::= sclp STAR */
+      case 142: /* selcollist ::= sclp STAR */
 {
    tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL);
-   yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0);
+   yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0);
 }
-  yymsp[-1].minor.yy224 = yylhsminor.yy224;
+  yymsp[-1].minor.yy290 = yylhsminor.yy290;
         break;
-      case 140: /* as ::= AS ids */
+      case 143: /* as ::= AS ids */
 { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;    }
         break;
-      case 141: /* as ::= ids */
+      case 144: /* as ::= ids */
 { yylhsminor.yy0 = yymsp[0].minor.yy0;    }
   yymsp[0].minor.yy0 = yylhsminor.yy0;
         break;
-      case 142: /* as ::= */
+      case 145: /* as ::= */
 { yymsp[1].minor.yy0.n = 0;  }
         break;
-      case 143: /* from ::= FROM tablelist */
-{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;}
+      case 146: /* from ::= FROM tablelist */
+{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;}
         break;
-      case 144: /* tablelist ::= ids cpxName */
-{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);}
-  yymsp[-1].minor.yy494 = yylhsminor.yy494;
+      case 147: /* tablelist ::= ids cpxName */
+{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);}
+  yymsp[-1].minor.yy498 = yylhsminor.yy498;
         break;
-      case 145: /* tablelist ::= tablelist COMMA ids cpxName */
-{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1);   }
-  yymsp[-3].minor.yy494 = yylhsminor.yy494;
+      case 148: /* tablelist ::= tablelist COMMA ids cpxName */
+{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1);   }
+  yymsp[-3].minor.yy498 = yylhsminor.yy498;
         break;
-      case 146: /* tmvar ::= VARIABLE */
+      case 149: /* tmvar ::= VARIABLE */
 {yylhsminor.yy0 = yymsp[0].minor.yy0;}
   yymsp[0].minor.yy0 = yylhsminor.yy0;
         break;
-      case 147: /* interval_opt ::= INTERVAL LP tmvar RP */
-      case 152: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==152);
+      case 150: /* interval_opt ::= INTERVAL LP tmvar RP */
+      case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==155);
 {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0;     }
         break;
-      case 148: /* interval_opt ::= */
-      case 153: /* sliding_opt ::= */ yytestcase(yyruleno==153);
+      case 151: /* interval_opt ::= */
+      case 156: /* sliding_opt ::= */ yytestcase(yyruleno==156);
 {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0;   }
         break;
-      case 149: /* fill_opt ::= */
-{yymsp[1].minor.yy494 = 0;     }
+      case 152: /* fill_opt ::= */
+{yymsp[1].minor.yy498 = 0;     }
         break;
-      case 150: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+      case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
 {
     tVariant A = {0};
     toTSDBType(yymsp[-3].minor.yy0.type);
     tVariantCreate(&A, &yymsp[-3].minor.yy0);
 
-    tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0);
-    yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494;
+    tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0);
+    yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498;
 }
         break;
-      case 151: /* fill_opt ::= FILL LP ID RP */
+      case 154: /* fill_opt ::= FILL LP ID RP */
 {
     toTSDBType(yymsp[-1].minor.yy0.type);
-    yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+    yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
 }
         break;
-      case 154: /* orderby_opt ::= */
-      case 162: /* groupby_opt ::= */ yytestcase(yyruleno==162);
-{yymsp[1].minor.yy494 = 0;}
+      case 157: /* orderby_opt ::= */
+      case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165);
+{yymsp[1].minor.yy498 = 0;}
         break;
-      case 155: /* orderby_opt ::= ORDER BY sortlist */
-      case 163: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==163);
-{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;}
+      case 158: /* orderby_opt ::= ORDER BY sortlist */
+      case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166);
+{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;}
         break;
-      case 156: /* sortlist ::= sortlist COMMA item sortorder */
+      case 159: /* sortlist ::= sortlist COMMA item sortorder */
 {
-    yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+    yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-3].minor.yy494 = yylhsminor.yy494;
+  yymsp[-3].minor.yy498 = yylhsminor.yy498;
         break;
-      case 157: /* sortlist ::= item sortorder */
+      case 160: /* sortlist ::= item sortorder */
 {
-  yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-1].minor.yy494 = yylhsminor.yy494;
+  yymsp[-1].minor.yy498 = yylhsminor.yy498;
         break;
-      case 158: /* item ::= ids cpxName */
+      case 161: /* item ::= ids cpxName */
 {
   toTSDBType(yymsp[-1].minor.yy0.type);
   yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
 
-  tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+  tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
 }
-  yymsp[-1].minor.yy312 = yylhsminor.yy312;
+  yymsp[-1].minor.yy134 = yylhsminor.yy134;
         break;
-      case 159: /* sortorder ::= ASC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; }
+      case 162: /* sortorder ::= ASC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; }
         break;
-      case 160: /* sortorder ::= DESC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;}
+      case 163: /* sortorder ::= DESC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;}
         break;
-      case 161: /* sortorder ::= */
-{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;}
+      case 164: /* sortorder ::= */
+{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;}
         break;
-      case 164: /* grouplist ::= grouplist COMMA item */
+      case 167: /* grouplist ::= grouplist COMMA item */
 {
-  yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);
+  yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[-2].minor.yy494 = yylhsminor.yy494;
+  yymsp[-2].minor.yy498 = yylhsminor.yy498;
         break;
-      case 165: /* grouplist ::= item */
+      case 168: /* grouplist ::= item */
 {
-  yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1);
+  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[0].minor.yy494 = yylhsminor.yy494;
+  yymsp[0].minor.yy498 = yylhsminor.yy498;
         break;
-      case 166: /* having_opt ::= */
-      case 176: /* where_opt ::= */ yytestcase(yyruleno==176);
-      case 212: /* expritem ::= */ yytestcase(yyruleno==212);
-{yymsp[1].minor.yy66 = 0;}
+      case 169: /* having_opt ::= */
+      case 179: /* where_opt ::= */ yytestcase(yyruleno==179);
+      case 215: /* expritem ::= */ yytestcase(yyruleno==215);
+{yymsp[1].minor.yy64 = 0;}
         break;
-      case 167: /* having_opt ::= HAVING expr */
-      case 177: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==177);
-{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;}
+      case 170: /* having_opt ::= HAVING expr */
+      case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180);
+{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;}
         break;
-      case 168: /* limit_opt ::= */
-      case 172: /* slimit_opt ::= */ yytestcase(yyruleno==172);
-{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;}
+      case 171: /* limit_opt ::= */
+      case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175);
+{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
         break;
-      case 169: /* limit_opt ::= LIMIT signed */
-      case 173: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==173);
-{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271;  yymsp[-1].minor.yy188.offset = 0;}
+      case 172: /* limit_opt ::= LIMIT signed */
+      case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176);
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
         break;
-      case 170: /* limit_opt ::= LIMIT signed OFFSET signed */
-      case 174: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==174);
-{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271;  yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;}
+      case 173: /* limit_opt ::= LIMIT signed OFFSET signed */
+      case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177);
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
         break;
-      case 171: /* limit_opt ::= LIMIT signed COMMA signed */
-      case 175: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==175);
-{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271;  yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;}
+      case 174: /* limit_opt ::= LIMIT signed COMMA signed */
+      case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178);
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
         break;
-      case 178: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; }
+      case 181: /* expr ::= LP expr RP */
+{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; }
         break;
-      case 179: /* expr ::= ID */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 182: /* expr ::= ID */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 180: /* expr ::= ID DOT ID */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 183: /* expr ::= ID DOT ID */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 181: /* expr ::= ID DOT STAR */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 184: /* expr ::= ID DOT STAR */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 182: /* expr ::= INTEGER */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 185: /* expr ::= INTEGER */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 183: /* expr ::= MINUS INTEGER */
-      case 184: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==184);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
-  yymsp[-1].minor.yy66 = yylhsminor.yy66;
+      case 186: /* expr ::= MINUS INTEGER */
+      case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
+  yymsp[-1].minor.yy64 = yylhsminor.yy64;
         break;
-      case 185: /* expr ::= FLOAT */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 188: /* expr ::= FLOAT */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 186: /* expr ::= MINUS FLOAT */
-      case 187: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==187);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
-  yymsp[-1].minor.yy66 = yylhsminor.yy66;
+      case 189: /* expr ::= MINUS FLOAT */
+      case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
+  yymsp[-1].minor.yy64 = yylhsminor.yy64;
         break;
-      case 188: /* expr ::= STRING */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 191: /* expr ::= STRING */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 189: /* expr ::= NOW */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 192: /* expr ::= NOW */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 190: /* expr ::= VARIABLE */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 193: /* expr ::= VARIABLE */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 191: /* expr ::= BOOL */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 194: /* expr ::= BOOL */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 192: /* expr ::= ID LP exprlist RP */
+      case 195: /* expr ::= ID LP exprlist RP */
 {
-  yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
+  yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
 }
-  yymsp[-3].minor.yy66 = yylhsminor.yy66;
+  yymsp[-3].minor.yy64 = yylhsminor.yy64;
         break;
-      case 193: /* expr ::= ID LP STAR RP */
+      case 196: /* expr ::= ID LP STAR RP */
 {
-  yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
+  yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
 }
-  yymsp[-3].minor.yy66 = yylhsminor.yy66;
+  yymsp[-3].minor.yy64 = yylhsminor.yy64;
         break;
-      case 194: /* expr ::= expr AND expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 197: /* expr ::= expr AND expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 195: /* expr ::= expr OR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 198: /* expr ::= expr OR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 196: /* expr ::= expr LT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 199: /* expr ::= expr LT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 197: /* expr ::= expr GT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 200: /* expr ::= expr GT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 198: /* expr ::= expr LE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 201: /* expr ::= expr LE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 199: /* expr ::= expr GE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 202: /* expr ::= expr GE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 200: /* expr ::= expr NE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 203: /* expr ::= expr NE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 201: /* expr ::= expr EQ expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 204: /* expr ::= expr EQ expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 202: /* expr ::= expr PLUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 205: /* expr ::= expr PLUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 203: /* expr ::= expr MINUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 206: /* expr ::= expr MINUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 204: /* expr ::= expr STAR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 207: /* expr ::= expr STAR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 205: /* expr ::= expr SLASH expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 208: /* expr ::= expr SLASH expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 206: /* expr ::= expr REM expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM);   }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 209: /* expr ::= expr REM expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM);   }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 207: /* expr ::= expr LIKE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 210: /* expr ::= expr LIKE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 208: /* expr ::= expr IN LP exprlist RP */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); }
-  yymsp[-4].minor.yy66 = yylhsminor.yy66;
+      case 211: /* expr ::= expr IN LP exprlist RP */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); }
+  yymsp[-4].minor.yy64 = yylhsminor.yy64;
         break;
-      case 209: /* exprlist ::= exprlist COMMA expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);}
-  yymsp[-2].minor.yy224 = yylhsminor.yy224;
+      case 212: /* exprlist ::= exprlist COMMA expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);}
+  yymsp[-2].minor.yy290 = yylhsminor.yy290;
         break;
-      case 210: /* exprlist ::= expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);}
-  yymsp[0].minor.yy224 = yylhsminor.yy224;
+      case 213: /* exprlist ::= expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);}
+  yymsp[0].minor.yy290 = yylhsminor.yy290;
         break;
-      case 211: /* expritem ::= expr */
-{yylhsminor.yy66 = yymsp[0].minor.yy66;}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 214: /* expritem ::= expr */
+{yylhsminor.yy64 = yymsp[0].minor.yy64;}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 213: /* cmd ::= RESET QUERY CACHE */
+      case 216: /* cmd ::= RESET QUERY CACHE */
 { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
         break;
-      case 214: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+      case 217: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 215: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+      case 218: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2716,14 +2732,14 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 216: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+      case 219: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 217: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+      case 220: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2734,7 +2750,7 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 218: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+      case 221: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
 
@@ -2748,25 +2764,25 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 219: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+      case 222: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
 {
     yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
 
     toTSDBType(yymsp[-2].minor.yy0.type);
     tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
-    A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1);
+    A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 220: /* cmd ::= KILL CONNECTION INTEGER */
+      case 223: /* cmd ::= KILL CONNECTION INTEGER */
 {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);}
         break;
-      case 221: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
+      case 224: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);}
         break;
-      case 222: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
+      case 225: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);}
         break;
       default:
diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp
index 63ed89ab9fe1de2222c6f5ea8ae05b331437f76d..e9611a3232de4a8c442ca3985a2c447b091bc981 100644
--- a/src/query/tests/resultBufferTest.cpp
+++ b/src/query/tests/resultBufferTest.cpp
@@ -10,7 +10,7 @@ namespace {
 // simple test
 void simpleTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -18,17 +18,144 @@ void simpleTest() {
   tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
   ASSERT_TRUE(pBufPage != NULL);
   
-  ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
-  ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
+  ASSERT_EQ(getResBufSize(pResultBuf), 1024);
   
   SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(list), 1);
   ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
-  
-  destroyResultBuf(pResultBuf, NULL);
+
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+
+  tFilePage* t = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t == pBufPage1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void writeDownTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+
+  *(int32_t*)(pBufPage->data) = nx;
+  writePageId = pageId;
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void recyclePageTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+  releaseResBufPage(pResultBuf, t4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t5 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t5 == pBufPage5);
+  ASSERT_TRUE(pageId == 5);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  *(int32_t*)(pBufPagex->data) = nx;
+  writePageId = pageId;   // update the data
+  releaseResBufPage(pResultBuf, pBufPagex);
+
+  tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 6);
+
+  destroyResultBuf(pResultBuf);
 }
 } // namespace
 
+
 TEST(testCase, resultBufferTest) {
+  srand(time(NULL));
   simpleTest();
+  writeDownTest();
+  recyclePageTest();
 }
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index 96c63b45ca522733de0eaea3fc9248e147dae8f6..902c8b66e4512e79e30d1e40bd6b201f787d56fb 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
@@ -12,10 +12,11 @@ INCLUDE_DIRECTORIES(inc)
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(./src SRC)
 ELSEIF (TD_DARWIN_64)
-  LIST(APPEND SRC ./src/thaship.c)
-  LIST(APPEND SRC ./src/trpc.c)
-  LIST(APPEND SRC ./src/tstring.c)
-  LIST(APPEND SRC ./src/tudp.c)
+  #LIST(APPEND SRC ./src/thaship.c)
+  #LIST(APPEND SRC ./src/trpc.c)
+  #LIST(APPEND SRC ./src/tstring.c)
+  #LIST(APPEND SRC ./src/tudp.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
 ELSEIF (TD_WINDOWS_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   LIST(APPEND SRC ./src/thaship.c)
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 95840af1a267dbbe320319c301e0b5566556de50..7425558535ddf459ae64659df5c18989f6ef58af 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -626,6 +626,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   pConn->pReqMsg = NULL;
   pConn->reqMsgLen = 0;
   pConn->pContext = NULL;
+  pConn->chandle = NULL;
 
   taosFreeId(pRpc->idPool, pConn->sid);
   tDebug("%s, rpc connection is released", pConn->info);
@@ -656,7 +657,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->sid = sid;
     pConn->tranId = (uint16_t)(random() & 0xFFFF);
     pConn->ownId = htonl(pConn->sid);
-    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid());
+    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid() + (int64_t)pConn->tranId);
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
@@ -913,7 +914,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
   }
 
   sid = pConn->sid;
-  pConn->chandle = pRecv->chandle;
+  if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle;
   pConn->peerIp = pRecv->ip; 
   pConn->peerPort = pRecv->port;
   if (pHead->port) pConn->peerPort = htons(pHead->port); 
@@ -1015,7 +1016,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     if (code != 0) { // parsing error
       if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
-        tDebug("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
+        tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 
       rpcProcessIncomingMsg(pConn, pHead);
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index decd7a8307eb17753a393d9f5f001bbce75db357..bdaae4c59785895a8a6d4d21e72044ec7f8948ee 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -242,7 +242,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
   destAdd.sin_addr.s_addr = ip;
   destAdd.sin_port = htons(port);
 
-  int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
+  int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
 
   return ret;
 }
diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt
index 6787de56f66ab87894909a4261bd7cdb2ebf3dfb..ac7ea8e4c119fd82ea6b1fc7b45a7c7dcab0c792 100644
--- a/src/rpc/test/CMakeLists.txt
+++ b/src/rpc/test/CMakeLists.txt
@@ -1,13 +1,13 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
 
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   LIST(APPEND CLIENT_SRC ./rclient.c)
   ADD_EXECUTABLE(rclient ${CLIENT_SRC})
   TARGET_LINK_LIBRARIES(rclient trpc)
diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9b7def6a68956b8a889a5d5cb81980e30a747321
--- /dev/null
+++ b/src/sync/CMakeLists.txt
@@ -0,0 +1,23 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  LIST(REMOVE_ITEM SRC ./src/tarbitrator.c)
+
+  ADD_LIBRARY(sync ${SRC})
+  TARGET_LINK_LIBRARIES(sync tutil pthread common)
+
+  LIST(APPEND BIN_SRC ./src/tarbitrator.c)
+  LIST(APPEND BIN_SRC ./src/taosTcpPool.c)
+  ADD_EXECUTABLE(tarbitrator ${BIN_SRC})
+  TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil)
+
+  ADD_SUBDIRECTORY(test)
+ENDIF ()
diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h
new file mode 100644
index 0000000000000000000000000000000000000000..be1e01cb2371b82acf4cf63e896a6fed68399222
--- /dev/null
+++ b/src/sync/inc/syncInt.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_SYNCINT_H
+#define TDENGINE_SYNCINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }}
+#define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }}
+#define sWarn(...)  { if (sDebugFlag & DEBUG_WARN)  { taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); }}
+#define sInfo(...)  { if (sDebugFlag & DEBUG_INFO)  { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+
+#define TAOS_SMSG_SYNC_DATA    1
+#define TAOS_SMSG_FORWARD      2
+#define TAOS_SMSG_FORWARD_RSP  3
+#define TAOS_SMSG_SYNC_REQ     4 
+#define TAOS_SMSG_SYNC_RSP     5
+#define TAOS_SMSG_SYNC_MUST    6
+#define TAOS_SMSG_STATUS       7
+
+#define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
+#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
+#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
+
+#pragma pack(push, 1)
+
+typedef struct {
+  char     type;        // msg type
+  char     pversion;    // protocol version
+  char     reserved[6]; // not used
+  int32_t  vgId;        // vg ID
+  int32_t  len;         // content length, does not include head
+  // char     cont[];      // message content starts from here
+} SSyncHead;
+
+typedef struct {
+  SSyncHead syncHead;
+  uint16_t  port;
+  char      fqdn[TSDB_FQDN_LEN];
+  int32_t   sourceId;  // only for arbitrator
+} SFirstPkt;
+
+typedef struct {
+  int8_t    role;
+  uint64_t  version;
+} SPeerStatus;
+
+typedef struct {
+  int8_t      role;
+  int8_t      ack;
+  uint64_t    version;
+  SPeerStatus peersStatus[];
+} SPeersStatus;
+
+typedef struct {
+  char      name[TSDB_FILENAME_LEN];
+  uint32_t  magic;
+  uint32_t  index;
+  uint64_t  fversion;
+  int32_t   size;
+} SFileInfo;
+
+typedef struct {
+  int8_t    sync;
+} SFileAck;
+
+typedef struct {
+  uint64_t  version;
+  int32_t   code;
+} SFwdRsp;
+  
+#pragma pack(pop)
+
+typedef struct {
+  char  *buffer;
+  int    bufferSize;
+  char  *offset;
+  int    forwards;
+  int    code;
+} SRecvBuffer;
+
+typedef struct {
+  uint64_t  version;
+  void     *mhandle;
+  int8_t    acks;
+  int8_t    nacks;
+  int8_t    confirmed;
+  int32_t   code;
+  uint64_t  time;
+} SFwdInfo;
+
+typedef struct {
+  int       first;
+  int       last;
+  int       fwds;  // number of forwards
+  SFwdInfo  fwdInfo[];
+} SSyncFwds;
+
+typedef struct SsyncPeer {
+  int32_t     nodeId;
+  uint32_t    ip;
+  uint16_t    port;
+  char        fqdn[TSDB_FQDN_LEN];  // peer ip string
+  char        id[TSDB_EP_LEN+16];   // peer vgId + end point
+  int8_t      role;
+  int8_t      sstatus;    // sync status
+  uint64_t    version;
+  uint64_t    sversion;   // track the peer version in retrieve process 
+  int         syncFd;
+  int         peerFd;     // forward FD
+  int         numOfRetrieves; // number of retrieves tried 
+  int         fileChanged;    // a flag to indicate file is changed during retrieving process
+  void       *timer;
+  void       *pConn;
+  int         notifyFd;
+  int         watchNum;
+  int        *watchFd;
+  int8_t      refCount;   // reference count
+  struct SSyncNode *pSyncNode;
+} SSyncPeer;
+
+typedef struct SSyncNode {
+  char         path[TSDB_FILENAME_LEN];
+  int8_t       replica;
+  int8_t       quorum;
+  uint32_t     vgId;
+  void        *ahandle;
+  int8_t       selfIndex;
+  SSyncPeer   *peerInfo[TAOS_SYNC_MAX_REPLICA+1];  // extra one for arbitrator
+  SSyncPeer   *pMaster;
+  int8_t       refCount;
+  SRecvBuffer *pRecv;
+  SSyncFwds   *pSyncFwds;  // saved forward info if quorum >1
+  void        *pFwdTimer;
+  FGetFileInfo    getFileInfo;
+  FGetWalInfo     getWalInfo;
+  FWriteToCache   writeToCache;
+  FConfirmForward confirmForward;
+  FNotifyRole     notifyRole;
+  FNotifyFlowCtrl notifyFlowCtrl;
+  FNotifyFileSynced notifyFileSynced;
+  pthread_mutex_t mutex;
+} SSyncNode;
+
+// sync module global
+extern int  tsSyncNum;
+extern char tsNodeFqdn[TSDB_FQDN_LEN];
+
+void *syncRetrieveData(void *param);
+void *syncRestoreData(void *param);
+int   syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
+void  syncRestartConnection(SSyncPeer *pPeer);
+void  syncBroadcastStatus(SSyncNode *pNode);
+void  syncAddPeerRef(SSyncPeer *pPeer);
+int   syncDecPeerRef(SSyncPeer *pPeer);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_VNODEPEER_H
diff --git a/src/os/linux/inc/tsysctl.h b/src/sync/inc/syncMain.h
similarity index 80%
rename from src/os/linux/inc/tsysctl.h
rename to src/sync/inc/syncMain.h
index 559d5653ee5f1083595d299c7dc61a99a0a35b46..d4ddb127334ea967cdc43dcc6a57dd4cb8c0ec3b 100644
--- a/src/os/linux/inc/tsysctl.h
+++ b/src/sync/inc/syncMain.h
@@ -13,18 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSCTL_H
-#define TDENGINE_TSYSCTL_H
+#ifndef TDENGINE_PLUGINS_SYNC_H
+#define TDENGINE_PLUGINS_SYNC_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef _ALPINE
-  #include <error.h>
-  #include <sys/sysctl.h>
-#else
-  #include <linux/sysctl.h>  
+#include <stdbool.h>
+#include <stdint.h>
+
+int32_t syncTest1();
+int32_t syncTest2();
+
+#ifdef __cplusplus
+}
 #endif
 
 #endif
diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h
new file mode 100644
index 0000000000000000000000000000000000000000..1e410acc26c3a5c82088bfd25d6a0d840835ba2f
--- /dev/null
+++ b/src/sync/inc/taosTcpPool.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TCP_POOL_H
+#define TDENGINE_TCP_POOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* ttpool_h;
+typedef void* tthread_h;
+
+typedef struct {
+  int       numOfThreads;
+  uint32_t  serverIp;
+  short     port;
+  int       bufferSize;
+  void     (*processBrokenLink)(void *ahandle);
+  int      (*processIncomingMsg)(void *ahandle, void *buffer);
+  void     (*processIncomingConn)(int fd, uint32_t ip);
+} SPoolInfo;
+
+ttpool_h   taosOpenTcpThreadPool(SPoolInfo *pInfo);
+void       taosCloseTcpThreadPool(ttpool_h);
+void      *taosAllocateTcpConn(void *, void *ahandle, int connFd);
+void       taosFreeTcpConn(void *);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TCP_POOL_H
+                                   
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
new file mode 100644
index 0000000000000000000000000000000000000000..5195e703649ad6875cd17bcdde2924487687415a
--- /dev/null
+++ b/src/sync/src/syncMain.c
@@ -0,0 +1,1226 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#include <stdint.h>
+//#include <stdbool.h>
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+// global configurable
+int       tsMaxSyncNum = 4;
+int       tsSyncTcpThreads = 2;
+int       tsMaxWatchFiles = 100;
+int       tsMaxFwdInfo = 200;
+int       tsSyncTimer = 1;
+//int       sDebugFlag = 135;
+//char      tsArbitrator[TSDB_FQDN_LEN] = {0};
+
+// module global, not configurable
+int       tsSyncNum;    // number of sync in process in whole system
+char      tsNodeFqdn[TSDB_FQDN_LEN];
+
+static int            tsNodeNum;    // number of nodes in system
+static ttpool_h       tsTcpPool;
+static void          *syncTmrCtrl = NULL;
+static void          *vgIdHash;
+static pthread_once_t syncModuleInit = PTHREAD_ONCE_INIT;
+
+// local functions
+static void  syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
+static void  syncRecoverFromMaster(SSyncPeer *pPeer);
+static void  syncCheckPeerConnection(void *param, void *tmrId);
+static void  syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void  syncProcessBrokenLink(void *param);
+static int   syncProcessPeerMsg(void *param, void *buffer);
+static void  syncProcessIncommingConnection(int connFd, uint32_t sourceIp); 
+static void  syncRemovePeer(SSyncPeer *pPeer);
+static void  syncAddArbitrator(SSyncNode *pNode);
+static void  syncAddNodeRef(SSyncNode *pNode);
+static void  syncDecNodeRef(SSyncNode *pNode);
+static void  syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
+static void  syncMonitorFwdInfos(void *param, void *tmrId);
+static void  syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
+static void  syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); 
+static void  syncRestartPeer(SSyncPeer *pPeer);
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
+
+char* syncRole[] = {
+  "offline",
+  "unsynced",
+  "syncing",
+  "slave",
+  "master"
+};
+
+static void syncModuleInitFunc() {
+  SPoolInfo info;
+
+  info.numOfThreads = tsSyncTcpThreads;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = syncProcessBrokenLink;
+  info.processIncomingMsg = syncProcessPeerMsg;
+  info.processIncomingConn = syncProcessIncommingConnection;
+
+  tsTcpPool = taosOpenTcpThreadPool(&info);
+  if (tsTcpPool == NULL) return;
+
+  syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
+  if (syncTmrCtrl == NULL) {
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    return;
+  }
+    
+  vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); 
+  if (vgIdHash == NULL) {
+    taosTmrCleanUp(syncTmrCtrl);
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    syncTmrCtrl = NULL;
+    return;
+  } 
+
+  tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn));
+}
+
+void *syncStart(const SSyncInfo *pInfo) 
+{
+  const SSyncCfg *pCfg = &pInfo->syncCfg;
+
+  SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1);
+  if (pNode == NULL) {
+    sError("no memory to allocate syncNode");
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return NULL;
+  }
+
+  pthread_once(&syncModuleInit, syncModuleInitFunc); 
+  if (tsTcpPool == NULL) {
+    free(pNode);
+    syncModuleInit = PTHREAD_ONCE_INIT;
+    sError("failed to init sync module(%s)", tstrerror(errno));
+    return NULL;
+  }
+
+  atomic_add_fetch_32(&tsNodeNum, 1);
+  tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
+  pthread_mutex_init(&pNode->mutex, NULL);
+
+  pNode->ahandle = pInfo->ahandle;
+  pNode->getFileInfo = pInfo->getFileInfo;
+  pNode->getWalInfo = pInfo->getWalInfo;
+  pNode->writeToCache = pInfo->writeToCache;
+  pNode->notifyRole = pInfo->notifyRole;
+  pNode->confirmForward = pInfo->confirmForward;
+  pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
+  pNode->notifyFileSynced = pInfo->notifyFileSynced;
+ 
+  pNode->selfIndex = -1;
+  pNode->vgId = pInfo->vgId;
+  pNode->replica = pCfg->replica;
+  pNode->quorum = pCfg->quorum;
+  if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
+
+  for (int i = 0; i < pCfg->replica; ++i) {
+    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
+    pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
+    if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  if (pNode->selfIndex < 0) {
+    sInfo("vgId:%d, this node is not configured", pNode->vgId);
+    terrno = TSDB_CODE_SYN_INVALID_CONFIG;
+    syncStop(pNode);
+    return NULL;
+  }
+
+  nodeVersion = pInfo->version;    // set the initial version
+  nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER;
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+
+  pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo*sizeof(SFwdInfo), 1);
+  if (pNode->pSyncFwds == NULL) {
+    sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+  if (pNode->pFwdTimer == NULL) {
+    sError("vgId:%d, failed to allocate timer", pNode->vgId);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  syncAddArbitrator(pNode);
+  syncAddNodeRef(pNode);
+  taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+ 
+  if (pNode->notifyRole) 
+   (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+
+  return pNode;
+}
+
+void syncStop(void *param) 
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+
+  if (pNode == NULL) return;
+  sInfo("vgId:%d, cleanup sync", pNode->vgId);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer) syncRemovePeer(pPeer); 
+  }
+
+  pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pPeer) syncRemovePeer(pPeer);
+
+  if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+  if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  syncDecNodeRef(pNode);
+}
+
+int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) 
+{
+  SSyncNode  *pNode = param;
+  int         i, j;
+
+  if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], 
+         pNewCfg->replica, pNode->replica);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (i = 0; i < pNode->replica; ++i) {
+    for (j = 0; j < pNewCfg->replica; ++j) {
+      if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && 
+          (pNode->peerInfo[i]->port == pNewCfg->nodeInfo[j].nodePort)) 
+        break;
+    }
+
+    if (j >= pNewCfg->replica) {
+      syncRemovePeer(pNode->peerInfo[i]);
+      pNode->peerInfo[i] = NULL;
+    }
+  }
+
+  SSyncPeer *newPeers[TAOS_SYNC_MAX_REPLICA];
+  for (i = 0; i < pNewCfg->replica; ++i) {
+    const SNodeInfo *pNewNode = &pNewCfg->nodeInfo[i];
+
+    for (j = 0; j < pNode->replica; ++j) {
+      if (pNode->peerInfo[j] && (strcmp(pNode->peerInfo[j]->fqdn, pNewNode->nodeFqdn) == 0) && 
+         (pNode->peerInfo[j]->port == pNewNode->nodePort))
+        break;
+    }
+
+    if (j >= pNode->replica) {
+      newPeers[i] = syncAddPeer(pNode, pNewNode);
+    } else {
+      newPeers[i] = pNode->peerInfo[j];
+    }
+
+    if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  pNode->replica = pNewCfg->replica;
+  pNode->quorum = pNewCfg->quorum;
+  if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
+  memcpy(pNode->peerInfo, newPeers, sizeof(SSyncPeer *) * pNewCfg->replica);
+
+  for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i)
+    pNode->peerInfo[i] = NULL;
+
+  syncAddArbitrator(pNode);
+
+  if (pNewCfg->replica <= 1) {
+    sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
+    nodeRole = TAOS_SYNC_ROLE_MASTER;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+  syncBroadcastStatus(pNode);
+
+  return 0;
+}
+
+int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype)
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+  SSyncHead  *pSyncHead;
+  SWalHead   *pWalHead = data;
+  int         fwdLen;
+  int         code = 0;
+
+  if (pNode == NULL) return 0;
+
+  // always update version
+  nodeVersion = pWalHead->version;
+  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER ) return 0;
+
+  // only pkt from RPC or CQ can be forwarded
+  if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+  // a hacker way to improve the performance
+  pSyncHead = (SSyncHead *) ( ((char *)pWalHead) - sizeof(SSyncHead));
+  pSyncHead->type = TAOS_SMSG_FORWARD;
+  pSyncHead->pversion = 0;
+  pSyncHead->len = sizeof(SWalHead) + pWalHead->len; 
+  fwdLen = pSyncHead->len + sizeof(SSyncHead);  //include the WAL and SYNC head 
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer == NULL || pPeer->peerFd <0) continue; 
+    if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; 
+  
+    if (pNode->quorum > 1 && code == 0) {
+      syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+      code = 1;
+    }
+
+    int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+    if (retLen == fwdLen) {
+      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+    } else {
+      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+void syncConfirmForward(void *param, uint64_t version, int32_t code)
+{
+  SSyncNode  *pNode = param;
+  if (pNode == NULL) return;
+  if (pNode->quorum <= 1) return;
+
+  SSyncPeer  *pPeer = pNode->pMaster;
+  if (pPeer == NULL) return;
+
+  char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+
+  SSyncHead   *pHead = (SSyncHead *) msg;
+  pHead->type = TAOS_SMSG_FORWARD_RSP;
+  pHead->len = sizeof(SFwdRsp);
+
+  SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+  pFwdRsp->version = version;
+  pFwdRsp->code = code;
+
+  int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+  int retLen = write(pPeer->peerFd, msg, msgLen);
+
+  if (retLen == msgLen) {
+    sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
+  } else {
+    sDebug("%s, failed to send forward ack, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+}
+
+void syncRecover(void *param) {
+  SSyncNode *pNode = param;
+  SSyncPeer *pPeer;
+
+  // to do: add a few lines to check if recover is OK 
+  // if take this node to unsync state, the whole system may not work
+
+  nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  nodeVersion = 0;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = (SSyncPeer *) pNode->peerInfo[i];
+    if (pPeer->peerFd >= 0) {
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+int syncGetNodesRole(void *param, SNodesRole *pNodesRole)
+{
+  SSyncNode *pNode = param;
+  
+  pNodesRole->selfIndex = pNode->selfIndex;
+  for (int i=0; i<pNode->replica; ++i) {
+    pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
+    pNodesRole->role[i] = pNode->peerInfo[i]->role;
+  }
+
+  return 0;
+}
+
+static void syncAddArbitrator(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+
+  // if not configured, return right away
+  if (tsArbitrator[0] == 0) {
+    if (pPeer) syncRemovePeer(pPeer);
+    pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    return;
+  }
+
+  SNodeInfo nodeInfo;
+  nodeInfo.nodeId = 0;
+  taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
+  nodeInfo.nodePort += TSDB_PORT_SYNC;
+  
+  if (pPeer) {
+    if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) {
+      return;
+    } else {
+      syncRemovePeer(pPeer);
+      pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    }
+  }   
+
+  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
+}
+
+static void syncAddNodeRef(SSyncNode *pNode)
+{
+   atomic_add_fetch_8(&pNode->refCount, 1);
+}
+
+static void syncDecNodeRef(SSyncNode *pNode)
+{
+  if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
+    pthread_mutex_destroy(&pNode->mutex);
+    tfree(pNode->pRecv);
+    tfree(pNode->pSyncFwds);
+    tfree(pNode);
+
+    if (atomic_sub_fetch_32(&tsNodeNum, 1) == 0) { 
+      if (tsTcpPool) taosCloseTcpThreadPool(tsTcpPool);
+      if (syncTmrCtrl) taosTmrCleanUp(syncTmrCtrl);
+      if (vgIdHash) taosHashCleanup(vgIdHash);
+      syncTmrCtrl = NULL;
+      tsTcpPool = NULL;
+      vgIdHash = NULL;
+      syncModuleInit = PTHREAD_ONCE_INIT;
+      sDebug("sync module is cleaned up");
+    }
+  }
+}
+
+void syncAddPeerRef(SSyncPeer *pPeer)
+{
+   atomic_add_fetch_8(&pPeer->refCount, 1);
+}
+
+int syncDecPeerRef(SSyncPeer *pPeer)
+{
+  if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
+    syncDecNodeRef(pPeer->pSyncNode);
+
+    sDebug("%s, resource is freed", pPeer->id);
+    tfree(pPeer->watchFd);
+    tfree(pPeer);
+    return 0;
+  }
+
+  return 1;
+}
+
+static void syncClosePeerConn(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+  tclose(pPeer->syncFd);
+  if (pPeer->peerFd >=0) {
+    pPeer->peerFd = -1;
+    taosFreeTcpConn(pPeer->pConn);
+  }
+}
+
+static void syncRemovePeer(SSyncPeer *pPeer)
+{
+  sInfo("%s, it is removed", pPeer->id);
+
+  pPeer->ip = 0;
+  syncClosePeerConn(pPeer);
+  syncDecPeerRef(pPeer);
+}
+
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) 
+{
+  uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn);
+  if (ip == -1) return NULL;
+ 
+  SSyncPeer *pPeer = (SSyncPeer *) calloc(1, sizeof(SSyncPeer));
+  if (pPeer == NULL) return NULL;
+
+  pPeer->nodeId = pInfo->nodeId;
+  tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn));
+  pPeer->ip = ip;
+  pPeer->port = pInfo->nodePort;
+  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%d", pNode->vgId, pPeer->fqdn, pPeer->port);
+
+  pPeer->peerFd = -1;
+  pPeer->syncFd = -1;
+  pPeer->role = TAOS_SYNC_ROLE_OFFLINE;
+  pPeer->pSyncNode = pNode;
+  pPeer->refCount = 1;
+
+  sInfo("%s, it is configured", pPeer->id);
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
+    sDebug("%s, start to check peer connection", pPeer->id);
+    taosTmrReset(syncCheckPeerConnection, 100, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+  
+  syncAddNodeRef(pNode);
+  return pPeer;
+}
+
+void syncBroadcastStatus(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if ( i == pNode->selfIndex ) continue;
+    pPeer = pNode->peerInfo[i];
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+  }
+} 
+
+static void syncResetFlowCtrl(SSyncNode *pNode) {
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pNode->peerInfo[i]->numOfRetrieves = 0;
+  }
+
+  if (pNode->notifyFlowCtrl)
+    (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);   
+}
+
+static void syncChooseMaster(SSyncNode *pNode) {
+  SSyncPeer *pPeer;
+  int        onlineNum = 0;
+  int        index = -1;
+  int        replica = pNode->replica;
+
+  sDebug("vgId:%d, choose master", pNode->vgId);
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE)
+      onlineNum++;
+  }
+
+  if (onlineNum == pNode->replica) {
+    // if all peers are online, peer with highest version shall be master
+    index = 0;
+    for (int i = 1; i < pNode->replica; ++i) {
+      if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version)
+        index = i;
+    }
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (index < 0 && onlineNum > replica/2.0) {
+    // over half of nodes are online
+    for (int i = 0; i < pNode->replica; ++i) {
+      //slave with highest version shall be master
+      pPeer = pNode->peerInfo[i];
+      if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
+        if (index < 0 || pPeer->version > pNode->peerInfo[index]->version)
+          index = i;
+      }
+    }
+  }
+
+  if (index >= 0) {
+    if (index == pNode->selfIndex) {
+      sInfo("vgId:%d, start to work as master", pNode->vgId);
+      nodeRole = TAOS_SYNC_ROLE_MASTER;
+      syncResetFlowCtrl(pNode);
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    } else {
+      pPeer = pNode->peerInfo[index];
+      sInfo("%s, it shall work as master", pPeer->id);
+    }
+  } else {
+    sDebug("vgId:%d, failed to choose master", pNode->vgId);
+  }
+} 
+ 
+static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) {
+  int onlineNum = 0;
+  int index = -1;
+  int replica = pNode->replica;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) 
+      onlineNum++;
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (onlineNum <= replica*0.5) {
+    if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+    }
+  } else {
+    for (int i=0; i<pNode->replica; ++i) {
+      SSyncPeer *pTemp = pNode->peerInfo[i];
+      if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue;
+      if ( index < 0 ) {
+        index = i;
+      } else { // multiple masters, it shall not happen 
+        if ( i == pNode->selfIndex ) {
+          sError("%s, peer is master, work as slave instead", pTemp->id);
+          nodeRole = TAOS_SYNC_ROLE_SLAVE;
+          (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+        }
+      }
+    }
+  }
+
+  SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL;
+  return pMaster;
+}
+
+static int syncValidateMaster(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int code = 0;
+
+  if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
+    sDebug("%s, slave has higher version, restart all connections!!!",  pPeer->id);
+    nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    code = -1;
+
+    for (int i = 0; i < pNode->replica; ++i) {
+      if ( i == pNode->selfIndex ) continue;
+      syncRestartPeer(pNode->peerInfo[i]);
+    }
+  } 
+
+  return code;
+}
+
+static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int8_t     peerOldRole = pPeer->role;
+  int8_t     selfOldRole = nodeRole;
+  int8_t     i, syncRequired = 0;
+
+  pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
+  pPeer->role = newRole;
+
+  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, 
+          syncRole[nodeRole], syncRole[pPeer->role]);  
+
+  SSyncPeer *pMaster = syncCheckMaster(pNode);
+
+  if ( pMaster ) {
+    // master is there
+    pNode->pMaster = pMaster;
+    sDebug("%s, it is the master, ver:%" PRIu64, pMaster->id, pMaster->version);
+
+    if (syncValidateMaster(pPeer) < 0) return;
+
+    if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) {
+      if ( nodeVersion < pMaster->version) {
+        syncRequired = 1;
+      } else {
+        sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version);
+        nodeRole = TAOS_SYNC_ROLE_SLAVE;
+        (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      }
+    } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
+      // nodeVersion = pMaster->version;
+    }
+  } else {
+    // master not there, if all peer's state and version are consistent, choose the master
+    int consistent = 0;
+    if (peersStatus) {
+      for (i = 0; i < pNode->replica; ++i) {
+        SSyncPeer *pTemp = pNode->peerInfo[i];
+        if (pTemp->role != peersStatus[i].role) break;
+        if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[i].version)) break; 
+      }
+ 
+      if (i >= pNode->replica) consistent = 1;
+    } else {
+      if (pNode->replica == 2) consistent = 1;
+    }
+
+    if (consistent)
+      syncChooseMaster(pNode);
+  }
+
+  if (syncRequired) {
+    syncRecoverFromMaster(pMaster);
+  }
+
+  if (peerOldRole != newRole || nodeRole != selfOldRole)
+    syncBroadcastStatus(pNode);
+
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) 
+    syncResetFlowCtrl(pNode);
+}
+
+static void syncRestartPeer(SSyncPeer *pPeer) {
+  sDebug("%s, restart connection", pPeer->id);
+
+  syncClosePeerConn(pPeer);
+
+  pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
+
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) )
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+}
+
+void syncRestartConnection(SSyncPeer *pPeer)
+{
+  if (pPeer->ip == 0) return;
+
+  syncRestartPeer(pPeer);
+  syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE);
+}
+
+static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  sDebug("%s, sync-req is received", pPeer->id);
+
+  if (pPeer->ip == 0) return;
+
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) {
+    sError("%s, I am not master anymore", pPeer->id);
+    tclose(pPeer->syncFd);
+    return;
+  }
+
+  if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started", pPeer->id);
+    return; // already started
+  }
+
+  // start a new thread to retrieve the data
+  syncAddPeerRef(pPeer);
+  pthread_attr_t  thattr;
+  pthread_t       thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+  int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    sError("%s, failed to create sync thread(%s)", pPeer->id, strerror(errno));
+    syncDecPeerRef(pPeer);
+  } else {
+    pPeer->sstatus = TAOS_SYNC_STATUS_START;
+    sDebug("%s, thread is created to retrieve data", pPeer->id);
+  }
+}
+
+static void syncNotStarted(void *param, void *tmrId)
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  pPeer->timer = NULL;
+  sInfo("%s, sync connection is still not up, restart", pPeer->id);
+  syncRestartConnection(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncTryRecoverFromMaster(void *param, void *tmrId) {
+  SSyncPeer   *pPeer = param;
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  syncRecoverFromMaster(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncRecoverFromMaster(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus);
+    return;
+  } 
+
+  taosTmrStopA(&pPeer->timer);
+  if (tsSyncNum >= tsMaxSyncNum) {
+    sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
+    taosTmrReset(syncTryRecoverFromMaster, 500, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  sDebug("%s, try to sync", pPeer->id)
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+  taosTmrReset(syncNotStarted, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+
+  if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt) ) {
+    sError("%s, failed to send sync-req to peer", pPeer->id);
+  } else {
+    nodeSStatus = TAOS_SYNC_STATUS_START;
+    sInfo("%s, sync-req is sent", pPeer->id);
+  }
+
+  return;
+}
+
+static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFwdRsp    *pFwdRsp = (SFwdRsp *) cont;
+  SSyncFwds  *pSyncFwds = pNode->pSyncFwds;
+  SFwdInfo   *pFwdInfo;
+
+  sDebug("%s, forward-rsp is received, ver:%" PRIu64, pPeer->id, pFwdRsp->version);
+  SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
+
+  if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
+    // find the forwardInfo from first
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      pFwdInfo = pSyncFwds->fwdInfo + (i+pSyncFwds->first)%tsMaxFwdInfo;
+      if (pFwdRsp->version == pFwdInfo->version) break;
+    }
+ 
+    syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code);
+    syncRemoveConfirmedFwdInfo(pNode);
+  }
+}
+
+
+static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SWalHead    *pHead = (SWalHead *)cont;
+
+  sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
+
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
+    //nodeVersion = pHead->version;
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  } else { 
+    if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+      syncSaveIntoBuffer(pPeer, pHead);
+    } else {
+      sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    }
+  }
+
+  return;
+}
+
+static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode    *pNode = pPeer->pSyncNode;
+  SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
+
+  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+         syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
+
+  pPeer->version = pPeersStatus->version;
+  syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role);
+
+  if (pPeersStatus->ack)
+    syncSendPeersStatusMsgToPeer(pPeer, 0);
+}
+
+static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
+  if (pPeer->peerFd <0) return -1;
+
+  int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
+  if (hlen != sizeof(SSyncHead)) {
+    sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
+    return -1;
+  }
+
+  // head.len = htonl(head.len);
+  if (pHead->len <0) {
+    sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
+    return -1;
+  } 
+
+  int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
+  if (bytes != pHead->len) {
+    sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int syncProcessPeerMsg(void *param, void *buffer)
+{
+  SSyncPeer  *pPeer = param;
+  SSyncHead   head;
+  char       *cont = (char *)buffer;
+
+  SSyncNode *pNode = pPeer->pSyncNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  int code = syncReadPeerMsg(pPeer, &head, cont);
+
+  if (code == 0) {
+    if (head.type == TAOS_SMSG_FORWARD) {
+      syncProcessForwardFromPeer(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_FORWARD_RSP) {
+      syncProcessFwdResponse(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_SYNC_REQ) {
+      syncProcessSyncRequest(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_STATUS) {
+      syncProcessPeersStatusMsg(cont, pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+#define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA
+
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  char       msg[statusMsgLen] = {0};
+
+  if (pPeer->peerFd <0 || pPeer->ip ==0) return;
+
+  SSyncHead    *pHead = (SSyncHead *) msg;
+  SPeersStatus *pPeersStatus = (SPeersStatus *) (msg + sizeof(SSyncHead));
+
+  pHead->type = TAOS_SMSG_STATUS;
+  pHead->len = statusMsgLen - sizeof(SSyncHead);
+
+  pPeersStatus->version = nodeVersion;
+  pPeersStatus->role = nodeRole;
+  pPeersStatus->ack = ack;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
+    pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
+  }
+
+  int retLen = write(pPeer->peerFd, msg, statusMsgLen);
+  if (retLen == statusMsgLen) {
+    sDebug("%s, status msg is sent", pPeer->id);
+  } else {
+    sDebug("%s, failed to send status msg, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+
+  return;
+}
+
+static void syncSetupPeerConnection(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  taosTmrStopA(&pPeer->timer);
+  if (pPeer->peerFd >= 0) {
+    sDebug("%s, send role version to peer", pPeer->id);
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+    return;
+  }
+
+  int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (connFd < 0) {
+    sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.vgId = pPeer->nodeId ? pNode->vgId:0;
+  firstPkt.syncHead.type = TAOS_SMSG_STATUS;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); 
+  firstPkt.port = tsSyncPort;
+  firstPkt.sourceId = pNode->vgId;  // tell arbitrator its vgId
+
+  if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
+    sDebug("%s, connection to peer server is setup", pPeer->id);
+    pPeer->peerFd = connFd; 
+    pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
+    pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+    syncAddPeerRef(pPeer);
+  } else {
+    sDebug("try later");
+    close(connFd);
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+}
+
+static void syncCheckPeerConnection(void *param, void *tmrId) 
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, check peer connection", pPeer->id);
+  syncSetupPeerConnection(pPeer); 
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncCreateRestoreDataThread(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+
+  pthread_attr_t thattr;
+  pthread_t thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+
+  syncAddPeerRef(pPeer);
+  int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret < 0) {
+    sError("%s, failed to create sync thread", pPeer->id);
+    tclose(pPeer->syncFd);
+    syncDecPeerRef(pPeer);
+  } else { 
+    sInfo("%s, sync connection is up", pPeer->id);
+  }
+}
+
+static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) 
+{
+  char  ipstr[24];
+  int   i;
+   
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  int32_t vgId = firstPkt.syncHead.vgId;
+  SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
+  if (ppNode == NULL || *ppNode == NULL) {
+    sError("vgId:%d, vgId could not be found", vgId);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SSyncNode *pNode = *ppNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  SSyncPeer *pPeer;
+  for (i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port))
+      break;
+  }
+
+  pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL;
+  if (pPeer == NULL) {
+    sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn);
+    taosCloseSocket(connFd);
+    // syncSendVpeerCfgMsg(sync);
+  } else {
+    // first packet tells what kind of link
+    if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) {
+      pPeer->syncFd = connFd;
+      syncCreateRestoreDataThread(pPeer);
+    } else {
+      sDebug("%s, TCP connection is already up, close one", pPeer->id);
+      syncClosePeerConn(pPeer);
+      pPeer->peerFd = connFd;
+      pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+      syncAddPeerRef(pPeer);
+      sDebug("%s, ready to exchange data", pPeer->id);
+      syncSendPeersStatusMsgToPeer(pPeer, 1);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return;
+}
+
+static void syncProcessBrokenLink(void *param) {
+  if (param == NULL) return;  // the connection for arbitrator
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  syncAddNodeRef(pNode);
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
+  pPeer->peerFd = -1;
+
+  if (syncDecPeerRef(pPeer) != 0) {
+    syncRestartConnection(pPeer);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+  syncDecNodeRef(pNode);
+}
+
+static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) 
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds >= tsMaxFwdInfo) {
+    pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+  } 
+
+  if (pSyncFwds->fwds > 0) 
+    pSyncFwds->last = (pSyncFwds->last+1) % tsMaxFwdInfo;
+  SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
+  pFwdInfo->version = version;
+  pFwdInfo->mhandle = mhandle;
+  pFwdInfo->acks = 0;
+  pFwdInfo->confirmed = 0;
+  pFwdInfo->time = time;
+
+  pSyncFwds->fwds++;
+  sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds);
+}
+
+static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode)
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+
+  int fwds = pSyncFwds->fwds;
+  for (int i=0; i<fwds; ++i) {
+    SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; 
+    if (pFwdInfo->confirmed == 0) break;
+
+    pSyncFwds->first = (pSyncFwds->first+1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+    if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last;
+    //sDebug("vgId:%d, fwd info is removed, ver:%d, fwds:%d", 
+    //        pNode->vgId, pFwdInfo->version, pSyncFwds->fwds);
+    memset(pFwdInfo, 0, sizeof(SFwdInfo));
+  }
+}
+
+static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) 
+{
+  int confirm = 0;
+  if (pFwdInfo->code == 0) pFwdInfo->code = code;
+
+  if (code == 0) {
+    pFwdInfo->acks++;
+    if (pFwdInfo->acks >= pNode->quorum-1) 
+      confirm = 1;
+  } else {
+    pFwdInfo->nacks++;
+    if (pFwdInfo->nacks > pNode->replica-pNode->quorum) 
+      confirm = 1;
+  }
+
+  if (confirm && pFwdInfo->confirmed == 0) {
+    sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);
+    (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code);
+    pFwdInfo->confirmed = 1;
+  }
+}
+
+static void syncMonitorFwdInfos(void *param, void *tmrId)
+{
+  SSyncNode *pNode = param;
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds > 0) {
+    pthread_mutex_lock(&(pNode->mutex));
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first+i) % tsMaxFwdInfo; 
+      if (time - pFwdInfo->time < 2000) break;
+      syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+    }
+
+    syncRemoveConfirmedFwdInfo(pNode);
+    pthread_mutex_unlock(&(pNode->mutex));
+  } 
+  
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+}
+ 
+
+
diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c
new file mode 100644
index 0000000000000000000000000000000000000000..07d99b916ef57062cd91b14f0a970d0e80943452
--- /dev/null
+++ b/src/sync/src/syncRestore.c
@@ -0,0 +1,329 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eindex) {
+  char       name[TSDB_FILENAME_LEN*2] = {0};
+  char       fname[TSDB_FILENAME_LEN*3] = {0};
+  uint32_t   magic; 
+  uint64_t   fversion;
+  int32_t    size;
+  uint32_t   index = sindex;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  if (sindex < 0 || eindex < sindex) return;
+
+  while (1) {
+    name[0] = 0;
+    magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion);
+    if (magic == 0) break;
+
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
+    remove(fname);
+    sDebug("%s, %s is removed", pPeer->id, fname);
+
+    index++;
+    if (index > eindex) break;
+  }        
+}
+
+static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) 
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SFileInfo  minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */  // master file info
+  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */  // slave file info
+  SFileAck   fileAck; 
+  int        code = -1;
+  char       name[TSDB_FILENAME_LEN * 2] = {0};
+  uint32_t   pindex = 0;    // index in last restore
+
+  *fversion = 0;
+  sinfo.index = 0;
+  while (1) {
+    // read file info
+    int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
+    if (ret < 0 ) break;
+
+    // if no more file from master, break;
+    if (minfo.name[0] == 0 || minfo.magic == 0) {
+      sDebug("%s, no more files to restore", pPeer->id);
+
+      // remove extra files after the current index
+      syncRemoveExtraFile(pPeer, sinfo.index+1, TAOS_SYNC_MAX_INDEX);
+      code = 0; 
+      break;
+    }
+   
+    // remove extra files on slave between the current and last index
+    syncRemoveExtraFile(pPeer, pindex+1, minfo.index-1);
+    pindex = minfo.index;
+
+    // check the file info
+    sinfo = minfo;
+    sDebug("%s, get file info:%s", pPeer->id, minfo.name);
+    sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
+
+    // if file not there or magic is not the same, file shall be synced
+    memset(&fileAck, 0, sizeof(fileAck));
+    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1:0;
+
+    // send file ack
+    ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+ 
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      sDebug("%s, %s is the same", pPeer->id, minfo.name);
+      continue;
+    }
+
+    // if sync is required, open file, receive from master, and write to file
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
+
+    int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+    if ( dfd < 0 ) {
+      sError("%s, failed to open file:%s", pPeer->id, name);
+      break;
+    }
+
+    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
+    fsync(dfd);
+    close(dfd);
+    if (ret<0) break;
+
+    sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size);
+
+  }
+
+  if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+   // data file is changed, code shall be set to 1 
+    *fversion = minfo.fversion;
+    code = 1;  
+  }
+
+  if (code < 0) {
+    sError("%s, failed to restore %s(%s)", pPeer->id, name, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRestoreWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         ret, code = -1;
+
+  void *buffer = calloc(1024000, 1);  // size for one record
+  if (buffer == NULL) return -1;
+
+  SWalHead *pHead = (SWalHead *)buffer;
+
+  while (1) {
+    ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
+    if (ret <0)  break;
+
+    if (pHead->len == 0) {code = 0; break;}  // wal sync over
+    
+    ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
+    if (ret <0)  break;
+
+    sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
+  }
+
+  if (code<0) {
+    sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  free(buffer);
+  return code;
+}
+
+static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SWalHead  *pHead = (SWalHead *) offset;
+
+  (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  offset += pHead->len + sizeof(SWalHead);
+
+  return offset;
+}
+
+static int syncProcessBufferedFwd(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+  int          forwards = 0;
+
+  sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
+
+  char *offset = pRecv->buffer;
+  while (forwards < pRecv->forwards) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+  
+  pthread_mutex_lock(&pNode->mutex);
+
+  while (forwards < pRecv->forwards && pRecv->code == 0) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+
+  nodeRole = TAOS_SYNC_ROLE_SLAVE;
+  sDebug("%s, finish processing buffered fwds:%d", pPeer->id, forwards);
+
+  pthread_mutex_unlock(&pNode->mutex);
+
+  return pRecv->code;
+}
+
+int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead)
+{ 
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+
+  if (pRecv == NULL) return -1;
+  int len = pHead->len + sizeof(SWalHead);
+
+  if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
+    memcpy(pRecv->offset, pHead, len);
+    pRecv->offset += len;
+    pRecv->forwards++;
+    sDebug("%s, fwd is saved into queue, ver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
+  } else {
+    sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize);
+    pRecv->code = -1;  // set error code
+  }
+
+  return pRecv->code;
+}
+
+static void syncCloseRecvBuffer(SSyncNode *pNode)
+{
+  if (pNode->pRecv) {
+    tfree(pNode->pRecv->buffer);
+  }
+
+  tfree(pNode->pRecv);
+}
+
+static int syncOpenRecvBuffer(SSyncNode *pNode) 
+{
+  syncCloseRecvBuffer(pNode);
+
+  SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
+  if (pRecv == NULL) return -1;
+
+  pRecv->bufferSize = 5000000;
+  pRecv->buffer = malloc(pRecv->bufferSize);
+  if (pRecv->buffer == NULL) {
+    free(pRecv);
+    return -1;
+  }
+
+  pRecv->offset = pRecv->buffer;
+  pRecv->forwards = 0;
+
+  pNode->pRecv = pRecv;
+
+  return 0;
+}
+
+static int syncRestoreDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  nodeSStatus = TAOS_SYNC_STATUS_FILE;
+  uint64_t fversion = 0;
+
+  sDebug("%s, start to restore file", pPeer->id);
+  int code = syncRestoreFile(pPeer, &fversion);
+  if (code < 0) {
+    sError("%s, failed to restore file", pPeer->id);
+    return -1;
+  }
+
+  // if code > 0, data file is changed, notify app, and pass the version 
+  if (code > 0 && pNode->notifyFileSynced) {
+    if ( (*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0 ) {
+      sError("%s, app not in ready state", pPeer->id);
+      return -1;
+    }
+  }
+
+  nodeVersion = fversion;
+
+  sDebug("%s, start to restore wal", pPeer->id);
+  if (syncRestoreWal(pPeer) < 0) {
+    sError("%s, failed to restore wal", pPeer->id);
+    return -1;
+  }
+
+  nodeSStatus = TAOS_SYNC_STATUS_CACHE;
+  sDebug("%s, start to insert buffered points", pPeer->id);
+  if (syncProcessBufferedFwd(pPeer) < 0) {
+    sError("%s, failed to insert buffered points", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRestoreData(void *param)
+{
+  SSyncPeer  *pPeer = (SSyncPeer *)param;
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  taosBlockSIGPIPE();
+  __sync_fetch_and_add(&tsSyncNum, 1);
+
+  (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING);
+
+  if (syncOpenRecvBuffer(pNode) < 0) {
+    sError("%s, failed to allocate recv buffer", pPeer->id);
+  } else { 
+    if ( syncRestoreDataStepByStep(pPeer) == 0) {
+      sInfo("%s, it is synced successfully", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_SLAVE;
+      syncBroadcastStatus(pNode);
+    } else {
+      sError("%s, failed to restore data, restart connection", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+
+  nodeSStatus = TAOS_SYNC_STATUS_INIT;
+  tclose(pPeer->syncFd)
+  syncCloseRecvBuffer(pNode);
+  __sync_fetch_and_sub(&tsSyncNum, 1);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}
+
diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c
new file mode 100644
index 0000000000000000000000000000000000000000..8aa7d101e7c3e1ba1fda6d26817d07871dbb6d03
--- /dev/null
+++ b/src/sync/src/syncRetrieve.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) 
+{
+  sDebug("%s, start to monitor:%s", pPeer->id, name);
+
+  if (pPeer->notifyFd <=0) {
+    pPeer->watchNum = 0;
+    pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+    if (pPeer->notifyFd < 0) {
+      sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+      return -1;
+    }
+
+    if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+    if (pPeer->watchFd == NULL) {
+      sError("%s, failed to allocate watchFd", pPeer->id);
+      return -1;
+    }
+
+    memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  }
+
+  int *wd = pPeer->watchFd + pPeer->watchNum;
+
+  if (*wd >= 0) {
+    if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
+      sError("%s, failed to remove wd:%d(%s)", pPeer->id, *wd, strerror(errno));
+      return -1;
+    }
+  }
+
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY);
+  if (*wd == -1) {
+    sError("%s, failed to add %s(%s)", pPeer->id, name, strerror(errno));
+    return -1;
+  }
+
+  pPeer->watchNum++;
+  pPeer->watchNum = (pPeer->watchNum +1) % tsMaxWatchFiles;
+
+  return 0;
+}
+
+static int syncAreFilesModified(SSyncPeer *pPeer) 
+{
+  if (pPeer->notifyFd <=0) return 0;
+
+  char buf[2048]; 
+  int len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  int code = 0; 
+  if (len >0) { 
+    sDebug("%s, processed file is changed", pPeer->id);    
+    pPeer->fileChanged = 1;
+    code = 1;
+  }
+
+  return code;  
+}
+
+static int syncRetrieveFile(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFileInfo   fileInfo;
+  SFileAck    fileAck;
+  int         code = -1;
+  char        name[TSDB_FILENAME_LEN * 2] = {0};
+
+  memset(&fileInfo, 0, sizeof(fileInfo));
+  memset(&fileAck, 0, sizeof(fileAck));
+
+  while (1) {
+    // retrieve file info
+    fileInfo.name[0] = 0;
+    fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion);   
+    //fileInfo.size = htonl(size);
+
+    // send the file info
+    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
+    if (ret < 0 ) break;
+
+    // if no file anymore, break
+    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { 
+      sDebug("%s, no more files to sync", pPeer->id);    
+      code = 0; break; 
+    }
+
+    // wait for the ack from peer
+    ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+
+    // set the peer sync version
+    pPeer->sversion = fileInfo.fversion;
+
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+    
+    // add the file into watch list
+    if ( syncAddIntoWatchList(pPeer, name) <0) break;
+
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      fileInfo.index++; 
+      sDebug("%s, %s is the same", pPeer->id, fileInfo.name);    
+      continue; 
+    }
+
+    // send the file to peer
+    int sfd = open(name, O_RDONLY);
+    if ( sfd < 0 ) break;
+
+    ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); 
+    close(sfd); 
+    if (ret <0) break;
+
+    sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size);    
+    fileInfo.index++; 
+
+    // check if processed files are modified 
+    if (syncAreFilesModified(pPeer) != 0) break;
+  }
+
+  if (code < 0) {
+    sError("%s, failed to retrieve file(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+/* if only a partial record is read out, set the IN_MODIFY flag in event,
+   so upper layer will reload the file to get a complete record */
+static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) 
+{ 
+  int ret;
+
+  ret = read(sfd, pHead, sizeof(SWalHead));
+  if (ret < 0) return -1;
+  if (ret == 0) return 0;
+
+  if (ret != sizeof(SWalHead)) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  ret = read(sfd, pHead->cont, pHead->len);
+  if (ret <0) return -1;
+
+  if (ret != pHead->len) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  return sizeof(SWalHead) + pHead->len;
+}    
+
+static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) 
+{ 
+  pPeer->watchNum = 0;
+  tclose(pPeer->notifyFd);
+  pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+  if (pPeer->notifyFd < 0) {
+    sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+  if (pPeer->watchFd == NULL) {
+    sError("%s, failed to allocate watchFd", pPeer->id);
+    return -1;
+  }
+
+  memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  int *wd = pPeer->watchFd;
+ 
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
+  if (*wd == -1) {
+    sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  return 0; 
+}
+
+static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) 
+{
+  char       buf[2048]; 
+  int  len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  if (len == 0) return 0;
+
+  struct inotify_event *event;
+  for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
+    event = (struct inotify_event *) ptr;
+    if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
+    if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
+  }
+
+  if (pEvent != 0)
+    sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
+
+  return 0;
+}
+
+static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) 
+{
+  SWalHead  *pHead = (SWalHead *) malloc(640000);
+  int        code = -1;
+  int32_t    bytes = 0;
+  int        sfd;
+
+  sfd = open(name, O_RDONLY);
+  if (sfd < 0) return -1;
+  lseek(sfd, offset, SEEK_SET);
+  sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
+
+  while (1) {
+    int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); 
+    if (wsize <0) break;
+    if (wsize == 0) { code = 0; break; }
+
+    sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+    if ( ret != wsize ) break;
+    pPeer->sversion = pHead->version;
+
+    bytes += wsize;
+ 
+    if (pHead->version >= fversion && fversion > 0) {
+      code = 0; 
+      bytes = 0; 
+      break;
+    }
+  }
+
+  free(pHead);
+  tclose(sfd); 
+
+  if (code == 0) return bytes;
+  return -1;
+}
+
+static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         code = -1;
+  char        fname[TSDB_FILENAME_LEN * 2];  // full path to wal file
+
+  if (syncAreFilesModified(pPeer) != 0) return -1;
+
+  while (1) {
+    int32_t  once = 0; // last WAL has once ever been processed 
+    int64_t  offset = 0;
+    uint64_t fversion = 0;
+    uint32_t event = 0;
+
+    // get full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+    sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
+
+    // monitor last wal
+    if (syncMonitorLastWal(pPeer, fname) <0) break;
+
+    while (1) {
+      int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
+      if (bytes < 0) break;
+
+      // check file changes
+      if (syncCheckLastWalChanges(pPeer, &event) <0) break;
+
+      // if file is not updated or updated once, set the fversion and sstatus
+      if (((event & IN_MODIFY) == 0) || once) {
+        if (fversion == 0) {
+          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
+          fversion = nodeVersion;    // must read data to fversion
+        }
+      }
+
+      // if all data up to fversion is read out, it is over
+      if (pPeer->sversion >= fversion && fversion > 0) {
+        code = 0; 
+        sDebug("%s, data up to fversion:%ld has been read out, bytes:%d", pPeer->id, fversion, bytes);
+        break;
+      }  
+
+      // if all data are read out, and no update
+      if ((bytes == 0) && ((event & IN_MODIFY) == 0)) {
+        // wal file is closed, break
+        if (event & IN_CLOSE_WRITE) { 
+          code = 0; 
+          sDebug("%s, current wal is closed", pPeer->id);
+          break;
+        }
+ 
+        // wal not closed, it means some data not flushed to disk, wait for a while
+        usleep(10000);
+      }
+
+      // if bytes>0, file is updated, or fversion is not reached but file still open, read again 
+      once = 1;
+      offset += bytes; 
+      sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes);
+      event = event & (~IN_MODIFY); // clear IN_MODIFY flag 
+    }
+
+    if (code < 0) break;
+    if (pPeer->sversion >= fversion && fversion > 0) break;  
+
+    index++;  wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);
+    if ( code < 0) break;  
+    if ( wname[0] == 0 ) {code = 0; break;}
+
+    // current last wal is closed, there is a new one 
+    sDebug("%s, last wal is closed, try new one", pPeer->id);
+  }
+
+  tclose(pPeer->notifyFd);
+
+  return code;
+}
+
+static int syncRetrieveWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  char        fname[TSDB_FILENAME_LEN * 3];
+  char        wname[TSDB_FILENAME_LEN * 2];
+  int32_t     size;
+  struct stat fstat;
+  int         code = -1;
+  uint32_t    index = 0;
+
+  while (1) {
+    // retrieve wal info
+    wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);   
+    if (code < 0) break;  // error
+    if (wname[0] == 0) {  // no wal file
+      sDebug("%s, no wal file", pPeer->id);
+      break;
+    }    
+      
+    if (code == 0) {   // last wal 
+      code = syncProcessLastWal(pPeer, wname, index);
+      break;
+    }
+
+    // get the full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+
+    // send wal file, 
+    // inotify is not required, old wal file won't be modified, even remove is ok
+    if ( stat(fname, &fstat) < 0 ) break;
+    size = fstat.st_size;
+
+    sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);    
+    int sfd = open(fname, O_RDONLY);
+    if (sfd < 0) break;
+
+    code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); 
+    close(sfd); 
+    if (code <0) break;
+
+    index++; 
+
+    if (syncAreFilesModified(pPeer) != 0) break; 
+  }
+
+  if (code == 0) {
+    sDebug("%s, wal retrieve is finished", pPeer->id);    
+    pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+    SWalHead walHead;
+    memset(&walHead, 0, sizeof(walHead));
+    code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
+  } else {
+    sError("%s, failed to send wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRetrieveDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+
+  if (write(pPeer->syncFd, (char *) &firstPkt, sizeof(firstPkt)) < 0) {
+    sError("%s, failed to send syncCmd", pPeer->id);
+    return -1;
+  }
+
+  pPeer->sversion = 0;  
+  pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
+  sDebug("%s, start to retrieve file", pPeer->id);
+  if (syncRetrieveFile(pPeer) < 0) {
+    sError("%s, failed to retrieve file", pPeer->id);
+    return -1;
+  }
+
+  // if no files are synced, there must be wal to sync, sversion must be larger than one
+  if (pPeer->sversion == 0) 
+    pPeer->sversion = 1;
+
+  sDebug("%s, start to retrieve wal", pPeer->id);
+  if (syncRetrieveWal(pPeer) < 0) {
+    sError("%s, failed to retrieve wal", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRetrieveData(void *param)
+{
+  SSyncPeer  *pPeer = (SSyncPeer *)param;
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  taosBlockSIGPIPE();
+
+  pPeer->fileChanged = 0;
+  pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (pPeer->syncFd < 0) {
+    sError("%s, failed to open socket to sync", pPeer->id);
+  } else {
+    sInfo("%s, sync tcp is setup", pPeer->id);
+  
+    if (syncRetrieveDataStepByStep(pPeer) == 0) {
+      sDebug("%s, sync retrieve process is successful", pPeer->id);
+    } else {
+      sError("%s, failed to retrieve data, restart connection", pPeer->id);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  if (pPeer->fileChanged) {
+    // if file is changed 3 times continuously, start flow control
+    pPeer->numOfRetrieves++;
+    if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl)  
+      (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2));
+  } else {
+    pPeer->numOfRetrieves = 0;
+    if (pNode->notifyFlowCtrl)
+      (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
+  }
+
+  pPeer->fileChanged = 0;
+  tclose(pPeer->notifyFd);
+  tclose(pPeer->syncFd);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}
diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c
new file mode 100644
index 0000000000000000000000000000000000000000..fa94caeed70488e696646f7467d819a323b43f04
--- /dev/null
+++ b/src/sync/src/taosTcpPool.c
@@ -0,0 +1,325 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tsocket.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+
+typedef struct SThreadObj {
+  pthread_t thread;
+  bool      stop;
+  int       pollFd;
+  int       numOfFds;
+  struct SPoolObj *pPool;
+} SThreadObj;
+
+typedef struct SPoolObj {
+  SPoolInfo    info;
+  SThreadObj **pThread;
+  pthread_t    thread;
+  int          nextId;
+  int          acceptFd;  // FD for accept new connection
+} SPoolObj;
+
+typedef struct {
+  SThreadObj *pThread;
+  void       *ahandle;
+  int         fd;
+  int         closedByApp; 
+} SConnObj;
+
+static void *taosAcceptPeerTcpConnection(void *argv);
+static void *taosProcessTcpData(void *param);
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool);
+static void taosStopPoolThread(SThreadObj* pThread);
+
+void *taosOpenTcpThreadPool(SPoolInfo *pInfo)
+{
+  pthread_attr_t thattr;
+
+  SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
+  if (pPool == NULL) {
+    uError("TCP server, no enough memory");
+    return NULL;
+  }
+
+  pPool->info = *pInfo;
+  
+  pPool->pThread = (SThreadObj **) calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
+  if (pPool->pThread == NULL) {
+    uError("TCP server, no enough memory");
+    free(pPool);
+    return NULL;
+  }
+
+  pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
+  if (pPool->acceptFd < 0) {
+    free(pPool->pThread); free(pPool);
+    uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
+    return NULL;
+  }
+
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&(pPool->thread), &thattr, (void *) taosAcceptPeerTcpConnection, pPool) != 0) {
+    uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
+    close(pPool->acceptFd);
+    free(pPool->pThread); free(pPool);
+    return NULL;
+  }
+
+  pthread_attr_destroy(&thattr);
+
+  uDebug("%p TCP pool is created", pPool);
+  return pPool;
+}
+
+void taosCloseTcpThreadPool(void *param)
+{
+  SPoolObj    *pPool = (SPoolObj *)param;
+  SThreadObj  *pThread;
+
+  shutdown(pPool->acceptFd, SHUT_RD);  
+  pthread_join(pPool->thread, NULL);
+
+  for (int i = 0; i < pPool->info.numOfThreads; ++i) {
+    pThread = pPool->pThread[i];
+    if (pThread) taosStopPoolThread(pThread); 
+  }
+
+  tfree(pPool->pThread);
+  free(pPool);
+  uDebug("%p TCP pool is closed", pPool);
+}
+
+void *taosAllocateTcpConn(void *param, void *pPeer, int connFd)
+{
+  struct epoll_event event;
+  SPoolObj *pPool = (SPoolObj *)param;
+
+  SConnObj *pConn = (SConnObj *) calloc(sizeof(SConnObj), 1);
+  if (pConn == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    return NULL;
+  }
+
+  SThreadObj *pThread = taosGetTcpThread(pPool);
+  if (pThread == NULL) {
+    free(pConn);
+    return NULL;
+  }
+
+  pConn->fd = connFd;
+  pConn->pThread = pThread;
+  pConn->ahandle = pPeer;
+  pConn->closedByApp = 0;
+
+  event.events = EPOLLIN | EPOLLRDHUP;
+  event.data.ptr = pConn;
+
+  if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
+    uError("failed to add fd:%d(%s)", connFd, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    free(pConn);
+    pConn = NULL;
+  } else {
+    pThread->numOfFds++;
+    uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
+  }
+
+  return pConn;
+}
+
+void taosFreeTcpConn(void *param)
+{
+  SConnObj *pConn = (SConnObj *)param;
+  SThreadObj   *pThread = pConn->pThread;
+
+  uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
+  pConn->closedByApp = 1;
+  shutdown(pConn->fd, SHUT_WR);
+}
+
+static void taosProcessBrokenLink(SConnObj *pConn) {
+  SThreadObj *pThread = pConn->pThread;
+  SPoolObj   *pPool = pThread->pPool;
+  SPoolInfo  *pInfo = &pPool->info;
+  
+  if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
+  (*pInfo->processBrokenLink)(pConn->ahandle);
+
+  pThread->numOfFds--;
+  epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
+  uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
+  tclose(pConn->fd);
+  free(pConn);
+}
+
+#define maxEvents 10
+
+static void *taosProcessTcpData(void *param) {
+  SThreadObj     *pThread = (SThreadObj *) param;
+  SPoolObj       *pPool = pThread->pPool;
+  SPoolInfo      *pInfo = &pPool->info;
+  SConnObj       *pConn = NULL;
+  struct epoll_event events[maxEvents];
+
+  void *buffer = malloc(pInfo->bufferSize);
+  taosBlockSIGPIPE();
+
+  while (1) {
+    if (pThread->stop) break; 
+    int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, -1);
+    if (pThread->stop) {
+      uDebug("%p TCP epoll thread is exiting...", pThread);
+      break;
+    }
+
+    if (fdNum < 0) { 
+      uError("epoll_wait failed (%s)", strerror(errno));
+      continue;
+    }
+
+    for (int i = 0; i < fdNum; ++i) {
+      pConn = events[i].data.ptr;
+      assert(pConn);
+
+      if (events[i].events & EPOLLERR) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLRDHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (pConn->closedByApp == 0) {
+        if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) {
+          taosFreeTcpConn(pConn);
+          continue;
+        }
+      } 
+    }
+  }
+
+  close(pThread->pollFd);
+  free(pThread);
+  free(buffer);
+  uDebug("%p TCP epoll thread exits", pThread);
+  return NULL;  
+}
+
+static void *taosAcceptPeerTcpConnection(void *argv) {
+  SPoolObj   *pPool = (SPoolObj *)argv;
+  SPoolInfo  *pInfo = &pPool->info;
+
+  taosBlockSIGPIPE();
+
+  while (1) {
+    struct sockaddr_in clientAddr;
+    socklen_t addrlen = sizeof(clientAddr);
+    int connFd = accept(pPool->acceptFd, (struct sockaddr *) &clientAddr, &addrlen);
+    if (connFd < 0) {
+      if (errno == EINVAL) {
+        uDebug("%p TCP server accept is exiting...", pPool);
+        break;
+      } else {
+        uError("TCP accept failure, reason:%s", strerror(errno));
+        continue;
+      }
+    }
+
+    //uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); 
+    taosKeepTcpAlive(connFd);
+    (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
+  }
+
+  tclose(pPool->acceptFd);
+  return NULL;
+}
+
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
+  SThreadObj *pThread = pPool->pThread[pPool->nextId];
+
+  if (pThread) return pThread;
+
+  pThread = (SThreadObj *) calloc(1, sizeof(SThreadObj));
+  if (pThread == NULL) return NULL;
+
+  pThread->pPool = pPool;
+  pThread->pollFd = epoll_create(10);  // size does not matter
+  if (pThread->pollFd < 0) {
+    free(pThread);
+    return NULL;
+  }
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  int ret = pthread_create(&(pThread->thread), &thattr, (void *) taosProcessTcpData, pThread);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    close(pThread->pollFd);
+    free(pThread);
+    return NULL;
+  }
+
+  uDebug("%p TCP epoll thread is created", pThread);
+  pPool->pThread[pPool->nextId] = pThread;
+  pPool->nextId++;
+  pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
+
+  return pThread;
+}
+
+static void taosStopPoolThread(SThreadObj* pThread) {
+  pThread->stop = true;
+  
+  if (pThread->thread == pthread_self()) {
+    pthread_detach(pthread_self());
+    return;
+  }
+
+  // save thread ID into a local variable, since pThread is freed when the thread exits 
+  pthread_t thread = pThread->thread;
+
+  // signal the thread to stop, try graceful method first,
+  // and use pthread_cancel when failed
+  struct epoll_event event = { .events = EPOLLIN };
+  eventfd_t fd = eventfd(1, 0);
+  if (fd == -1) {
+    // failed to create eventfd, call pthread_cancel instead, which may result in data corruption
+    uError("failed to create eventfd(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
+    uError("failed to call epoll_ctl(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  }
+
+  pthread_join(thread, NULL);
+  tclose(fd);
+}
+
diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c
new file mode 100644
index 0000000000000000000000000000000000000000..c308c2a4540f577c13e85b5cc8fc6094c92b60ff
--- /dev/null
+++ b/src/sync/src/tarbitrator.c
@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
+static void arbProcessBrokenLink(void *param);
+static int  arbProcessPeerMsg(void *param, void *buffer);
+static sem_t    tsArbSem;
+static ttpool_h tsArbTcpPool;
+
+typedef struct {
+  char  id[TSDB_EP_LEN+24];
+  int   nodeFd;
+  void *pConn;
+} SNodeConn;
+
+int main(int argc, char *argv[]) {
+  char     arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      tsServerPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      debugFlag = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-g")==0 && i < argc-1) {
+      if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue; 
+      tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath));
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", tsServerPort);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", debugFlag);
+      printf("  [-g logFilePath]: log file pathe, default:%s\n", arbLogPath);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+  
+ if (sem_init(&tsArbSem, 0, 0) != 0) {
+    printf("failed to create exit semphore\n");
+    exit(EXIT_FAILURE);
+  }
+
+  /* Set termination handler. */
+  struct sigaction act = {{0}};
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = arbSignalHandler;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  tsAsyncLog = 0;
+  strcat(arbLogPath, "/arbitrator.log");
+  taosInitLog(arbLogPath, 1000000, 10);
+
+  taosGetFqdn(tsNodeFqdn);
+  tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
+
+  SPoolInfo info;
+  info.numOfThreads = 1;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = arbProcessBrokenLink;
+  info.processIncomingMsg = arbProcessPeerMsg;
+  info.processIncomingConn = arbProcessIncommingConnection;
+  tsArbTcpPool = taosOpenTcpThreadPool(&info);
+  
+  if (tsArbTcpPool == NULL) {
+   sDebug("failed to open TCP thread pool, exit...");
+   return -1;
+  }
+
+  sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsServerPort);
+
+  for (int res = sem_wait(&tsArbSem); res != 0; res = sem_wait(&tsArbSem)) {
+    if (res != EINTR) break;
+  }
+
+  taosCloseTcpThreadPool(tsArbTcpPool);
+  sInfo("TAOS arbitrator is shut down\n");
+  closelog();
+
+  return 0;
+}
+
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp)
+{
+  char  ipstr[24];
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SNodeConn *pNode = (SNodeConn *) calloc(sizeof(SNodeConn), 1);
+  if (pNode == NULL) {
+    sError("failed to allocate memory(%s)", strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); 
+  if (firstPkt.syncHead.vgId) {  
+    sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
+    tfree(pNode);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  sDebug("%s, arbitrator request is accepted", pNode->id);
+  pNode->nodeFd = connFd;
+  pNode->pConn = taosAllocateTcpConn(tsArbTcpPool, pNode, connFd);
+
+  return;
+}
+
+static void arbProcessBrokenLink(void *param) {
+  SNodeConn *pNode = param;
+
+  sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
+  tfree(pNode);
+}
+
+static int arbProcessPeerMsg(void *param, void *buffer)
+{
+  SNodeConn  *pNode = param;
+  SSyncHead   head;
+  int         bytes = 0;
+  char       *cont = (char *)buffer;
+
+  int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
+  if (hlen != sizeof(head)) {
+    sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
+    return -1;
+  }
+
+  bytes = taosReadMsg(pNode->nodeFd, cont, head.len);
+  if (bytes != head.len) {
+    sDebug("%s, failed to read, bytes:%d len:%d", pNode->id, bytes, head.len);
+    return -1;
+  }
+
+  sDebug("%s, msg is received, len:%d", pNode->id, head.len);
+  return 0;
+}
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) {
+
+  struct sigaction act = {{0}};
+  act.sa_handler = SIG_IGN;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
+
+  // inform main thread to exit
+  sem_post(&tsArbSem);
+}
+
diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..89d739080f0bf327deca784350f58b8cc53b7933
--- /dev/null
+++ b/src/sync/test/CMakeLists.txt
@@ -0,0 +1,20 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(../inc)
+
+  LIST(APPEND CLIENT_SRC ./syncClient.c)
+  ADD_EXECUTABLE(syncClient ${CLIENT_SRC})
+  TARGET_LINK_LIBRARIES(syncClient sync trpc common)
+
+  LIST(APPEND SERVER_SRC ./syncServer.c)
+  ADD_EXECUTABLE(syncServer ${SERVER_SRC})
+  TARGET_LINK_LIBRARIES(syncServer sync trpc common)
+ENDIF ()
+
+
diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c
new file mode 100644
index 0000000000000000000000000000000000000000..cd873b758be0cc85857f22d9d0e18c5c47f8efc0
--- /dev/null
+++ b/src/sync/test/syncClient.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "trpc.h"
+#include "taoserror.h"
+
+typedef struct {
+  int       index;
+  SRpcEpSet epSet;
+  int       num;
+  int       numOfReqs;
+  int       msgSize;
+  sem_t     rspSem; 
+  sem_t    *pOverSem; 
+  pthread_t thread;
+  void     *pRpc;
+} SInfo;
+
+void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  SInfo *pInfo = (SInfo *)pMsg->ahandle;
+  uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
+
+  if (pEpSet) pInfo->epSet = *pEpSet;
+  rpcFreeCont(pMsg->pCont);
+
+  sem_post(&pInfo->rspSem); 
+}
+
+int tcount = 0;
+
+void *sendRequest(void *param) {
+  SInfo  *pInfo = (SInfo *)param;
+  SRpcMsg rpcMsg = {0}; 
+  
+  uDebug("thread:%d, start to send request", pInfo->index);
+
+  while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
+    pInfo->num++;
+    rpcMsg.pCont = rpcMallocCont(pInfo->msgSize);
+    rpcMsg.contLen = pInfo->msgSize;
+    rpcMsg.ahandle = pInfo;
+    rpcMsg.msgType = 1;
+    uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
+    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+    if ( pInfo->num % 20000 == 0 ) 
+      uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
+    sem_wait(&pInfo->rspSem);
+  }
+
+  uDebug("thread:%d, it is over", pInfo->index);
+  tcount++;
+
+  return NULL;
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit  rpcInit;
+  SRpcEpSet epSet;
+  char      secret[TSDB_KEY_LEN] = "mypassword";
+  int      msgSize = 128;
+  int      numOfReqs = 0;
+  int      appThreads = 1;
+  char     serverIp[40] = "127.0.0.1";
+  struct   timeval systemTime;
+  int64_t  startTime, endTime;
+  pthread_attr_t thattr;
+
+  // server info
+  epSet.numOfEps = 1;
+  epSet.inUse = 0;
+  epSet.port[0] = 7000;
+  epSet.port[1] = 7000;
+  strcpy(epSet.fqdn[0], serverIp);
+  strcpy(epSet.fqdn[1], "192.168.0.1");
+
+  // client info
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 0;
+  rpcInit.label        = "APP";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processResponse;
+  rpcInit.sessions     = 100;
+  rpcInit.idleTime     = tsShellActivityTimer*1000;
+  rpcInit.user         = "michael";
+  rpcInit.secret       = secret;
+  rpcInit.ckey         = "key";
+  rpcInit.spi          = 1;
+  rpcInit.connType     = TAOS_CONN_CLIENT;
+
+  for (int i=1; i<argc; ++i) { 
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      epSet.port[0] = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
+      strcpy(epSet.fqdn[0], argv[++i]); 
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-n")==0 && i < argc-1) {
+      numOfReqs = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-a")==0 && i < argc-1) {
+      appThreads = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-u")==0 && i < argc-1) {
+      rpcInit.user = argv[++i];
+    } else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
+      rpcInit.secret = argv[++i];
+    } else if (strcmp(argv[i], "-spi")==0 && i < argc-1) {
+      rpcInit.spi = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-i ip]: first server IP address, default is:%s\n", serverIp);
+      printf("  [-p port]: server port number, default is:%d\n", epSet.port[0]);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-a threads]: number of app threads, default is:%d\n", appThreads);
+      printf("  [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-u user]: user name for the connection, default is:%s\n", rpcInit.user);
+      printf("  [-k secret]: password for the connection, default is:%s\n", rpcInit.secret);
+      printf("  [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+
+  taosInitLog("client.log", 100000, 10);
+
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to initialize RPC");
+    return -1;
+  }
+
+  uInfo("client is initialized");
+
+  gettimeofday(&systemTime, NULL);
+  startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;
+
+  SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads);
+ 
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+  for (int i=0; i<appThreads; ++i) {
+    pInfo->index = i;
+    pInfo->epSet = epSet;
+    pInfo->numOfReqs = numOfReqs;
+    pInfo->msgSize = msgSize;
+    sem_init(&pInfo->rspSem, 0, 0);
+    pInfo->pRpc = pRpc;
+    pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo);
+    pInfo++;
+  }
+
+  do {
+    usleep(1);
+  } while ( tcount < appThreads);
+
+  gettimeofday(&systemTime, NULL);
+  endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;  
+  float usedTime = (endTime - startTime)/1000.0;  // mseconds
+
+  uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
+  uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
+
+  taosCloseLog();
+
+  return 0;
+}
+
+
diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c
new file mode 100644
index 0000000000000000000000000000000000000000..44019ad96ec2d3ce321db339578564fb74a9bca1
--- /dev/null
+++ b/src/sync/test/syncServer.c
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include <stdint.h>
+#include "os.h"
+#include "tulog.h"
+#include "tglobal.h"
+#include "tsocket.h"
+#include "trpc.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+
+int msgSize = 128;
+int commit = 0;
+int dataFd = -1;
+void *qhandle = NULL;
+int walNum = 0;
+uint64_t  tversion = 0;
+void *syncHandle;
+int   role;
+int  nodeId;
+char path[256];
+int  numOfWrites ;
+SSyncInfo syncInfo;
+SSyncCfg *pCfg;
+
+int writeIntoWal(SWalHead *pHead)
+{ 
+  if (dataFd < 0) {
+    char  walName[280];
+    snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum);
+    remove(walName);
+    dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);  
+    if (dataFd < 0) { 
+      uInfo("failed to open wal file:%s(%s)", walName, strerror(errno));
+      return -1;
+    } else {
+      walNum++;
+      uInfo("file:%s is opened to write, walNum:%d", walName, walNum);
+    }
+  }
+
+  if (write(dataFd, pHead, sizeof(SWalHead) + pHead->len) < 0) {
+    uError("ver:%" PRIu64 ", failed to write wal file(%s)", pHead->version, strerror(errno));
+  } else {
+    uDebug("ver:%" PRIu64 ", written to wal", pHead->version);
+  }
+
+  numOfWrites++;
+  if (numOfWrites >= 10000) {
+    uInfo("%d request have been written into disk", numOfWrites);
+    close(dataFd);
+    dataFd = -1;
+    numOfWrites = 0;
+  }
+ 
+  return 0;
+}
+
+void confirmForward(void *ahandle, void *mhandle, int32_t code)
+{
+  SRpcMsg  *pMsg = (SRpcMsg *)mhandle;
+  SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+
+  uDebug("ver:%" PRIu64 ", confirm is received", pHead->version);
+
+  rpcFreeCont(pMsg->pCont);
+
+  SRpcMsg    rpcMsg;
+  rpcMsg.pCont = rpcMallocCont(msgSize);
+  rpcMsg.contLen = msgSize;
+  rpcMsg.handle = pMsg->handle;
+  rpcMsg.code = code;
+  rpcSendResponse(&rpcMsg);
+
+  taosFreeQitem(mhandle); 
+}
+
+int processRpcMsg(void *item) {
+  SRpcMsg   *pMsg = (SRpcMsg *)item;
+  SWalHead  *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+  int        code = -1;
+
+  if (role != TAOS_SYNC_ROLE_MASTER) {
+    uError("not master, write failed, role:%s", syncRole[role]);
+  } else {
+
+    pHead->version = ++tversion;
+    pHead->msgType = pMsg->msgType;
+    pHead->len = pMsg->contLen;
+
+    uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version);
+    writeIntoWal(pHead); 
+    syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC);
+
+    code = 0;
+  }
+
+  if (pCfg->quorum <= 1) { 
+    taosFreeQitem(item); 
+    rpcFreeCont(pMsg->pCont);
+
+    SRpcMsg    rpcMsg;
+    rpcMsg.pCont = rpcMallocCont(msgSize);
+    rpcMsg.contLen = msgSize;
+    rpcMsg.handle = pMsg->handle;
+    rpcMsg.code = code;
+    rpcSendResponse(&rpcMsg);
+  }
+
+  return code;
+}
+
+int processFwdMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", forward is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  }
+
+  uDebug("ver:%" PRIu64 ", forward from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  if (pCfg->quorum > 1) syncConfirmForward(syncHandle, pHead->version, 0);
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+int processWalMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", wal is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  };
+
+  uDebug("ver:%" PRIu64 ", wal from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+void *processWriteQueue(void *param) {
+  int        type;
+  void      *item;
+
+  while (1) {
+    int ret = taosReadQitem(qhandle, &type, &item);
+    if (ret <= 0) {
+      usleep(1000);
+      continue;
+    }     
+
+    if (type == TAOS_QTYPE_RPC) {
+      processRpcMsg(item);
+    } else if (type == TAOS_QTYPE_WAL) {
+      processWalMsg(item);
+    } else if (type == TAOS_QTYPE_FWD) {
+      processFwdMsg(item);
+    } 
+
+  }
+
+  return NULL;
+}
+
+int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
+  // app shall retrieve the auth info based on meterID from DB or a data file
+  // demo code here only for simple demo
+  int ret = 0;
+
+  if (strcmp(meterId, "michael") == 0) {
+    *spi = 1;
+    *encrypt = 0;
+    strcpy(secret, "mypassword");
+    strcpy(ckey, "key");
+  } else if (strcmp(meterId, "jeff") == 0) {
+    *spi = 0;
+    *encrypt = 0;
+  } else {
+    ret = -1;  // user not there
+  }
+
+  return ret;
+}
+
+void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  
+  SRpcMsg *pTemp;
+
+  pTemp = taosAllocateQitem(sizeof(SRpcMsg));
+  memcpy(pTemp, pMsg, sizeof(SRpcMsg));
+  
+  uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen);
+  taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); 
+}
+
+uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) 
+{
+  uint32_t     magic;
+  struct stat  fstat;
+  char         aname[280];
+
+  if (*index == 2) {
+    uInfo("wait for a while .....");
+    sleep(3);
+  }
+
+  if (name[0] == 0) {
+    // find the file 
+    snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index);
+    sprintf(name, "data/data.%d", *index); 
+  } else {
+    snprintf(aname, sizeof(aname), "%s/%s", path, name);
+  }
+
+  uInfo("get file info:%s", aname);
+  if ( stat(aname, &fstat) < 0 ) return 0; 
+
+  *size = fstat.st_size;
+  magic = fstat.st_size;
+
+  return magic;
+}
+
+int  getWalInfo(void *ahandle, char *name, uint32_t *index) {
+
+  struct stat  fstat;
+  char         aname[280];
+
+  name[0] = 0;
+  if (*index + 1> walNum) return 0;
+
+  snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index);
+  sprintf(name, "wal/wal.%d", *index); 
+  uInfo("get wal info:%s", aname);
+
+  if ( stat(aname, &fstat) < 0 ) return -1;
+
+  if (*index >= walNum-1) return 0;  // no more
+
+  return 1;
+
+}
+
+int writeToCache(void *ahandle, void *data, int type) {
+  SWalHead *pHead = data;
+
+  uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type);
+
+  int   msgSize = pHead->len + sizeof(SWalHead);
+  void *pMsg = taosAllocateQitem(msgSize);
+  memcpy(pMsg, pHead, msgSize);
+  taosWriteQitem(qhandle, type, pMsg); 
+
+  return 0;
+}
+
+void confirmFwd(void *ahandle, int64_t version) {
+
+  return;
+}
+
+void notifyRole(void *ahandle, int8_t r) {
+  role = r;
+  printf("current role:%s\n", syncRole[role]);
+}
+
+
+void initSync() {
+
+  pCfg->replica = 1;
+  pCfg->quorum = 1;
+  syncInfo.vgId = 1;
+  syncInfo.ahandle = &syncInfo;
+  syncInfo.getFileInfo = getFileInfo;
+  syncInfo.getWalInfo = getWalInfo;
+  syncInfo.writeToCache = writeToCache;
+  syncInfo.confirmForward = confirmForward;
+  syncInfo.notifyRole = notifyRole;
+
+  pCfg->nodeInfo[0].nodeId = 1;
+  pCfg->nodeInfo[0].nodePort = 7010;
+  taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  pCfg->nodeInfo[1].nodeId = 2;
+  pCfg->nodeInfo[1].nodePort = 7110;
+  taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
+
+  pCfg->nodeInfo[2].nodeId = 3;
+  pCfg->nodeInfo[2].nodePort = 7210;
+  taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+
+  pCfg->nodeInfo[3].nodeId = 4;
+  pCfg->nodeInfo[3].nodePort = 7310;
+  taosGetFqdn(pCfg->nodeInfo[3].nodeFqdn);
+
+  pCfg->nodeInfo[4].nodeId = 5;
+  pCfg->nodeInfo[4].nodePort = 7410;
+  taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn);
+}
+
+void doSync()
+{
+  for (int i=0; i<5; ++i) {
+    if (tsSyncPort == pCfg->nodeInfo[i].nodePort) 
+      nodeId = pCfg->nodeInfo[i].nodeId;
+  }
+
+  snprintf(path, sizeof(path), "/root/test/d%d", nodeId);
+  strcpy(syncInfo.path, path);
+
+  if ( syncHandle == NULL) {
+      syncHandle = syncStart(&syncInfo);
+  } else {
+      if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL;
+  }
+
+  uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort);
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit rpcInit;
+  char     dataName[20] = "server.data";
+  pCfg =  &syncInfo.syncCfg;
+
+  initSync();
+
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 7000;
+  rpcInit.label        = "SER";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processRequestMsg;
+  rpcInit.sessions     = 1000;
+  rpcInit.idleTime     = tsShellActivityTimer*1500; 
+  rpcInit.afp          = retrieveAuthInfo;
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      rpcInit.localPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-w")==0 && i < argc-1) {
+      commit = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
+      syncInfo.version = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
+      pCfg->replica = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q")==0 && i < argc-1) {
+      pCfg->quorum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", rpcInit.localPort);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit);
+      printf("  [-v version]: initial node version, default is:%ld\n", syncInfo.version);
+      printf("  [-r replica]: replicacation number, default is:%d\n", pCfg->replica);
+      printf("  [-q quorum]: quorum, default is:%d\n", pCfg->quorum);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+ 
+  uDebugFlag = rpcDebugFlag;
+  dDebugFlag = rpcDebugFlag; 
+  //tmrDebugFlag = rpcDebugFlag; 
+  tsAsyncLog = 0;
+  taosInitLog("server.log", 1000000, 10);
+
+  rpcInit.connType = TAOS_CONN_SERVER;
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to start RPC server");
+    return -1;
+  }
+
+  tsSyncPort = rpcInit.localPort + 10;
+  qhandle = taosOpenQueue();
+
+  doSync();
+
+  pthread_attr_t thattr;
+  pthread_t      thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&thread, &thattr, processWriteQueue, NULL) != 0) {
+    uError("failed to create thread, reason:%s", strerror(errno));
+    return -1;
+  }
+
+  printf("server is running, localPort:%d\n", rpcInit.localPort);
+  SNodesRole nroles;
+
+  while (1) {
+    char c = getchar();
+
+    switch(c) {
+      case '1':
+        pCfg->replica = 1; doSync();
+        break;        
+      case '2':
+        pCfg->replica = 2; doSync();
+        break;
+      case '3':
+        pCfg->replica = 3; doSync();
+        break;
+      case '4':
+        pCfg->replica = 4; doSync();
+        break;
+      case '5':
+        pCfg->replica = 5; doSync();
+        break;
+      case 's':
+        syncGetNodesRole(syncHandle, &nroles);
+        for (int i=0; i<pCfg->replica; ++i) 
+          printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]);
+        break;
+      default:
+        break;
+    }
+
+    if (c=='q') break;
+
+  }
+
+  syncStop(syncHandle);
+
+  if (dataFd >= 0) {
+    close(dataFd);
+    remove(dataName);
+  }
+
+  return 0;
+}
+
+
diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt
index b2154969d6209243511768f43686e2b47d787936..450afe337bb287f204567e743d1aee5b37f86ba3 100644
--- a/src/tsdb/CMakeLists.txt
+++ b/src/tsdb/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   # INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/common/inc)
diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h
index 3e92c017651fb19e59eb6b144b052f721a296c65..6a54cb23998ff48ffeb2acb552e6ce5f7a6cbeac 100644
--- a/src/tsdb/inc/tsdbMain.h
+++ b/src/tsdb/inc/tsdbMain.h
@@ -483,7 +483,7 @@ void tsdbDestroyHelper(SRWHelper* pHelper);
 void tsdbResetHelper(SRWHelper* pHelper);
 int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
 int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError);
-void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
+int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
 int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
 int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
 int  tsdbWriteCompInfo(SRWHelper* pHelper);
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index f69887869862c913002c6d553454d3c43168e835..6e7b39830eaea94b322cc44963f8569a453f019c 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -12,23 +12,14 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <libgen.h>
+#define _DEFAULT_SOURCE
+#include "os.h"
 
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tsdbMain.h"
 #include "tutil.h"
 #include "ttime.h"
-#include "tfile.h"
 
 #ifdef TSDB_IDX
 const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"};
@@ -317,7 +308,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
-  if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -383,7 +374,7 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
   pFile->fd = -1;
   if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
 
-  if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index 89a19c15af7df11f119720832ef7fd09613a62a3..b04b0be9bad1b7cac44227a0ade32ed77175f0ea 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -12,6 +12,10 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+// no test file errors here
+#undef TAOS_RANDOM_FILE_FAIL
+
 #include "tsdbMain.h"
 #include "os.h"
 #include "talgo.h"
@@ -22,8 +26,6 @@
 #include "ttime.h"
 #include "tulog.h"
 
-#include <pthread.h>
-#include <sys/stat.h>
 
 #define TSDB_CFG_FILE_NAME "config"
 #define TSDB_DATA_DIR_NAME "data"
@@ -545,7 +547,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
 
-  if (twrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -586,7 +588,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
     goto _err;
   }
 
-  if (tread(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -788,7 +790,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
     for (int i = 1; i < pMeta->maxTables; i++) {
       STable *pTable = pMeta->tables[i];
       if (pTable == NULL) continue;
-      tsdbSetHelperTable(&rhelper, pTable, pRepo);
+      if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err;
       SCompIdx *pIdx = &(rhelper.curCompIdx);
 
       if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey;
diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c
index a65d956eb07d17cd928641de373222fcf6f1b22f..506f3c11c104bed4a84641f7d95538a62bdbfec7 100644
--- a/src/tsdb/src/tsdbMemTable.c
+++ b/src/tsdb/src/tsdbMemTable.c
@@ -610,10 +610,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
 
     taosRLockLatch(&(pIter->pTable->latch));
 
-    tsdbSetHelperTable(pHelper, pIter->pTable, pRepo);
+    if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
 
     if (pIter->pIter != NULL) {
-      tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1));
+      if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        goto _err;
+      }
 
       if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
         taosRUnLockLatch(&(pIter->pTable->latch));
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index 09bbbd8f4d199f994fdde6623fae75f65a628f24..d389a96feeeaa2dc32174b3e7370cc61b87f7f97 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -69,9 +69,9 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
 
   if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) {
     if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) {
-      tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
-                TABLE_TID(pTable), TABLE_UID(pTable));
-      return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+      tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo),
+                TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid]));
+      return 0;
     } else {
       tsdbError("vgId:%d table %s at tid %d uid %" PRIu64
                 " exists, replace it with new table, this can be not reasonable",
@@ -1295,4 +1295,4 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
   tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables);
 
   return 0;
-}
\ No newline at end of file
+}
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
index 28f297457334f847f0f58c04825ad71c39ff63d2..8d3908666ed0cd9a517fabec3b7e7ea20a0885c7 100644
--- a/src/tsdb/src/tsdbRWHelper.c
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tscompression.h"
 #include "tsdbMain.h"
-#include "tfile.h"
 
 #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
 #define TSDB_KEY_COL_OFFSET 0
@@ -262,7 +262,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
   return 0;
 }
 
-void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
+int tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
   ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD));
 
   // Clear members and state used by previous table
@@ -273,8 +273,15 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
   pHelper->tableInfo.uid = pTable->tableId.uid;
   STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
 
-  tdInitDataCols(pHelper->pDataCols[0], pSchema);
-  tdInitDataCols(pHelper->pDataCols[1], pSchema);
+  if (tdInitDataCols(pHelper->pDataCols[0], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (tdInitDataCols(pHelper->pDataCols[1], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
 
   if (pHelper->idxH.numOfIdx > 0) {
     while (true) {
@@ -309,6 +316,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
 
   helperSetState(pHelper, TSDB_HELPER_TABLE_SET);
   ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1));
+
+  return 0;
 }
 
 int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
@@ -379,7 +388,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
         return -1;
       }
 
-      if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
+      if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
         tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
                   helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -424,7 +433,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     pIdx->tid = pHelper->tableInfo.tid;
     ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-    if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+    if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
       tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                 pFile->fname, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -481,7 +490,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
 
   pFile->info.offset = offset;
 
-  if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
+  if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -514,7 +523,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
         return -1;
       }
 
-      if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
+      if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
                   pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -585,7 +594,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) {
       }
 
       pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len);
-      if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+      if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                   helperHeadF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -626,7 +635,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
     return -1;
   }
 
-  if (tread(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
+  if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
     tsdbError("vgId:%d failed to read %zu bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -841,7 +850,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
                                        sizeof(TSCKSUM));
 
   // Write the whole block to file
-  if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+  if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1222,7 +1231,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl
     return -1;
   }
 
-  if (tread(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
+  if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1337,7 +1346,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-  if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+  if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index c0f6c82b99728b02d276c1b3328e806212756b67..0e3a657fde547f186623734e9dcf24b9087e9f27 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -30,11 +30,6 @@
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)
 #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
 
-enum {
-  QUERY_RANGE_LESS_EQUAL = 0,
-  QUERY_RANGE_GREATER_EQUAL = 1,
-};
-
 enum {
   TSDB_QUERY_TYPE_ALL      = 1,
   TSDB_QUERY_TYPE_LAST     = 2,
@@ -210,7 +205,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   if (pQueryHandle->pColumns == NULL) {
     goto out_of_memory;
   }
-  
+
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {{0}, 0};
   
@@ -222,29 +217,29 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
     taosArrayPush(pQueryHandle->pColumns, &colInfo);
     pQueryHandle->statis[i].colId = colInfo.info.colId;
   }
-  
+
   pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
   if (pQueryHandle->pTableCheckInfo == NULL) {
     goto out_of_memory;
   }
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL);
-  
+
   for (int32_t i = 0; i < sizeOfGroup; ++i) {
     SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
-    
+
     size_t gsize = taosArrayGetSize(group);
     assert(gsize > 0);
-    
+
     for (int32_t j = 0; j < gsize; ++j) {
       STable* pTable = (STable*) taosArrayGetP(group, j);
-      
+
       STableCheckInfo info = {
           .lastKey = pQueryHandle->window.skey,
           .tableId = pTable->tableId,
           .pTableObj = pTable,
       };
-      
+
       assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
       info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
 
@@ -280,17 +275,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
 
 SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   assert(pHandle != NULL);
-  
+
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle;
-  
+
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   SArray* res = taosArrayInit(size, POINTER_BYTES);
-  
+
   for(int32_t i = 0; i < size; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     taosArrayPush(res, &pCheckInfo->pTableObj);
   }
-  
+
   return res;
 }
 
@@ -306,11 +301,11 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond
 static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) {
   STable* pTable = pCheckInfo->pTableObj;
   assert(pTable != NULL);
-  
+
   if (pCheckInfo->initBuf) {
     return true;
   }
-  
+
   pCheckInfo->initBuf = true;
   int32_t order = pHandle->order;
 
@@ -318,7 +313,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pHandle->mem == NULL && pHandle->imem == NULL) {
     return false;
   }
-  
+
   assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
 
   // TODO: add uid check
@@ -338,17 +333,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) {
     return false;
   }
-  
+
   bool memEmpty  = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
   bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
   if (memEmpty && imemEmpty) { // buffer is empty
     return false;
   }
-  
+
   if (!memEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -357,11 +352,11 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   if (!imemEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -370,7 +365,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   return true;
 }
 
@@ -473,7 +468,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
   pHandle->cur.fid = -1;
-  
+
   STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
 
   STable* pTable = pCheckInfo->pTableObj;
@@ -491,17 +486,17 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   pCheckInfo->lastKey = dataRowKey(row);  // first timestamp in buffer
   tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle,
       pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo);
-  
+
   // all data in mem are checked already.
   if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) ||
       (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_TRAVERSE(pHandle->order))) {
     return false;
   }
-  
+
   int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
   STimeWindow* win = &pHandle->cur.win;
   pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle);
-  
+
   // update the last key value
   pCheckInfo->lastKey = win->ekey + step;
   pHandle->cur.lastKey = win->ekey + step;
@@ -510,7 +505,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   if (!ASCENDING_TRAVERSE(pHandle->order)) {
     SWAP(win->skey, win->ekey, TSKEY);
   }
-  
+
   return true;
 }
 
@@ -519,31 +514,31 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   if (key == TSKEY_INITIAL_VAL) {
     return INT32_MIN;
   }
-  
+
   int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
     fid = INT32_MIN;
   }
-  
+
   if (fid > 0L && fid > INT32_MAX) {
     fid = INT32_MAX;
   }
-  
+
   return fid;
 }
 
 static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
   int32_t firstSlot = 0;
   int32_t lastSlot = numOfBlocks - 1;
-  
+
   int32_t midSlot = firstSlot;
-  
+
   while (1) {
     numOfBlocks = lastSlot - firstSlot + 1;
     midSlot = (firstSlot + (numOfBlocks >> 1));
-    
+
     if (numOfBlocks == 1) break;
-    
+
     if (skey > pBlock[midSlot].keyLast) {
       if (numOfBlocks == 2) break;
       if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break;
@@ -555,7 +550,7 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK
       break;  // got the slot
     }
   }
-  
+
   return midSlot;
 }
 
@@ -644,7 +639,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
                     .uid = (_checkInfo)->tableId.uid})
 
 
-static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
+static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
   STsdbRepo *pRepo = pQueryHandle->pTsdb;
   bool       blockLoaded = false;
   int64_t    st = taosGetTimestampUs();
@@ -678,8 +673,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 
   int64_t elapsedTime = (taosGetTimestampUs() - st);
   pQueryHandle->cost.blockLoadTime += elapsedTime;
-  tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, elapsedTime);
 
+  tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p",
+      pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo);
   return blockLoaded;
 }
 
@@ -692,18 +688,17 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
 
   TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
-  
+
   if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
       (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-    
+
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
         (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
 
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
 
-      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step,
-                                        pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
+      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
       pQueryHandle->realNumOfRows = cur->rows;
 
       // update the last key value
@@ -711,13 +706,13 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
       if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
         SWAP(cur->win.skey, cur->win.ekey, TSKEY);
       }
-      
+
       cur->mixBlock = true;
       cur->blockCompleted = false;
       return;
     }
-  
-    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
+
+    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot);
     doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
   } else {
     /*
@@ -742,39 +737,41 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-    // query ended in current block
+    // query ended in/started from current block
     if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
 
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
-      
+
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
         cur->pos =
             binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = 0;
       }
-      
+
+      assert(pCheckInfo->lastKey <= pBlock->keyLast);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {  // the whole block is loaded in to buffer
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
     }
   } else {  //desc order, query ended in current block
     if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
-  
+
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
         cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
-      
+
+      assert(pCheckInfo->lastKey >= pBlock->keyFirst);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
@@ -790,7 +787,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
   TSKEY* keyList;
 
   assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
-  
+
   if (num <= 0) return -1;
 
   keyList = (TSKEY*)pValue;
@@ -849,13 +846,19 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
   char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
-  
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
-  
+
   int32_t num = end - start + 1;
+  assert(num >= 0);
+
+  if (num == 0) {
+    return numOfRows;
+  }
+
   int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-  
+
   //data in buffer has greater timestamp, copy data in file block
   int32_t i = 0, j = 0;
   while(i < requiredNumOfCols && j < pCols->numOfCols) {
@@ -928,7 +931,7 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
 
     i++;
   }
-  
+
   pQueryHandle->cur.win.ekey = tsArray[end];
   pQueryHandle->cur.lastKey = tsArray[end] + step;
 
@@ -995,24 +998,99 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
   }
 }
 
+static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) {
+  if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    return;
+  }
+
+  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+  if (numOfRows < pQueryHandle->outputCapacity) {
+    int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
+    for(int32_t i = 0; i < numOfCols; ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+      memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+    }
+  }
+}
+
+static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos,
+    int32_t numOfExisted, int32_t *start, int32_t *end) {
+  *start = -1;
+
+  if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    int32_t remain = endPos - startPos + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1;
+    } else {
+      *end = endPos;
+    }
+
+    *start = startPos;
+  } else {
+    int32_t remain = (startPos - endPos) + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted);
+    } else {
+      *end = endPos;
+    }
+
+    *start = *end;
+    *end = startPos;
+  }
+}
+
+static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  pCheckInfo->lastKey = cur->lastKey;
+  pQueryHandle->realNumOfRows = numOfRows;
+  cur->rows = numOfRows;
+  cur->pos = endPos;
+}
+
+static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  if (cur->rows > 0) {
+    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey);
+    } else {
+      assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey);
+    }
+
+    SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0);
+    assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]);
+  } else {
+    cur->win = pQueryHandle->window;
+
+    int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
+    cur->lastKey = pQueryHandle->window.ekey + step;
+  }
+}
+
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
 static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
-  
+
   initTableMemIterator(pQueryHandle, pCheckInfo);
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
+      cur->pos >= 0 && cur->pos < pBlock->numOfRows);
+
+  TSKEY* tsArray = pCols->cols[0].pData;
 
   // for search the endPos, so the order needs to reverse
   int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
 
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
-  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+  int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
 
   STable* pTable = pCheckInfo->pTableObj;
-
   int32_t endPos = cur->pos;
+
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
     endPos = blockInfo.rows - 1;
     cur->mixBlock = (cur->pos != 0);
@@ -1024,48 +1102,36 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
     cur->mixBlock = true;
   }
-  
+
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
-  int32_t pos = cur->pos;
-  
-  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == 0);
-  TSKEY* tsArray = pCols->cols[0].pData;
-  
   int32_t numOfRows = 0;
-  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
+  int32_t pos = cur->pos;
+  cur->win = TSWINDOW_INITIALIZER;
 
   // no data in buffer, load data from file directly
   if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
     int32_t start = cur->pos;
     int32_t end = endPos;
+
     if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
-      end = cur->pos;
-      start = endPos;
-    }
-    
-    cur->win.skey = tsArray[start];
-    cur->win.ekey = tsArray[end];
-    
-    // todo opt in case of no data in buffer
+      SWAP(start, end, int32_t);
+    }
+
     numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-    
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
 
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
-    }
-  
+    // the time window should always be right order: skey <= ekey
+    cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]};
+    cur->lastKey = tsArray[endPos];
     pos += (end - start + 1) * step;
-    cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-        ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
-    
-    pCheckInfo->lastKey = cur->lastKey;
-    pQueryHandle->realNumOfRows = numOfRows;
-    cur->rows = numOfRows;
+
+    cur->blockCompleted =
+        (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+         ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+    moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+    updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+    doCheckGeneratedBlockRange(pQueryHandle);
     return;
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
@@ -1111,30 +1177,18 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
           moveToNextRowInMem(pCheckInfo);
         }
-        
-        int32_t start = -1;
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = end - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
 
-          start = pos;
-        } else {
-          int32_t remain = (pos - end) + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
+        int32_t qstart = 0, qend = 0;
+        getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend);
 
-          start = end;
-          end = pos;
-        }
+        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend);
+        pos += (qend - qstart + 1) * step;
 
-        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-        pos += (end - start + 1) * step;
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart];
+        cur->lastKey  = cur->win.ekey + step;
       }
     } while (numOfRows < pQueryHandle->outputCapacity);
-    
+
     if (numOfRows < pQueryHandle->outputCapacity) {
       /**
        * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
@@ -1148,54 +1202,29 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
           cur->win.skey = tsArray[pos];
         }
 
-        int32_t start = -1;
-        int32_t end = -1;
-
-        // all remain data are qualified, but check the remain capacity in the first place.
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = endPos - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
-
-          start = pos;
-          end = endPos;
-        } else {
-          int32_t remain = pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
-
-          start = endPos;
-          end = pos;
-        }
+        int32_t start = -1, end = -1;
+        getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end);
 
         numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
+
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start];
+        cur->lastKey  = cur->win.ekey + step;
       }
     }
   }
-  
-  cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-      ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+  cur->blockCompleted =
+      (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+       ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
 
   if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
     SWAP(cur->win.skey, cur->win.ekey, TSKEY);
-  
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
-    }
   }
-  
-  pCheckInfo->lastKey = cur->lastKey;
-  pQueryHandle->realNumOfRows = numOfRows;
-  cur->rows = numOfRows;
-  cur->pos = pos;
+
+  moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+  updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+  doCheckGeneratedBlockRange(pQueryHandle);
 
   tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey,
       cur->win.ekey, cur->rows, pQueryHandle->qinfo);
@@ -1332,16 +1361,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     cleanBlockOrderSupporter(&sup, 0);
     return TSDB_CODE_TDB_OUT_OF_MEMORY;
   }
-  
+
   int32_t cnt = 0;
   int32_t numOfQualTables = 0;
-  
+
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j);
     if (pTableCheck->numOfBlocks <= 0) {
       continue;
     }
-    
+
     SCompBlock* pBlock = pTableCheck->pCompInfo->blocks;
     sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks;
 
@@ -1437,37 +1466,39 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
-    (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
-      tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo)
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
+                pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
       pQueryHandle->pFileGroup = NULL;
+      assert(pQueryHandle->numOfBlocks == 0);
       break;
     }
 
     if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
-    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks,
-           numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
-    
+
+    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
+              pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
+
     assert(numOfBlocks >= 0);
     if (numOfBlocks == 0) {
       continue;
     }
-    
+
     // todo return error code to query engine
-    if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) {
+    if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
+
     assert(numOfBlocks >= pQueryHandle->numOfBlocks);
     if (pQueryHandle->numOfBlocks > 0) {
       break;
     }
   }
-  
+
   // no data in file anymore
-  if (pQueryHandle->numOfBlocks <= 0) {
+  if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) {
     if (code == TSDB_CODE_SUCCESS) {
       assert(pQueryHandle->pFileGroup == NULL);
     }
@@ -1476,10 +1507,11 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
     *exists = false;
     return code;
   }
-  
+
+  assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0);
   cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1;
   cur->fid = pQueryHandle->pFileGroup->fileId;
-  
+
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
   *exists = loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo);
 
@@ -1495,7 +1527,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     pQueryHandle->locateStart = true;
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
     int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
-    
+
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
 
@@ -1504,7 +1536,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     // check if current file block is all consumed
     STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
     STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
-    
+
     // current block is done, try next
     if (!cur->mixBlock || cur->blockCompleted) {
       if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
@@ -1515,10 +1547,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
         // next block of the same file
         int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
         cur->slot += step;
-        
+
         cur->mixBlock = false;
         cur->blockCompleted = false;
-        
+
         STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
         *exists = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo);
 
@@ -1540,10 +1572,10 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     if (hasMoreDataInCache(pQueryHandle)) {
       return true;
     }
-    
+
     pQueryHandle->activeIndex += 1;
   }
-  
+
   return false;
 }
 
@@ -1561,14 +1593,14 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
     pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
     pQueryHandle->order = TSDB_ORDER_DESC;
-    
+
     if (!tsdbNextDataBlock(pHandle)) {
       return false;
     }
-    
+
     /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
     /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
-  
+
     if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
       // data already retrieve, discard other data rows and return
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
@@ -1576,7 +1608,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
       }
-  
+
       pQueryHandle->cur.win  = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
       pQueryHandle->window   = pQueryHandle->cur.win;
       pQueryHandle->cur.rows = 1;
@@ -1593,7 +1625,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pSecQueryHandle->checkFiles  = true;
       pSecQueryHandle->activeIndex = 0;
       pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock;
-  
+
       if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) {
         free(pSecQueryHandle);
         return false;
@@ -1603,24 +1635,24 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       // allocate buffer in order to load data blocks from file
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
-  
+
       pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
       pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
 
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData colInfo = {{0}, 0};
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
-        
+
         colInfo.info = pCol->info;
         colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes);
         taosArrayPush(pSecQueryHandle->pColumns, &colInfo);
       }
-  
+
       size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
       pSecQueryHandle->pTableCheckInfo = taosArrayInit(si, sizeof(STableCheckInfo));
       STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
       assert(pMeta != NULL);
-  
+
       for (int32_t j = 0; j < si; ++j) {
         STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
         STableCheckInfo info = {
@@ -1628,10 +1660,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
             .tableId = pCheckInfo->tableId,
             .pTableObj = pCheckInfo->pTableObj,
         };
-        
+
         taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info);
       }
-  
+
       tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo);
       tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo);
       pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn);
@@ -1641,17 +1673,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
       tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
-  
+
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
-  
+
         SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
         assert(pCol->info.colId == pCol1->info.colId);
-        
+
         memcpy(pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
       }
-  
+
       SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
 
       // it is ascending order
@@ -1675,7 +1707,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
     pQueryHandle->checkFiles = false;
     return true;
   }
-  
+
   if (pQueryHandle->checkFiles) {
     bool exists = true;
     int32_t code = getDataBlocksInFiles(pQueryHandle, &exists);
@@ -1688,11 +1720,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pQueryHandle->cost.checkForNextTime += elapsedTime;
       return exists;
     }
-  
+
     pQueryHandle->activeIndex = 0;
     pQueryHandle->checkFiles  = false;
   }
-  
+
   // TODO: opt by consider the scan order
   bool ret = doHasDataInBuffer(pQueryHandle);
   terrno = TSDB_CODE_SUCCESS;
@@ -1705,15 +1737,15 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
   assert(!ASCENDING_TRAVERSE(pQueryHandle->order));
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
-  
+
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   TSKEY key = TSKEY_INITIAL_VAL;
   int32_t index = -1;
-  
+
   for(int32_t i = 0; i < numOfTables; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     if (pCheckInfo->pTableObj->lastKey > key) {
@@ -1721,36 +1753,36 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
       index = i;
     }
   }
-  
+
   if (index == -1) {
     // todo add failure test cases
     return;
   }
-  
+
   // erase all other elements in array list
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   for (int32_t i = 0; i < size; ++i) {
     if (i == index) {
       continue;
     }
-    
+
     STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     tSkipListDestroyIter(pTableCheckInfo->iter);
-    
+
     if (pTableCheckInfo->pDataCols != NULL) {
       tfree(pTableCheckInfo->pDataCols->buf);
     }
-    
+
     tfree(pTableCheckInfo->pDataCols);
     tfree(pTableCheckInfo->pCompInfo);
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = key;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {key, key};
 }
@@ -1759,13 +1791,13 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
   // filter the queried time stamp in the first place
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
   pQueryHandle->order = TSDB_ORDER_DESC;
-  
+
   assert(pQueryHandle->window.skey == pQueryHandle->window.ekey);
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   int32_t i = 0;
   while(i < numOfTables) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
@@ -1773,21 +1805,21 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
         pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) {
       break;
     }
-    
+
     i++;
   }
-  
+
   // there are no data in all the tables
   if (i == numOfTables) {
     return;
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = pQueryHandle->window.skey;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL};
 }
@@ -1811,7 +1843,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
       tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
           pQueryHandle->window.ekey);
-      
+
       break;
     }
 
@@ -1826,24 +1858,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
       moveToNextRowInMem(pCheckInfo);
       break;
     }
-    
+
   } while(moveToNextRowInMem(pCheckInfo));
 
   assert(numOfRows <= maxRowsToRead);
-  
+
   // if the buffer is not full in case of descending order query, move the data in the front of the buffer
   if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) {
     int32_t emptySize = maxRowsToRead - numOfRows;
-    
+
     for(int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
       memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
     }
   }
-  
+
   int64_t elapsedTime = taosGetTimestampUs() - st;
-  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle,
-            elapsedTime, numOfRows, numOfCols);
+  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle,
+            elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo);
 
   return numOfRows;
 }
@@ -1852,7 +1884,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
   SQueryFilePos* cur = &pHandle->cur;
   STable* pTable = NULL;
-  
+
   // there are data in file
   if (pHandle->cur.fid >= 0) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
@@ -1874,13 +1906,13 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
  */
 int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
-  
+
   SQueryFilePos* c = &pHandle->cur;
   if (c->mixBlock) {
     *pBlockStatis = NULL;
     return TSDB_CODE_SUCCESS;
   }
-  
+
   STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[c->slot];
   assert((c->slot >= 0 && c->slot < pHandle->numOfBlocks) || ((c->slot == pHandle->numOfBlocks) && (c->slot == 0)));
 
@@ -1900,7 +1932,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   for(int32_t i = 0; i < numOfCols; ++i) {
     pHandle->statis[i].colId = colIds[i];
   }
-  
+
   tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols);
 
   // always load the first primary timestamp column data
@@ -1949,31 +1981,31 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
     } else {
       SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
       assert(pHandle->realNumOfRows <= binfo.rows);
-  
+
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
-      
+
       if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
         SCompBlock* pBlock = pBlockInfo->compBlock;
-        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
-  
+        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot);
+
         // todo refactor
         int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-  
+
         // if the buffer is not full in case of descending order query, move the data in the front of the buffer
         if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
           int32_t emptySize = pHandle->outputCapacity - numOfRows;
           int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns);
-    
+
           for(int32_t i = 0; i < reqNumOfCols; ++i) {
             SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
             memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
           }
         }
-        
+
         return pHandle->pColumns;
       }
     }
@@ -1984,11 +2016,11 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
   SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
-    
+
     STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
     taosArrayPush(list, pTable);
   }
-  
+
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
@@ -1998,12 +2030,12 @@ static void destroyHelper(void* param) {
     return;
   }
 
-  
+
   tQueryInfo* pInfo = (tQueryInfo*)param;
   if (pInfo->optr != TSDB_RELATION_IN) {
     tfree(pInfo->q);
   }
-  
+
 //  tVariantDestroy(&(pInfo->q));
   free(param);
 }
@@ -2015,7 +2047,7 @@ void filterPrepare(void* expr, void* param) {
   }
 
   pExpr->_node.info = calloc(1, sizeof(tQueryInfo));
-  
+
   STSchema*   pTSSchema = (STSchema*) param;
   tQueryInfo* pInfo = pExpr->_node.info;
   tVariant*   pCond = pExpr->_node.pRight->pVal;
@@ -2025,7 +2057,7 @@ void filterPrepare(void* expr, void* param) {
   pInfo->optr     = pExpr->_node.optr;
   pInfo->compare  = getComparFunc(pSchema->type, pInfo->optr);
   pInfo->param    = pTSSchema;
-  
+
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
   } else {
@@ -2038,25 +2070,24 @@ typedef struct STableGroupSupporter {
   int32_t    numOfCols;
   SColIndex* pCols;
   STSchema*  pTagSchema;
-//  void*      tsdbMeta;
 } STableGroupSupporter;
 
 int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
   STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
   STable* pTable1 = *(STable**) p1;
   STable* pTable2 = *(STable**) p2;
-  
+
   for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
     SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
     int32_t colIndex = pColIndex->colIndex;
-    
+
     assert(colIndex >= TSDB_TBNAME_COLUMN_INDEX);
-    
+
     char *  f1 = NULL;
     char *  f2 = NULL;
     int32_t type = 0;
     int32_t bytes = 0;
-    
+
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       f1 = (char*) TABLE_NAME(pTable1);
       f2 = (char*) TABLE_NAME(pTable2);
@@ -2090,14 +2121,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
       return ret;
     }
   }
-  
+
   return 0;
 }
 
 void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp,
     __ext_compar_fn_t compareFn) {
   STable* pTable = taosArrayGetP(pTableList, 0);
-  
+
   SArray* g = taosArrayInit(16, POINTER_BYTES);
   taosArrayPush(g, &pTable);
   tsdbRefTable(pTable);
@@ -2105,10 +2136,10 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
   for (int32_t i = 1; i < numOfTables; ++i) {
     STable** prev = taosArrayGet(pTableList, i - 1);
     STable** p = taosArrayGet(pTableList, i);
-    
+
     int32_t ret = compareFn(prev, p, pSupp);
     assert(ret == 0 || ret == -1);
-    
+
     tsdbRefTable(*p);
     assert((*p)->type == TSDB_CHILD_TABLE);
 
@@ -2120,20 +2151,20 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
       taosArrayPush(g, p);
     }
   }
-  
+
   taosArrayPush(pGroups, &g);
 }
 
 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) {
   assert(pTableList != NULL);
   SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
-  
+
   size_t size = taosArrayGetSize(pTableList);
   if (size == 0) {
     tsdbDebug("no qualified tables");
     return pTableGroup;
   }
-  
+
   if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table
     SArray* sa = taosArrayInit(size, POINTER_BYTES);
     for(int32_t i = 0; i < size; ++i) {
@@ -2143,7 +2174,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
       tsdbRefTable(*pTable);
       taosArrayPush(sa, pTable);
     }
-    
+
     taosArrayPush(pTableGroup, &sa);
     tsdbDebug("all %zu tables belong to one group", size);
   } else {
@@ -2151,18 +2182,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
     pSupp->numOfCols = numOfOrderCols;
     pSupp->pTagSchema = pTagSchema;
     pSupp->pCols = pCols;
-    
+
     taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn);
     createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn);
     tfree(pSupp);
   }
-  
+
   return pTableGroup;
 }
 
 bool indexedNodeFilterFp(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
-  
+
   STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
@@ -2172,7 +2203,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
   } else {
     val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
   }
-  
+
   int32_t ret = 0;
   if (val == NULL) { //the val is possible to be null, so check it out carefully
     ret = -1; // val is missing in table tags value pairs
@@ -2209,7 +2240,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
     default:
       assert(false);
   }
-  
+
   return true;
 }
 
@@ -2239,7 +2270,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     goto _error;
   }
-  
+
   if (pTable->type != TSDB_SUPER_TABLE) {
     tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
         pTable->name->data);
@@ -2252,7 +2283,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   //NOTE: not add ref count for super table
   SArray* res = taosArrayInit(8, POINTER_BYTES);
   STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
-  
+
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
     int32_t ret = getAllTableList(pTable, res);
@@ -2263,7 +2294,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     pGroupInfo->numOfTables = taosArrayGetSize(res);
     pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
-      
+
     tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
     taosArrayDestroy(res);
 
@@ -2299,7 +2330,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   } CATCH( code ) {
     CLEANUP_EXECUTE();
     terrno = code;
-    goto _error; 
+    goto _error;
     // TODO: more error handling
   } END_TRY
 
@@ -2335,12 +2366,12 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p
 
   pGroupInfo->numOfTables = 1;
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
-  
+
   SArray* group = taosArrayInit(1, POINTER_BYTES);
-  
+
   taosArrayPush(group, &pTable);
   taosArrayPush(pGroupInfo->pGroupList, &group);
-  
+
   return TSDB_CODE_SUCCESS;
 
   _error:
@@ -2428,8 +2459,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   tsdbDestroyHelper(&pQueryHandle->rhelper);
 
   SIOCostSummary* pCost = &pQueryHandle->cost;
-  tsdbDebug(":io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
-      pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
+  tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
+      pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
 
   tfree(pQueryHandle);
 }
diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt
index d4350fc8b250541aa8eed3d7686fd3363734442e..2521c582d10d1d90aa91ef210892be2283316240 100644
--- a/src/util/CMakeLists.txt
+++ b/src/util/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 
@@ -11,10 +11,11 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
+  TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4)
+  ADD_SUBDIRECTORY(tests)
+
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
     FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64)
     FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64)
     IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST)
@@ -28,9 +29,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     MESSAGE(STATUS "Failed to find iconv, use default encoding method")
   ENDIF ()
 
-  ADD_SUBDIRECTORY(tests)
 ELSEIF (TD_WINDOWS_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
@@ -68,44 +67,11 @@ ELSEIF (TD_WINDOWS_64)
   LIST(APPEND SRC ./src/tutil.c)
   LIST(APPEND SRC ./src/version.c)
   ADD_LIBRARY(tutil ${SRC}) 
-  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
-  LIST(APPEND SRC ./src/hash.c)
-  LIST(APPEND SRC ./src/ihash.c)
-  LIST(APPEND SRC ./src/lz4.c)
-  LIST(APPEND SRC ./src/shash.c)
-  LIST(APPEND SRC ./src/tbase64.c)
-  LIST(APPEND SRC ./src/tcache.c)
-  LIST(APPEND SRC ./src/tcompression.c)
-  LIST(APPEND SRC ./src/textbuffer.c)
-  LIST(APPEND SRC ./src/tglobalcfg.c)
-  LIST(APPEND SRC ./src/thash.c)
-  LIST(APPEND SRC ./src/thashutil.c)
-  LIST(APPEND SRC ./src/thistogram.c)
-  LIST(APPEND SRC ./src/tidpool.c)
-  LIST(APPEND SRC ./src/tinterpolation.c)
-  LIST(APPEND SRC ./src/tlog.c)
-  LIST(APPEND SRC ./src/tlosertree.c)
-  LIST(APPEND SRC ./src/tmd5.c)
-  LIST(APPEND SRC ./src/tmem.c)
-  LIST(APPEND SRC ./src/tmempool.c)
-  LIST(APPEND SRC ./src/tmodule.c)
-  LIST(APPEND SRC ./src/tnote.c)
-  LIST(APPEND SRC ./src/tpercentile.c)
-  LIST(APPEND SRC ./src/tsched.c)
-  LIST(APPEND SRC ./src/tskiplist.c)
-  LIST(APPEND SRC ./src/tsocket.c)
-  LIST(APPEND SRC ./src/tstrbuild.c)
-  LIST(APPEND SRC ./src/ttime.c)
-  LIST(APPEND SRC ./src/ttimer.c)
-  LIST(APPEND SRC ./src/ttokenizer.c)
-  LIST(APPEND SRC ./src/ttypes.c)
-  LIST(APPEND SRC ./src/tutil.c)
-  LIST(APPEND SRC ./src/version.c)
-  LIST(APPEND SRC ./src/hash.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4)
 ENDIF()
 
 #  TARGET_LINK_LIBRARIES(tutil mstorage)
diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h
index 5a3545fd8f20169f7dc5a7a57f2994db40337ef2..1e2aeae394a4c49c104fe24de4a4a643884bdc37 100644
--- a/src/util/inc/tcache.h
+++ b/src/util/inc/tcache.h
@@ -68,8 +68,6 @@ typedef struct {
   int64_t         refreshTime;
   STrashElem *    pTrash;
   char*           name;
-//  void *          tmrCtrl;
-//  void *          pTimer;
   SCacheStatis    statistics;
   SHashObj *      pHashTable;
   __cache_free_fn_t freeFp;
diff --git a/src/util/inc/tchecksum.h b/src/util/inc/tchecksum.h
index 493c993c2554ba7574ee308edd099b25f9e82f37..54043d68e137eaf35622df90a465a0eabb184c59 100644
--- a/src/util/inc/tchecksum.h
+++ b/src/util/inc/tchecksum.h
@@ -35,8 +35,6 @@ extern "C" {
 typedef uint32_t TSCKSUM;
 
 static FORCE_INLINE TSCKSUM taosCalcChecksum(TSCKSUM csi, const uint8_t *stream, uint32_t ssize) {
-  assert(ssize >= 0 && stream != NULL);
-
   return (*crc32c)(csi, stream, (size_t)ssize);
 }
 
diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h
deleted file mode 100644
index de52a409674b9518fa908bd8131024c5b1c58010..0000000000000000000000000000000000000000
--- a/src/util/inc/tfile.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TFILE_H
-#define TDENGINE_TFILE_H
-
-#ifdef TAOS_RANDOM_FILE_FAIL
-
-void taosSetRandomFileFailFactor(int factor);
-void taosSetRandomFileFailOutput(const char *path);
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line);
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line);
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line);
-
-#define tread(fd, buf, count)  taos_tread(fd, buf, count, __FILE__, __LINE__)
-#define twrite(fd, buf, count)  taos_twrite(fd, buf, count, __FILE__, __LINE__)
-#define lseek(fd, offset, whence)  taos_lseek(fd, offset, whence, __FILE__, __LINE__)
-
-#endif  // TAOS_RANDOM_FILE_FAIL
-
-#endif  // TDENGINE_TFILE_H
diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h
index a4ed9311e24bf5098f9cca8c05edd0998b8d6253..e8380294da47de1c9eb65059263ac820f3d816be 100644
--- a/src/util/inc/tlist.h
+++ b/src/util/inc/tlist.h
@@ -55,6 +55,8 @@ int        tdListPrepend(SList *list, void *data);
 int        tdListAppend(SList *list, void *data);
 SListNode *tdListPopHead(SList *list);
 SListNode *tdListPopTail(SList *list);
+SListNode *tdListGetHead(SList *list);
+SListNode *tsListGetTail(SList *list);
 SListNode *tdListPopNode(SList *list, SListNode *node);
 void       tdListMove(SList *src, SList *dst);
 void       tdListDiscard(SList *list);
diff --git a/src/util/inc/ttime.h b/src/util/inc/ttime.h
index 576c9a51f667604c1a8c5b89ddc47858eb0f1cb4..25d1bdb23e853eeefa20976eea23cfb0a4909827 100644
--- a/src/util/inc/ttime.h
+++ b/src/util/inc/ttime.h
@@ -20,9 +20,8 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
-#include <time.h>
-#include "tutil.h"
+#include "os.h"
+#include "taosdef.h"
 
 //@return timestamp in second
 int32_t taosGetTimestampSec();
diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h
index 34f35c380745172f5fc6a6d4aaf094cdf82c68ac..e312513ff3f1f2294360ab2fce25310bcd9bf6d7 100644
--- a/src/util/inc/tutil.h
+++ b/src/util/inc/tutil.h
@@ -25,150 +25,24 @@ extern "C" {
 #include "tcrc32c.h"
 #include "taosdef.h"
 
-#ifndef STDERR_FILENO
-#define STDERR_FILENO (2)
-#endif
-
-#define FD_VALID(x) ((x) > STDERR_FILENO)
-#define FD_INITIALIZER  ((int32_t)-1)
-
-#define WCHAR wchar_t
-
-#define tfree(x)         \
-  do {                   \
-    if (x) {             \
-      free((void *)(x)); \
-      x = 0;             \
-    }                    \
-  } while(0);
-
-#define tstrncpy(dst, src, size)   \
-  do {                             \
-    strncpy((dst), (src), (size)); \
-    (dst)[(size)-1] = 0;           \
-  } while (0);
-
-#define tclose(x) taosCloseSocket(x)
-
-// Pointer p drift right by b bytes
-#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
-#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
-
-#ifndef NDEBUG
-#define ASSERT(x) assert(x)
-#else
-#define ASSERT(x)
-#endif
-
-#ifdef UNUSED
-#undefine UNUSED
-#endif
-#define UNUSED(x) ((void)(x))
-
-#ifdef UNUSED_FUNC
-#undefine UNUSED_FUNC
-#endif
-
-#ifdef UNUSED_PARAM
-#undef UNUSED_PARAM
-#endif
-
-#if defined(__GNUC__)
-#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
-#define UNUSED_FUNC __attribute__((unused))
-#else
-#define UNUSED_PARAM(x) x
-#define UNUSED_FUNC
-#endif
-
-#ifdef tListLen
-#undefine tListLen
-#endif
-#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
-
-#if defined(__GNUC__)
-#define FORCE_INLINE inline __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
-#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
-  
-#define DEFAULT_COMP(x, y)       \
-  do {                           \
-    if ((x) == (y)) {            \
-      return 0;                  \
-    } else {                     \
-      return (x) < (y) ? -1 : 1; \
-    }                            \
-  } while (0)
-
-#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
-
-// align to 8bytes
-#define ALIGN8(n) ALIGN_NUM(n, 8)
-
-#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
-#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
-#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
-#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
-#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
-#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
-
-#define POW2(x) ((x) * (x))
-
-uint32_t taosRand(void);
-
-uint32_t trand(void);
-
-size_t twcslen(const wchar_t *wcs);
-
 int32_t strdequote(char *src);
-
-size_t strtrim(char *src);
-
-char *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
-
-char **strsplit(char *src, const char *delim, int32_t *num);
-
-char* strtolower(char *dst, const char *src);
-
-char* strntolower(char *dst, const char *src, int32_t n);
-
+size_t  strtrim(char *src);
+char *  strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char ** strsplit(char *src, const char *delim, int32_t *num);
+char *  strtolower(char *dst, const char *src);
+char *  strntolower(char *dst, const char *src, int32_t n);
 int64_t strnatoi(char *num, int32_t len);
-
-//char* strreplace(const char* str, const char* pattern, const char* rep);
-
-char *strbetween(char *string, char *begin, char *end);
-
-char *paGetToken(char *src, char **token, int32_t *tokenLen);
-
-void taosMsleep(int32_t mseconds);
+char *  strbetween(char *string, char *begin, char *end);
+char *  paGetToken(char *src, char **token, int32_t *tokenLen);
 
 int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
-
 int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
 
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
-/**
- *
- * @param fileNamePattern
- * @param dstPath
- */
-void getTmpfilePath(const char *fileNamePattern, char *dstPath);
-
-bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t* len);
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes);
-
-void taosRandStr(char* str, int32_t size);
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-
-bool taosValidateEncodec(const char *encodec);
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer);
+int  taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
+
+char *   taosIpStr(uint32_t ipInt);
+uint32_t ip2uint(const char *const ip_addr);
 
 static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, char *target) {
   MD5_CTX context;
@@ -178,53 +52,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha
   memcpy(target, context.digest, TSDB_KEY_LEN);
 }
 
-int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
-
-char *taosIpStr(uint32_t ipInt);
-
-uint32_t ip2uint(const char *const ip_addr);
-
-void taosRemoveDir(char *rootDir);
-int  tmkdir(const char *pathname, mode_t mode); 
-void taosMvDir(char* destDir, char *srcDir);
-
-#define TAOS_ALLOC_MODE_DEFAULT 0
-#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
-#define TAOS_ALLOC_MODE_DETECT_LEAK 2
-void taosSetAllocMode(int mode, const char* path, bool autoDump);
-void taosDumpMemoryLeak();
-
-void * tmalloc(size_t size);
-void * tcalloc(size_t nmemb, size_t size);
-size_t tsizeof(void *ptr);
-void   tmemset(void *ptr, int c);
-void * trealloc(void *ptr, size_t size);
-void   tzfree(void *ptr);
-
-#ifdef TAOS_MEM_CHECK
-
-void *  taos_malloc(size_t size, const char *file, uint32_t line);
-void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
-void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
-void    taos_free(void *ptr, const char *file, uint32_t line);
-char *  taos_strdup(const char *str, const char *file, uint32_t line);
-char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
-ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
-
-#ifndef TAOS_MEM_CHECK_IMPL
-
-#define malloc(size) taos_malloc(size, __FILE__, __LINE__)
-#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
-#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
-#define free(ptr) taos_free(ptr, __FILE__, __LINE__)
-#define strdup(str) taos_strdup(str, __FILE__, __LINE__)
-#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
-#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
-
-#endif  // TAOS_MEM_CHECK_IMPL
-
-#endif // TAOS_MEM_CHECK
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 5dff6286f076408d2fd5d693452eb70fcb3551f7..695529d02f6b51b46cab5af43b69f2b1928c9925 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -14,10 +14,9 @@
  */
 
 #include "os.h"
-
 #include "hash.h"
 #include "tulog.h"
-#include "tutil.h"
+#include "taosdef.h"
 
 static FORCE_INLINE void __wr_lock(void *lock) {
   if (lock == NULL) {
diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c
index 4b37cbd2d5cb0c9bd7432a6d9db59a5a62234b3e..4af435798cccf989eef5c75b74f72e8cfe5ccaa0 100644
--- a/src/util/src/tbuffer.c
+++ b/src/util/src/tbuffer.c
@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "os.h"
 #include <stdlib.h>
 #include <memory.h>
 #include <assert.h>
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 92d4b2caacb061bf144920900548875b06904a21..dc9128d4a95d210e82aec4c01ef679540c9945d8 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -259,7 +259,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
     return NULL;
   }
 
-  pthread_attr_t thattr = {{0}};
+  pthread_attr_t thattr;
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
@@ -288,7 +288,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
       uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64
              "bytes size:%" PRId64 "bytes",
              pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime,
-             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, dataSize);
+             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
     } else {
       uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
     }
@@ -343,7 +343,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke
   SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
   if (ptNode != NULL) {
      T_REF_INC(*ptNode);
-    (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan;
+    (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan;
   }
 
   __cache_unlock(pCacheObj);
@@ -381,7 +381,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
 }
 
 void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
-  if (pCacheObj == NULL || data == NULL) return NULL;
+  if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
   
   size_t          offset = offsetof(SCacheDataNode, data);
   SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset);
@@ -419,7 +419,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   // note: extend lifespan before dec ref count
   bool inTrashCan = pNode->inTrashCan;
 
-  if (pCacheObj->extendLifespan && (!inTrashCan)) {
+  if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) {
     atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs());
     uDebug("cache:%s data:%p extend life time to %"PRId64 "  before release", pCacheObj->name, pNode->data, pNode->expireTime);
   }
@@ -457,8 +457,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   } else {
     // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
     int32_t ref = T_REF_DEC(pNode);
-    uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, ref,
-        inTrashCan);
+
+    uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data,
+           ref, inTrashCan);
   }
 }
 
@@ -572,6 +573,7 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
   free(pElem);
 }
 
+// TODO add another lock when scanning trashcan
 void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
   __cache_wr_lock(pCacheObj);
 
@@ -643,6 +645,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t
   __cache_wr_lock(pCacheObj);
   while (taosHashIterNext(pIter)) {
     SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
+
     if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) {
       taosCacheReleaseNode(pCacheObj, pNode);
       continue;
@@ -674,6 +677,7 @@ void* taosCacheTimedRefresh(void *handle) {
 
     // check if current cache object will be deleted every 500ms.
     if (pCacheObj->deleting) {
+      uDebug("%s refresh threads quit", pCacheObj->name);
       break;
     }
 
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index 889d38ff207315ed489f7877e3ee9f566a5cde41..95645882547110cb36fdc16eafe8cc7d1fe09bcf 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -1,6 +1,6 @@
 #include "taosdef.h"
 #include "tcompare.h"
-#include <tarray.h>
+#include "tarray.h"
 #include "tutil.h"
 
 int32_t compareInt32Val(const void *pLeft, const void *pRight) {
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index d561e8ba5f79a5de8faf9feeb8a857442e36d05e..59bc09e87783c25f48c18acc7d170a929ea4b231 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -118,7 +118,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
       
       wordfree(&full_path);
 
-      int code = tmkdir(option, 0755);
+      int code = taosMkDir(option, 0755);
       if (code != 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         uError("config option:%s, input value:%s, directory not exist, create fail:%s",
diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c
index 0704285da5fd1cc390c11891e2983c32ed5f5265..d7bf9d78577be5a5d83bfbbedc4a334f792bb9b8 100644
--- a/src/util/src/tkvstore.c
+++ b/src/util/src/tkvstore.c
@@ -12,22 +12,15 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <errno.h>
-#include <fcntl.h>
-#include <libgen.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 
-#include "hash.h"
+#define _DEFAULT_SOURCE
 #include "os.h"
+#include "hash.h"
 #include "taoserror.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tkvstore.h"
 #include "tulog.h"
-#include "tfile.h"
 
 #define TD_KVSTORE_HEADER_SIZE 512
 #define TD_KVSTORE_MAJOR_VERSION 1
@@ -182,7 +175,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
     goto _err;
   }
 
-  if (tsendfile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -242,13 +235,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
   ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
   ASSERT(tlen == sizeof(SKVRecord));
 
-  if (twrite(pStore->fd, buf, tlen) < tlen) {
+  if (taosTWrite(pStore->fd, buf, tlen) < tlen) {
     uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  if (twrite(pStore->fd, cont, contLen) < contLen) {
+  if (taosTWrite(pStore->fd, cont, contLen) < contLen) {
     uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
     return -1;
   }
@@ -286,8 +279,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
   void *pBuf = buf;
   tdEncodeKVRecord(&pBuf, &rInfo);
 
-  if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
-    uError("failed to write %" PRId64 " bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
+  if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
+    uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -335,7 +328,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
     return -1;
   }
 
-  if (tread(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -366,7 +359,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
   ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
-  if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -499,7 +492,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
   ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
 
   while (true) {
-    ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord));
+    ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord));
     if (tsize == 0) break;
     if (tsize < sizeof(SKVRecord)) {
       uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
@@ -562,7 +555,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
       goto _err;
     }
 
-    if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) {
+    if (taosTRead(pStore->fd, buf, pRecord->size) < pRecord->size) {
       uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
              strerror(errno), pRecord->offset);
       terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c
index f402c2307e68cee6100a28e53dce1935af44fb3f..8c2ad83de117aaf528565a711a2aa3732984e0a9 100644
--- a/src/util/src/tlist.c
+++ b/src/util/src/tlist.c
@@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
   SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
   if (node == NULL) return -1;
 
+  node->next = node->prev = NULL;
   memcpy((void *)(node->data), data, list->eleSize);
   tdListPrependNode(list, node);
 
@@ -121,6 +122,22 @@ SListNode *tdListPopTail(SList *list) {
   return node;
 }
 
+SListNode *tdListGetHead(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->head;
+}
+
+SListNode *tsListGetTail(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->tail;
+}
+
 SListNode *tdListPopNode(SList *list, SListNode *node) {
   if (list->head == node) {
     list->head = node->next;
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index c2c56f29db2664cb4647f3fead8230b733555b51..96ae417297c934e71ecf1feb1ab1ff849da702d7 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -14,6 +14,9 @@
  */
 
 #define _DEFAULT_SOURCE
+// no test file errors here
+#undef TAOS_RANDOM_FILE_FAIL
+
 #include "os.h"
 #include "tulog.h"
 #include "tlog.h"
@@ -301,11 +304,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   lseek(tsLogObj.logHandle->fd, 0, SEEK_END);
 
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "                new log file                      \n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
 
   return 0;
 }
@@ -355,7 +358,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
 
     if (tsLogObj.maxLines > 0) {
@@ -365,7 +368,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 void taosDumpData(unsigned char *msg, int32_t len) {
@@ -384,7 +387,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
     pos += 3;
     if (c >= 16) {
       temp[pos++] = '\n';
-      twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+      taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
       c = 0;
       pos = 0;
     }
@@ -392,7 +395,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
 
   temp[pos++] = '\n';
 
-  twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+  taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
 
   return;
 }
@@ -432,7 +435,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
     
     if (tsLogObj.maxLines > 0) {
@@ -442,7 +445,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 #if 0
@@ -571,7 +574,7 @@ static void *taosAsyncOutputLog(void *param) {
     while (1) {
       log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT);
       if (log_size) {
-        twrite(tLogBuff->fd, tempBuffer, log_size);
+        taosTWrite(tLogBuff->fd, tempBuffer, log_size);
         LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff);
       } else {
         break;
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 12a7fc2b9b6174cced2446a8ebe80ebb1da97b00..91916c92d7f1feb2b7b594ae1d7541cb95e4db07 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -13,6 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+// no test file errors here
+#undef TAOS_RANDOM_FILE_FAIL
+
 #include "tnote.h"
 
 taosNoteInfo  m_HttpNote;
@@ -264,7 +267,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
     buffer[len] = 0;
 
     if (pNote->taosNoteFd >= 0)  {
-        twrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+        taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
 
         if (pNote->taosNoteMaxLines > 0) {
             pNote->taosNoteLines++;
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 5de61a3d57a0cc480781ec581bdc76d54419bf87..6f4d2b8afd81f3bb85405f2d6a1f0f71144878f3 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -164,7 +164,7 @@ int taosNonblockwrite(int fd, char *ptr, int nbytes) {
       return -1;
     }
 
-    nwritten = (int)send(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+    nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
     if (nwritten <= 0) {
       if (errno == EAGAIN || errno == EINTR) continue;
 
@@ -304,10 +304,10 @@ int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientI
     //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
     close(sockFd);
     sockFd = -1;
+  } else {
+    taosKeepTcpAlive(sockFd);
   }
 
-  // taosKeepTcpAlive(sockFd);
-
   return sockFd;
 }
 
@@ -349,7 +349,7 @@ int taosKeepTcpAlive(int sockFd) {
 
   struct linger linger = {0};
   linger.l_onoff = 1;
-  //linger.l_linger = 0;
+  linger.l_linger = 3;
   if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
     uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
     close(sockFd);
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 7b73b1e17c54a2278ebd15420eea96bb84bfe547..eaa60d739b10ce90646cd439e3dbe589e06e80bb 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -14,11 +14,6 @@
  */
 
 #include "os.h"
-
-#ifdef USE_LIBICONV
-#include "iconv.h"
-#endif
-
 #include "tcrc32c.h"
 #include "tglobal.h"
 #include "ttime.h"
@@ -27,76 +22,6 @@
 #include "tulog.h"
 #include "taoserror.h"
 
-
-#ifdef WINDOWS
-uint32_t taosRand(void)
-{
-  return rand();
-}
-#else
-uint32_t taosRand(void)
-{
-/*
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-*/
-  return rand();
-}
-
-uint32_t trand(void)
-{
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-}
-#endif
-
-size_t twcslen(const wchar_t *wcs) {
-#ifdef WINDOWS
-  int *wstr = (int *)wcs;
-  if (NULL == wstr) {
-    return 0;
-  }
-
-  size_t n = 0;
-  while (1) {
-    if (0 == *wstr++) {
-      break;
-    }
-    n++;
-  }
-
-  return n;
-#else
-  return wcslen(wcs);
-#endif
-}
-
 int32_t strdequote(char *z) {
   if (z == NULL) {
     return 0;
@@ -234,6 +159,7 @@ char* strtolower(char *dst, const char *src) {
   *p = 0;
   return dst;
 }
+
 char* strntolower(char *dst, const char *src, int32_t n) {
   int esc = 0;
   char quote = 0, *p = dst, c;
@@ -343,96 +269,6 @@ int64_t strnatoi(char *num, int32_t len) {
   return ret;
 }
 
-#if 0
-FORCE_INLINE size_t getLen(size_t old, size_t size) {
-  if (old == 1) {
-    old = 2;
-  }
-
-  while (old < size) {
-    old = (old * 1.5);
-  }
-
-  return old;
-}
-
-static char *ensureSpace(char *dest, size_t *curSize, size_t size) {
-  if (*curSize < size) {
-    *curSize = getLen(*curSize, size);
-
-    char *tmp = realloc(dest, *curSize);
-    if (tmp == NULL) {
-      free(dest);
-      return NULL;
-    }
-
-    return tmp;
-  }
-
-  return dest;
-}
-
-char *strreplace(const char *str, const char *pattern, const char *rep) {
-  if (str == NULL || pattern == NULL || rep == NULL) {
-    return NULL;
-  }
-
-  const char *s = str;
-
-  size_t oldLen = strlen(str);
-  size_t newLen = oldLen;
-
-  size_t repLen = strlen(rep);
-  size_t patternLen = strlen(pattern);
-
-  char *dest = calloc(1, oldLen + 1);
-  if (dest == NULL) {
-    return NULL;
-  }
-
-  if (patternLen == 0) {
-    return strcpy(dest, str);
-  }
-
-  int32_t start = 0;
-
-  while (1) {
-    char *p = strstr(str, pattern);
-    if (p == NULL) {  // remain does not contain pattern
-      size_t remain = (oldLen - (str - s));
-      size_t size = remain + start + 1;
-
-      dest = ensureSpace(dest, &newLen, size);
-      if (dest == NULL) {
-        return NULL;
-      }
-
-      strcpy(dest + start, str);
-      dest[start + remain] = 0;
-      break;
-    }
-
-    size_t len = p - str;
-    size_t size = start + len + repLen + 1;
-
-    dest = ensureSpace(dest, &newLen, size);
-    if (dest == NULL) {
-      return NULL;
-    }
-
-    memcpy(dest + start, str, len);
-
-    str += (len + patternLen);
-    start += len;
-
-    memcpy(dest + start, rep, repLen);
-    start += repLen;
-  }
-
-  return dest;
-}
-#endif
-
 char *strbetween(char *string, char *begin, char *end) {
   char *result = NULL;
   char *_begin = strstr(string, begin);
@@ -491,170 +327,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
   return 0;
 }
 
-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
-  int32_t ts = taosGetTimestampSec();
-
-  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
-
-  char *delimiterPos = strrchr(fullPath, delimiter);
-  if (delimiterPos == NULL) return -1;
-
-  int32_t fileNameLen = 0;
-  if (suffix)
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
-  else
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
-
-  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
-  if (*dstPath == NULL) {
-    *dstPath = calloc(1, len + 1);
-    if (*dstPath == NULL) return -1;
-  }
-
-  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
-  strncat(*dstPath, fname, (size_t)fileNameLen);
-  (*dstPath)[len] = 0;
-
-  return rename(fullPath, *dstPath);
-}
-
-void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
-  const char* tdengineTmpFileNamePrefix = "tdengine-";
-  
-  char tmpPath[PATH_MAX];
-
-#ifdef WINDOWS
-  char *tmpDir = getenv("tmp");
-  if (tmpDir == NULL) {
-    tmpDir = "";
-  }
-#else
-  char *tmpDir = "/tmp/";
-#endif
-  
-  strcpy(tmpPath, tmpDir);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
-  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
-    strcat(tmpPath, fileNamePrefix);
-    strcat(tmpPath, "-%d-%s");
-  }
-  
-  char rand[8] = {0};
-  taosRandStr(rand, tListLen(rand) - 1);
-  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
-}
-
-void taosRandStr(char* str, int32_t size) {
-  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
-  int32_t len = 39;
-
-  for (int32_t i = 0; i < size; ++i) {
-    str[i] = set[taosRand() % len];
-  }
-}
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
-#if defined WINDOWS
-  for (int i = 0; i < bytes; ++i) {
-    int32_t f1 = *(int32_t*)((char*)f1_ucs4 + i * 4);
-    int32_t f2 = *(int32_t*)((char*)f2_ucs4 + i * 4);
-
-    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
-      return f1 - f2;
-    }
-    else if (f1 == 0 && f2 == 0) {
-      return 0;
-    }
-
-    if (f1 != f2) {
-      return f1 - f2;
-    }
-  }
-  return 0;
-
-#if 0
-  int32_t ucs4_max_len = bytes + 4;
-  char *f1_mbs = calloc(bytes, 1);
-  char *f2_mbs = calloc(bytes, 1);
-  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
-    return -1;
-  }
-  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
-    return -1;
-  }
-  int32_t ret = strcmp(f1_mbs, f2_mbs);
-  free(f1_mbs);
-  free(f2_mbs);
-  return ret;
-#endif
-
-#else
-  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
-#endif
-}
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
-  size_t ucs4_input_len = ucs4_max_len;
-  size_t outLen = ucs4_max_len;
-  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
-    iconv_close(cd);
-    return -1;
-  }
-  iconv_close(cd);
-  return (int32_t)(ucs4_max_len - outLen);
-#else
-  mbstate_t state = {0};
-  int32_t len = (int32_t) wcsnrtombs(NULL, (const wchar_t **) &ucs4, ucs4_max_len / 4, 0, &state);
-  if (len < 0) {
-    return -1;
-  }
-  memset(&state, 0, sizeof(state));
-  len = wcsnrtombs(mbs, (const wchar_t **) &ucs4, ucs4_max_len / 4, (size_t) len, &state);
-  if (len < 0) {
-    return -1;
-  }
-  return len;
-#endif
-}
-
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t* len) {
-  memset(ucs4, 0, ucs4_max_len);
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
-  size_t ucs4_input_len = mbsLength;
-  size_t outLeft = ucs4_max_len;
-  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
-    iconv_close(cd);
-    return false;
-  }
-  
-  iconv_close(cd);
-  if (len != NULL) {
-    *len = ucs4_max_len - outLeft;
-  }
-  
-  return true;
-#else
-  mbstate_t state = {0};
-  int32_t len = mbsnrtowcs((wchar_t *) ucs4, (const char **) &mbs, mbsLength, ucs4_max_len / 4, &state);
-  return len >= 0;
-#endif
-}
-
-bool taosValidateEncodec(const char *encodec) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
-  if (cd == (iconv_t)(-1)) {
-    return false;
-  }
-  iconv_close(cd);
-#endif
-  return true;
-}
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
   if (versionStr == NULL || versionNubmer == NULL) {
     return false;
@@ -734,112 +406,3 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) {
   *(int64_t*)(&dv) = *(int64_t*)pBuf;
   return dv; 
 }
-
-typedef struct CharsetPair {
-  char *oldCharset;
-  char *newCharset;
-} CharsetPair;
-
-char *taosCharsetReplace(char *charsetstr) {
-  CharsetPair charsetRep[] = {
-      { "utf8", "UTF-8" }, { "936", "CP936" },
-  };
-
-  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
-    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
-      return strdup(charsetRep[i].newCharset);
-    }
-  }
-
-  return strdup(charsetstr);
-}
-
-void *tmalloc(size_t size) {
-  if (size <= 0) return NULL;
-
-  void *ret = malloc(size + sizeof(size_t));
-  if (ret == NULL) return NULL;
-
-  *(size_t *)ret = size;
-
-  return (void *)((char *)ret + sizeof(size_t));
-}
-
-void *tcalloc(size_t nmemb, size_t size) {
-  size_t tsize = nmemb * size;
-  void * ret = tmalloc(tsize);
-  if (ret == NULL) return NULL;
-
-  tmemset(ret, 0);
-  return ret;
-}
-
-size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
-
-void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
-
-void * trealloc(void *ptr, size_t size) {
-  if (ptr == NULL) return tmalloc(size);
-
-  if (size <= tsizeof(ptr)) return ptr;
-
-  void * tptr = (void *)((char *)ptr - sizeof(size_t));
-  size_t tsize = size + sizeof(size_t);
-  tptr = realloc(tptr, tsize);
-  if (tptr == NULL) return NULL;
-
-  *(size_t *)tptr = size;
-
-  return (void *)((char *)tptr + sizeof(size_t));
-}
-
-void tzfree(void *ptr) {
-  if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
-  }
-}
-
-void taosRemoveDir(char *rootDir) {
-  DIR *dir = opendir(rootDir);
-  if (dir == NULL) return;
-
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-     
-    char filename[1024];
-    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
-    if (de->d_type & DT_DIR) {
-      taosRemoveDir(filename);
-    } else {
-      (void)remove(filename);
-      uInfo("file:%s is removed", filename);
-    }
-  }
-
-  closedir(dir);
-  rmdir(rootDir);
-
-  uInfo("dir:%s is removed", rootDir);
-}
-
-int tmkdir(const char *path, mode_t mode) {
-  int code = mkdir(path, 0755);
-  if (code < 0 && errno == EEXIST) code = 0;
-  return code;
-}
-
-void taosMvDir(char* destDir, char *srcDir) {
-  if (0 == tsEnableVnodeBak) {
-    uInfo("vnode backup not enabled");
-    return;
-  }
-
-  char shellCmd[1024+1] = {0}; 
-  
-  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
-  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
-  tSystem(shellCmd);
-  uInfo("shell cmd:%s is executed", shellCmd);
-}
-
diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt
index a1c56b32b581144bd49c1b980aa1dbf59b690ce0..9b6b03209e6fc9abce2ee4871de649c9cc688dbe 100644
--- a/src/vnode/CMakeLists.txt
+++ b/src/vnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index 74cfbf1e731fd67c47f7375198a8353740429060..38f7c8e6056644591993027b410283b41fb8560c 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -37,6 +37,7 @@ extern int32_t vDebugFlag;
 typedef struct {
   int32_t      vgId;      // global vnode group ID
   int32_t      refCount;  // reference count
+  int32_t      delay;
   int8_t       status; 
   int8_t       role;   
   int8_t       accessState;
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index dd5ec1843941b8b691c53df9a4d3cb3f54f6521d..8a56a2c50045dd813caebe33717ab740b76afb9d 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -44,17 +44,9 @@ static int      vnodeProcessTsdbStatus(void *arg, int status);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion);
 static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
+static void     vnodeCtrlFlow(void *handle, int32_t mseconds); 
 static int      vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
 
-#ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void    syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
-#endif
-
 int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
@@ -286,17 +278,14 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.writeToCache = vnodeWriteToQueue;
   syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp; 
   syncInfo.notifyRole = vnodeNotifyRole;
+  syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
   pVnode->sync = syncStart(&syncInfo);
 
-#ifndef _SYNC
-  pVnode->role = TAOS_SYNC_ROLE_MASTER;
-#else
   if (pVnode->sync == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
   }
-#endif
 
   pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId);
   pVnode->events = NULL;
@@ -430,10 +419,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
   if (pVnode->status != TAOS_VN_STATUS_READY) return;
   if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
 
-  // still need report status when unsynced
-  if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) {
-  } else if (pVnode->tsdb == NULL) {
-  } else {
+  if (pVnode->tsdb) {
     tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
   }
 
@@ -562,6 +548,13 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
     cqStop(pVnode->cq);
 }
 
+static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) {
+  SVnodeObj *pVnode = ahandle;
+  if (pVnode->delay != mseconds) 
+    vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds);
+  pVnode->delay = mseconds;
+}
+
 static int vnodeResetTsdb(SVnodeObj *pVnode)
 {
   char rootDir[128] = "\0";
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 973df7c5a10f2ba84a77718602c8ffddd4b14134..066770e1bb9de5d463a1abd2b4078c3d41f9c574 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -71,11 +71,45 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) {
   pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
   pRead->pCont = qhandle;
   pRead->contLen = 0;
+  pRead->rpcMsg.handle = NULL;
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
   taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }
 
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) {
+  bool continueExec = false;
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) {
+    if (continueExec) {
+      vDebug("QInfo:%p add to query task queue for exec", handle);
+      vnodePutItemIntoReadQueue(pVnode, handle);
+      pRet->qhandle = handle;
+      *freeHandle = false;
+    } else {
+      vDebug("QInfo:%p exec completed", handle);
+      *freeHandle = true;
+    }
+  } else {
+    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+    *freeHandle = true;
+  }
+
+  return code;
+}
+
+static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+  pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+  pRet->len = sizeof(SRetrieveTableRsp);
+
+  memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+  SRetrieveTableRsp* pRsp = pRet->rsp;
+
+  pRsp->completed = true;
+}
+
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   void    *pCont = pReadMsg->pCont;
   int32_t  contLen = pReadMsg->contLen;
@@ -98,6 +132,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
       vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     } else {
       assert(*qhandle == (void*) killQueryMsg->qhandle);
+
       qKillQuery(*qhandle);
       qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
     }
@@ -110,7 +145,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
-    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo);
+    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
 
     SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code    = code;
@@ -133,7 +168,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
         pRsp->qhandle = htobe64((uint64_t) pQInfo);
       }
 
-      pQInfo = NULL;
       if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
         vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
@@ -153,16 +187,34 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   } else {
     assert(pCont != NULL);
+
     handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont);
     if (handle == NULL) {
       vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle);
       code = TSDB_CODE_QRY_INVALID_QHANDLE;
     } else {
       vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont);
-      code = TSDB_CODE_VND_ACTION_IN_PROGRESS;
-      qTableQuery(*handle); // do execute query
+
+      bool freehandle = false;
+      bool buildRes = qTableQuery(*handle); // do execute query
+
+      // build query rsp
+      if (buildRes) {
+        // update the connection info according to the retrieve connection
+        pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle);
+        assert(pReadMsg->rpcMsg.handle != NULL);
+
+        vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+        code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle);
+
+        // todo test the error code case
+        if (code == TSDB_CODE_SUCCESS) {
+          code = TSDB_CODE_QRY_HAS_RSP;
+        }
+      }
+
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle);
     }
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
   }
 
   return code;
@@ -176,7 +228,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
   pRetrieve->free = htons(pRetrieve->free);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
@@ -185,16 +237,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
     vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
-
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->useconds  = 0;
-    pRsp->completed = true;
-
+    
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
@@ -203,35 +247,25 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
 
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->completed = true;
-    pRsp->useconds  = 0;
-
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
   bool freeHandle = true;
-  code = qRetrieveQueryResultInfo(*handle);
+  bool buildRes   = false;
+
+  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
   if (code != TSDB_CODE_SUCCESS) {
     //TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  } else { // if failed to dump result, free qhandle immediately
-    if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) {
-      if (qHasMoreResultsToRetrieve(*handle)) {
-        vnodePutItemIntoReadQueue(pVnode, *handle);
-        pRet->qhandle = *handle;
-        freeHandle = false;
-      } else {
-        qKillQuery(*handle);
-        freeHandle = true;
-      }
+  } else { // result is not ready, return immediately
+    if (!buildRes) {
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      return TSDB_CODE_QRY_NOT_READY;
     }
+
+    code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle);
   }
 
   qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle);
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 6b9b8ca4fd5a5c42028936e48b53f45ffa811903..900ff1fbbacc034b20bc0a04df5758e595bb3f32 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -78,6 +78,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     // assign version
     pVnode->version++;
     pHead->version = pVnode->version;
+    if (pVnode->delay) usleep(pVnode->delay*1000);
+
   } else { // from wal or forward 
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt
index 684616ba0e734375663e57cc6ad177c1f0c7f99e..a9c20f643a18eefad94dfb245fcd9692048ee3bf 100644
--- a/src/wal/CMakeLists.txt
+++ b/src/wal/CMakeLists.txt
@@ -1,9 +1,9 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(inc)
 
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC)
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
index a0e2ccb51d9d8a27bdf7482873f7c7ad76c7f607..bf3b0f63ea85be6a491bafdfc6f8d9928f3e6ccd 100644
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@@ -13,14 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h> 
-
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tlog.h"
 #include "tchecksum.h"
@@ -29,7 +22,6 @@
 #include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
-#include "tfile.h"
 
 #define walPrefix "wal"
 
@@ -109,7 +101,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     }
   }
 
-  if (tmkdir(path, 0755) != 0) {
+  if (taosMkDir(path, 0755) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
     walRelease(pWal);
@@ -214,7 +206,7 @@ int walWrite(void *handle, SWalHead *pHead) {
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(twrite(pWal->fd, pHead, contLen) != contLen) {
+  if(taosTWrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -373,7 +365,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   wDebug("wal:%s, start to restore", name);
 
   while (1) {
-    int ret = tread(fd, pHead, sizeof(SWalHead));
+    int ret = taosTRead(fd, pHead, sizeof(SWalHead));
     if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
@@ -388,7 +380,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
       break;
     } 
 
-    ret = tread(fd, pHead->cont, pHead->len);
+    ret = taosTRead(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -428,7 +420,7 @@ int walHandleExistingFiles(const char *path) {
       if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
         snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
         snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
-        if (tmkdir(opath, 0755) != 0) {
+        if (taosMkDir(opath, 0755) != 0) {
           wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
           terrno = TAOS_SYSTEM_ERROR(errno);
           break; 
diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt
index f007bc38826274d104cdd14e6cdb276b32da9cfd..bfa6aa15401ef4aa0550ebc154a41969446f2e99 100644
--- a/src/wal/test/CMakeLists.txt
+++ b/src/wal/test/CMakeLists.txt
@@ -2,15 +2,15 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(../inc)
 
   LIST(APPEND WALTEST_SRC ./waltest.c)
   ADD_EXECUTABLE(waltest ${WALTEST_SRC})
-  TARGET_LINK_LIBRARIES(waltest twal)
+  TARGET_LINK_LIBRARIES(waltest twal osdetail tutil)
 
 ENDIF ()
 
diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
index 4e307db07935be293e4ec74566aa158fb1fece9e..52067343ef051799150c65f91643a6475ad68157 100644
--- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
+++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
@@ -94,7 +94,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>24.1.1</version>
+      <version>29.0-jre</version>
     </dependency>
 
     <dependency>
diff --git a/tests/examples/go/src/taosapp/taosapp.go b/tests/examples/go/src/taosapp/taosapp.go
index 5c7b92f6fbc22d8f5a784409b797891a19829417..30126ea571beeb84f8a46e7053ede97f8202ee71 100644
--- a/tests/examples/go/src/taosapp/taosapp.go
+++ b/tests/examples/go/src/taosapp/taosapp.go
@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 package main
- 
+
 import (
 	"database/sql"
     "time"
     "log"
     "fmt"
-	_ "taosSql"
+	_ "github.com/taosdata/driver-go/taosSql"
 )
 
 func main() {
@@ -29,7 +29,7 @@ func main() {
 
     fmt.Printf("\n======== start demo test ========\n")
     // open connect to taos server
-    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/demodb")
+    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/")
     if err != nil {
         log.Fatalf("Open database error: %s\n", err)
     }
@@ -58,11 +58,11 @@ func main() {
 
 func drop_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
-    res, err := db.Exec("drop database " + demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database if exists " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -73,10 +73,10 @@ func create_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // create database
     res, err := db.Exec("create database " + demodb)
-    checkErr(err)
+    checkErr(err, "create db, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -89,10 +89,10 @@ func use_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // use database
     res, err := db.Exec("use " +  demodb)  // notes: must no quote to db name
-    checkErr(err)
+    checkErr(err, "use db db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -103,10 +103,10 @@ func create_table(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
     // create table
     res, err := db.Exec("create table " + demot + " (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)")
-    checkErr(err)
+    checkErr(err, "create table db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -120,10 +120,10 @@ func insert_data(db *sql.DB, demot string) {
                                       " (now+1s, 101, 'shanghai', 11, true, 'two', 789.123, 789.123)" +
                                       " (now+2s, 102, 'shenzhen', 12,  false, 'three', 456.789, 456.789)")
 
-    checkErr(err)
+    checkErr(err, "insert data, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "insert data res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -133,7 +133,7 @@ func select_data(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     rows, err := db.Query("select * from ? " , demot)  // go text mode
-    checkErr(err)
+    checkErr(err, "select db.Query")
 
     fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %4s%s %5s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
@@ -148,7 +148,7 @@ func select_data(db *sql.DB, demot string) {
         var dv float64
 
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
-        checkErr(err)
+        checkErr(err, "select rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -170,15 +170,11 @@ func select_data(db *sql.DB, demot string) {
 func drop_database_stmt(db *sql.DB,demodb string) {
     st := time.Now().Nanosecond()
     // drop test db
-    stmt, err := db.Prepare("drop database ?")
-    checkErr(err)
-    defer stmt.Close()
-
-    res, err := stmt.Exec(demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -189,15 +185,15 @@ func create_database_stmt(db *sql.DB,demodb string)  {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("create database ?")
-    checkErr(err)
+    checkErr(err, "create db, db.Prepare")
 
     //var res driver.Result
     res, err := stmt.Exec(demodb)
-    checkErr(err)
+    checkErr(err, "create db, stmt.Exec")
 
     //fmt.Printf("Query OK, %d row(s) affected()", res.RowsAffected())
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -208,13 +204,13 @@ func use_database_stmt (db *sql.DB,demodb string) {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("use " + demodb)
-    checkErr(err)
+    checkErr(err, "use db, db.Prepare")
 
     res, err := stmt.Exec()
-    checkErr(err)
+    checkErr(err, "use db, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -224,14 +220,14 @@ func create_table_stmt (db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // create table
     // (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)
-    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(8), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
-    checkErr(err)
+    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(10), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
+    checkErr(err, "create table db.Prepare")
 
     res, err := stmt.Exec(demot, "ts", "id", "name", "len", "flag", "notes", "fv", "dv")
-    checkErr(err)
+    checkErr(err, "create table stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -241,15 +237,15 @@ func insert_data_stmt(db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // insert data into table
     stmt, err := db.Prepare("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?)")
-    checkErr(err)
+    checkErr(err, "insert db.Prepare")
 
     res, err := stmt.Exec(demot, "now" ,    1000, "'haidian'" ,    6, true, "'AI world'",  6987.654, 321.987,
                                  "now+1s", 1001, "'changyang'" ,  7, false,  "'DeepMode'", 12356.456, 128634.456,
                                  "now+2s", 1002, "'chuangping'" , 8, true, "'database'",  3879.456, 65433478.456,)
-    checkErr(err)
+    checkErr(err, "insert data, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -259,12 +255,12 @@ func select_data_stmt(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     stmt, err := db.Prepare("select ?, ?, ?, ?, ?, ?, ?, ? from ?" )  // go binary mode
-    checkErr(err)
+    checkErr(err, "db.Prepare")
 
     rows, err := stmt.Query("ts", "id","name","len", "flag","notes", "fv", "dv", demot)
-    checkErr(err)
+    checkErr(err, "stmt.Query")
 
-    fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
+    fmt.Printf("%10s%s%8s %5s %8s%s %s %10s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
     for rows.Next() {
         var ts string
@@ -279,7 +275,7 @@ func select_data_stmt(db *sql.DB, demot string) {
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
         //fmt.Println("start scan fields from row.rs, &fv:", &fv)
         //err = rows.Scan(&fv)
-        checkErr(err)
+        checkErr(err, "rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -298,8 +294,9 @@ func select_data_stmt(db *sql.DB, demot string) {
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
 }
 
-func checkErr(err error) {
+func checkErr(err error, prompt string) {
     if err != nil {
+	fmt.Printf("%s\n", prompt)
         panic(err)
     }
 }
diff --git a/tests/perftest-scripts/cassandraTestQ1Loop.sh b/tests/perftest-scripts/cassandraTestQ1Loop.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e65ec2575c8070041c93a369dbb1b261d9cb37b2
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ1Loop.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ1OUT=cassandraTestQ1.out
+
+function runTest {
+  totalG0=0
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q1.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q1.txt \
+      > $CASTESTQ1OUT
+    G0=`grep "devgroup=0" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG0=`echo "scale=4; $totalG0 + $G0" | bc`
+    G10=`grep "devgroup=10" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup=20" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup=30" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup=40" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup=50" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup=60" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup=70" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup=80" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup=90" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+  done
+  avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90"
+  echo "Cassandra, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90"
+}
+
+################ Main ################
+
+master=false
+develop=true
+verbose=false
+
+clients=1
+
+while : ; do
+  case $1 in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      break ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ2Loop.sh b/tests/perftest-scripts/cassandraTestQ2Loop.sh
new file mode 100755
index 0000000000000000000000000000000000000000..76e42cf89d6a8f0d1a395b41d9a357def3d37efb
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ2Loop.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ2OUT=cassandraTestQ2.out
+
+function runTest {
+  totalCount10=0
+  totalCount20=0
+  totalCount30=0
+  totalCount40=0
+  totalCount50=0
+  totalCount60=0
+  totalCount70=0
+  totalCount80=0
+  totalCount90=0
+  totalCount100=0
+
+  totalAvg10=0
+  totalAvg20=0
+  totalAvg30=0
+  totalAvg40=0
+  totalAvg50=0
+  totalAvg60=0
+  totalAvg70=0
+  totalAvg80=0
+  totalAvg90=0
+  totalAvg100=0
+
+  totalSum10=0
+  totalSum20=0
+  totalSum30=0
+  totalSum40=0
+  totalSum50=0
+  totalSum60=0
+  totalSum70=0
+  totalSum80=0
+  totalSum90=0
+  totalSum100=0
+
+  totalMax10=0
+  totalMax20=0
+  totalMax30=0
+  totalMax40=0
+  totalMax50=0
+  totalMax60=0
+  totalMax70=0
+  totalMax80=0
+  totalMax90=0
+  totalMax100=0
+
+  totalMin10=0
+  totalMin20=0
+  totalMin30=0
+  totalMin40=0
+  totalMin50=0
+  totalMin60=0
+  totalMin70=0
+  totalMin80=0
+  totalMin90=0
+  totalMin100=0
+
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q2.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q2.txt \
+      > $CASTESTQ2OUT
+
+    Count10=`cat $CASTESTQ2OUT | grep count | grep "devgroup<10" | awk '{print $2}'`
+    totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc`
+    Count20=`cat $CASTESTQ2OUT | grep count | grep "devgroup<20" | awk '{print $2}'`
+    totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc`
+    Count30=`cat $CASTESTQ2OUT | grep count | grep "devgroup<30" | awk '{print $2}'`
+    totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc`
+    Count40=`cat $CASTESTQ2OUT | grep count | grep "devgroup<40" | awk '{print $2}'`
+    totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc`
+    Count50=`cat $CASTESTQ2OUT | grep count | grep "devgroup<50" | awk '{print $2}'`
+    totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc`
+    Count60=`cat $CASTESTQ2OUT | grep count | grep "devgroup<60" | awk '{print $2}'`
+    totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc`
+    Count70=`cat $CASTESTQ2OUT | grep count | grep "devgroup<70" | awk '{print $2}'`
+    totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc`
+    Count80=`cat $CASTESTQ2OUT | grep count | grep "devgroup<80" | awk '{print $2}'`
+    totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc`
+    Count90=`cat $CASTESTQ2OUT | grep count | grep "devgroup<90" | awk '{print $2}'`
+    totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc`
+    Count100=`cat $CASTESTQ2OUT | grep count | grep "test allow filtering;" | awk '{print $2}'`
+    totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc`
+
+    Avg10=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<10" | awk '{print $2}'`
+    totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc`
+    Avg20=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<20" | awk '{print $2}'`
+    totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc`
+    Avg30=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<30" | awk '{print $2}'`
+    totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc`
+    Avg40=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<40" | awk '{print $2}'`
+    totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc`
+    Avg50=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<50" | awk '{print $2}'`
+    totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc`
+    Avg60=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<60" | awk '{print $2}'`
+    totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc`
+    Avg70=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<70" | awk '{print $2}'`
+    totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc`
+    Avg80=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<80" | awk '{print $2}'`
+    totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc`
+    Avg90=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<90" | awk '{print $2}'`
+    totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc`
+    Avg100=`cat $CASTESTQ2OUT | grep avg | grep "test allow filtering;" | awk '{print $2}'`
+    totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc`
+
+    Sum10=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<10" | awk '{print $2}'`
+    totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc`
+    Sum20=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<20" | awk '{print $2}'`
+    totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc`
+    Sum30=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<30" | awk '{print $2}'`
+    totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc`
+    Sum40=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<40" | awk '{print $2}'`
+    totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc`
+    Sum50=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<50" | awk '{print $2}'`
+    totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc`
+    Sum60=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<60" | awk '{print $2}'`
+    totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc`
+    Sum70=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<70" | awk '{print $2}'`
+    totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc`
+    Sum80=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<80" | awk '{print $2}'`
+    totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc`
+    Sum90=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<90" | awk '{print $2}'`
+    totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc`
+    Sum100=`cat $CASTESTQ2OUT | grep sum | grep "test allow filtering;" | awk '{print $2}'`
+    totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc`
+
+    Max10=`cat $CASTESTQ2OUT | grep max | grep "devgroup<10" | awk '{print $2}'`
+    totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc`
+    Max20=`cat $CASTESTQ2OUT | grep max | grep "devgroup<20" | awk '{print $2}'`
+    totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc`
+    Max30=`cat $CASTESTQ2OUT | grep max | grep "devgroup<30" | awk '{print $2}'`
+    totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc`
+    Max40=`cat $CASTESTQ2OUT | grep max | grep "devgroup<40" | awk '{print $2}'`
+    totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc`
+    Max50=`cat $CASTESTQ2OUT | grep max | grep "devgroup<50" | awk '{print $2}'`
+    totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc`
+    Max60=`cat $CASTESTQ2OUT | grep max | grep "devgroup<60" | awk '{print $2}'`
+    totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc`
+    Max70=`cat $CASTESTQ2OUT | grep max | grep "devgroup<70" | awk '{print $2}'`
+    totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc`
+    Max80=`cat $CASTESTQ2OUT | grep max | grep "devgroup<80" | awk '{print $2}'`
+    totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc`
+    Max90=`cat $CASTESTQ2OUT | grep max | grep "devgroup<90" | awk '{print $2}'`
+    totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc`
+    Max100=`cat $CASTESTQ2OUT | grep max | grep "test allow filtering;" | awk '{print $2}'`
+    totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc`
+
+    Min10=`cat $CASTESTQ2OUT | grep min | grep "devgroup<10" | awk '{print $2}'`
+    totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc`
+    Min20=`cat $CASTESTQ2OUT | grep min | grep "devgroup<20" | awk '{print $2}'`
+    totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc`
+    Min30=`cat $CASTESTQ2OUT | grep min | grep "devgroup<30" | awk '{print $2}'`
+    totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc`
+    Min40=`cat $CASTESTQ2OUT | grep min | grep "devgroup<40" | awk '{print $2}'`
+    totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc`
+    Min50=`cat $CASTESTQ2OUT | grep min | grep "devgroup<50" | awk '{print $2}'`
+    totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc`
+    Min60=`cat $CASTESTQ2OUT | grep min | grep "devgroup<60" | awk '{print $2}'`
+    totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc`
+    Min70=`cat $CASTESTQ2OUT | grep min | grep "devgroup<70" | awk '{print $2}'`
+    totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc`
+    Min80=`cat $CASTESTQ2OUT | grep min | grep "devgroup<80" | awk '{print $2}'`
+    totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc`
+    Min90=`cat $CASTESTQ2OUT | grep min | grep "devgroup<90" | awk '{print $2}'`
+    totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc`
+    Min100=`cat $CASTESTQ2OUT | grep min | grep "test allow filtering;" | awk '{print $2}'`
+    totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc`
+
+  done
+  avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100"
+  echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100"
+  echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100"
+  echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100"
+  echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100"
+}
+
+################ Main ################
+
+verbose=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ3Loop.sh b/tests/perftest-scripts/cassandraTestQ3Loop.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ce81fcb818bd90bd99ab9b31a58db611445efffd
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ3Loop.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ3OUT=cassandraTestQ3.out
+
+function runTest {
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  totalG100=0
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q3.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q3.txt \
+      > $CASTESTQ3OUT
+    G10=`grep "devgroup<10" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup<20" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup<30" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup<40" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup<50" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup<60" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup<70" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup<80" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup<90" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+    G100=`grep "test group by " $CASTESTQ3OUT| awk '{print $2}'`
+    totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
+  done
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
+}
+
+################ Main ################
+
+verbose=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ4Loop.sh b/tests/perftest-scripts/cassandraTestQ4Loop.sh
new file mode 100755
index 0000000000000000000000000000000000000000..85e45787b89a7700f0b6de3eb87ee8143f05ca92
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ4Loop.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+DATA_DIR=/mnt/root/testdata
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ4OUT=cassandraTestQ4.out
+
+function runTest {
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  totalG100=0
+  for i in `seq 1 $NUM_LOOP`; do
+    if $regeneratedata ; then
+      printTo "java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+        -datadir $DATA_DIR \
+        -numofFiles 100 \
+        -rowsperrequest 100 \
+        -writeclients 4 \
+        -conf $CASTEST_DIR/application.conf \
+        -timetest"
+      java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+        -datadir $DATA_DIR \
+        -numofFiles 100 \
+        -rowsperrequest 100 \
+        -writeclients 4 \
+        -conf $CASTEST_DIR/application.conf \
+        -timetest
+    fi 
+
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q4.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q4.txt \
+      > $CASTESTQ4OUT
+    G10=`grep "devgroup<10" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup<20" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup<30" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup<40" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup<50" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup<60" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup<70" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup<80" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup<90" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+    G100=`grep "test group by minute;" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
+  done
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
+}
+
+################ Main ################
+
+master=false
+develop=true
+verbose=false
+regeneratedata=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -r)
+      regeneratedata=true
+      ;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestWriteLoop.sh b/tests/perftest-scripts/cassandraTestWriteLoop.sh
index 4e42c4b3df4b9bb5c606a58541fd65cd395166e2..a218f0d0a04d30870e50eb06542c76cbd508662b 100755
--- a/tests/perftest-scripts/cassandraTestWriteLoop.sh
+++ b/tests/perftest-scripts/cassandraTestWriteLoop.sh
@@ -3,7 +3,6 @@
 DATA_DIR=/mnt/root/testdata
 NUM_LOOP=1
 NUM_OF_FILES=100
-OUT_FILE=cassandraWrite.out
 
 rowsPerRequest=(1 10 50 100 500 1000 2000)
 
@@ -14,46 +13,56 @@ function printTo {
 }
 
 function runTest {
-  for c in `seq 1 $clients`; do
-    avgRPR[$c]=0
-  done
+  declare -A avgRPR
 
-  printf "R/R, "
-  for c in `seq 1 $clients`; do
-    if [ "$c" == "1" ]; then
-      printf "$c client, "
-    else
-      printf "$c clients, "
-    fi
+  for r in ${!rowsPerRequest[@]}; do
+    for c in `seq 1 $clients`; do
+      avgRPR[$r, $c]=0
+    done
   done
-  printf "\n"
 
-  for r in ${rowsPerRequest[@]}; do
-    printf "$r, "
+  for r in ${!rowsPerRequest[@]}; do
     for c in `seq 1 $clients`; do
       totalRPR=0
+      OUT_FILE=cassandraWrite-rows${rowsPerRequest[$r]}-clients$c.out
       for i in `seq 1 $NUM_LOOP`; do
         printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
             -datadir $DATA_DIR \
             -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
+            -rowsperrequest ${rowsPerRequest[$r]} \
             -writeclients $c \
             -conf $CAS_TEST_DIR/application.conf"
         java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
             -datadir $DATA_DIR \
             -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
+            -rowsperrequest ${rowsPerRequest[$r]} \
             -writeclients $c \
             -conf $CAS_TEST_DIR/application.conf \
-            2>&1 > $OUT_FILE
+            2>&1 | tee $OUT_FILE
         RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'`
         totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
-        printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
+        printTo "r:$r rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR"
       done
-      avgRPR[$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+      avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+      printTo "r:$r c:$c avgRPR:${avgRPR[$r,$c]}"
     done
+
+  done
+
+  printf "R/R, "
+  for c in `seq 1 $clients`; do
+    if [ "$c" == "1" ]; then
+      printf "$c client, "
+    else
+      printf "$c clients, "
+    fi
+  done
+  printf "\n"
+
+  for r in ${!rowsPerRequest[@]}; do
+    printf "${rowsPerRequest[$r]}, "
     for c in `seq 1 $clients`; do
-      printf "${avgRPR[$c]}, "
+      printf "${avgRPR[$r,$c]}, "
     done
     printf "\n"
   done
diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py
index 6ac12c679554862e98a080fe91d4baab29556055..36c06ac09daf26ef0e23130a01111919501c6f46 100644
--- a/tests/pytest/client/client.py
+++ b/tests/pytest/client/client.py
@@ -42,8 +42,8 @@ class TDTestCase:
 
         ret = tdSql.query('show dnodes')
 
-        ret = tdSql.execute('alter dnode "%s" debugFlag 135' % tdSql.getData(0,1))
-        tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (tdSql.getData(0, 1), ret))
+        ret = tdSql.execute('alter dnode "%s" debugFlag 135' % tdSql.getData(0,0))
+        tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (tdSql.getData(0, 0), ret))
 
     def stop(self):
         tdSql.close()
diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py
index 9e38e04b6392edf74a2d9c9648c5a8b7637bbe2a..98181180e200fade047cfcc8f1be68434220af13 100755
--- a/tests/pytest/crash_gen.py
+++ b/tests/pytest/crash_gen.py
@@ -42,6 +42,13 @@ import os
 import io
 import signal
 import traceback
+
+try:
+    import psutil
+except:
+    print("Psutil module needed, please install: sudo pip3 install psutil")
+    sys.exit(-1)
+
 # Require Python 3
 if sys.version_info[0] < 3:
     raise Exception("Must be using Python 3")
@@ -52,13 +59,12 @@ if sys.version_info[0] < 3:
 # Command-line/Environment Configurations, will set a bit later
 # ConfigNameSpace = argparse.Namespace
 gConfig = argparse.Namespace()  # Dummy value, will be replaced later
+gSvcMgr = None # TODO: refactor this hack, use dep injection
 logger = None
 
-
 def runThread(wt: WorkerThread):
     wt.run()
 
-
 class CrashGenError(Exception):
     def __init__(self, msg=None, errno=None):
         self.msg = msg
@@ -69,8 +75,7 @@ class CrashGenError(Exception):
 
 
 class WorkerThread:
-    def __init__(self, pool: ThreadPool, tid,
-                 tc: ThreadCoordinator,
+    def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator,
                  # te: TaskExecutor,
                  ):  # note: main thread context!
         # self._curStep = -1
@@ -131,18 +136,28 @@ class WorkerThread:
 
         # clean up
         if (gConfig.per_thread_db_connection):  # type: ignore
-            self._dbConn.close()
+            if self._dbConn.isOpen: #sometimes it is not open
+                self._dbConn.close()
+            else:
+                logger.warning("Cleaning up worker thread, dbConn already closed")
 
     def _doTaskLoop(self):
         # while self._curStep < self._pool.maxSteps:
         # tc = ThreadCoordinator(None)
         while True:
             tc = self._tc  # Thread Coordinator, the overall master
-            tc.crossStepBarrier()  # shared barrier first, INCLUDING the last one
+            try:
+                tc.crossStepBarrier()  # shared barrier first, INCLUDING the last one
+            except threading.BrokenBarrierError as err: # main thread timed out
+                print("_bto", end="")
+                logger.debug("[TRD] Worker thread exiting due to main thread barrier time-out")
+                break
+
             logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid))
             self.crossStepGate()   # then per-thread gate, after being tapped
             logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid))
             if not self._tc.isRunning():
+                print("_wts", end="")
                 logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
                 break
 
@@ -159,6 +174,7 @@ class WorkerThread:
             logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid))
 
             self._dbInUse = False  # there may be changes between steps
+        # print("_wtd", end=None) # worker thread died
 
     def verifyThreadSelf(self):  # ensure we are called by this own thread
         if (threading.get_ident() != self._thread.ident):
@@ -187,30 +203,24 @@ class WorkerThread:
         # self._curStep += 1  # off to a new step...
 
     def tapStepGate(self):  # give it a tap, release the thread waiting there
-        self.verifyThreadAlive()
+        # self.verifyThreadAlive()
         self.verifyThreadMain()  # only allowed for main thread
 
-        logger.debug("[TRD] Tapping worker thread {}".format(self._tid))
-        self._stepGate.set()  # wake up!
-        time.sleep(0)  # let the released thread run a bit
+        if self._thread.is_alive():
+            logger.debug("[TRD] Tapping worker thread {}".format(self._tid))
+            self._stepGate.set()  # wake up!
+            time.sleep(0)  # let the released thread run a bit
+        else:
+            print("_tad", end="") # Thread already dead
 
     def execSql(self, sql):  # TODO: expose DbConn directly
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.execute(sql)
-        else:
-            return self._tc.getDbManager().getDbConn().execute(sql)
+        return self.getDbConn().execute(sql)
 
     def querySql(self, sql):  # TODO: expose DbConn directly
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.query(sql)
-        else:
-            return self._tc.getDbManager().getDbConn().query(sql)
+        return self.getDbConn().query(sql)
 
     def getQueryResult(self):
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.getQueryResult()
-        else:
-            return self._tc.getDbManager().getDbConn().getQueryResult()
+        return self.getDbConn().getQueryResult()
 
     def getDbConn(self):
         if (gConfig.per_thread_db_connection):
@@ -228,6 +238,8 @@ class WorkerThread:
 
 
 class ThreadCoordinator:
+    WORKER_THREAD_TIMEOUT = 30
+
     def __init__(self, pool: ThreadPool, dbManager):
         self._curStep = -1  # first step is 0
         self._pool = pool
@@ -248,14 +260,14 @@ class ThreadCoordinator:
     def getDbManager(self) -> DbManager:
         return self._dbManager
 
-    def crossStepBarrier(self):
-        self._stepBarrier.wait()
+    def crossStepBarrier(self, timeout=None):
+        self._stepBarrier.wait(timeout) 
 
     def requestToStop(self):
         self._runStatus = MainExec.STATUS_STOPPING
         self._execStats.registerFailure("User Interruption")
 
-    def _runShouldEnd(self, transitionFailed, hasAbortedTask):
+    def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout):
         maxSteps = gConfig.max_steps  # type: ignore
         if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9
             return True
@@ -265,6 +277,8 @@ class ThreadCoordinator:
             return True
         if hasAbortedTask:
             return True
+        if workerTimeout:
+            return True
         return False
 
     def _hasAbortedTask(self): # from execution of previous step
@@ -296,7 +310,7 @@ class ThreadCoordinator:
         # let other threads go past the pool barrier, but wait at the
         # thread gate
         logger.debug("[TRD] Main thread about to cross the barrier")
-        self.crossStepBarrier()
+        self.crossStepBarrier(timeout=self.WORKER_THREAD_TIMEOUT)
         self._stepBarrier.reset()  # Other worker threads should now be at the "gate"
         logger.debug("[TRD] Main thread finished crossing the barrier")
 
@@ -327,6 +341,7 @@ class ThreadCoordinator:
                 # end, and maybe signal them to stop
             else:
                 raise
+        return transitionFailed
 
         self.resetExecutedTasks()  # clear the tasks after we are done
         # Get ready for next step
@@ -342,11 +357,21 @@ class ThreadCoordinator:
         self._execStats.startExec()  # start the stop watch
         transitionFailed = False
         hasAbortedTask = False
-        while not self._runShouldEnd(transitionFailed, hasAbortedTask):
+        workerTimeout = False
+        while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout):
             if not gConfig.debug: # print this only if we are not in debug mode                
                 print(".", end="", flush=True)
                         
-            self._syncAtBarrier() # For now just cross the barrier
+            try:
+                self._syncAtBarrier() # For now just cross the barrier
+            except threading.BrokenBarrierError as err:
+                logger.info("Main loop aborted, caused by worker thread time-out")
+                self._execStats.registerFailure("Aborted due to worker thread timeout")
+                print("\n\nWorker Thread time-out detected, important thread info:")
+                ts = ThreadStacks()
+                ts.print(filterInternal=True)
+                workerTimeout = True
+                break
 
             # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate"
             # We use this period to do house keeping work, when all worker
@@ -358,12 +383,20 @@ class ThreadCoordinator:
                 break # do transition only if tasks are error free
 
             # Ending previous step
-            transitionFailed = self._doTransition() # To start, we end step -1 first
+            try:
+                transitionFailed = self._doTransition() # To start, we end step -1 first
+            except taos.error.ProgrammingError as err:
+                transitionFailed = True
+                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+                logger.info("Transition failed: errno=0x{:X}, msg: {}".format(errno2, err))
+
             # Then we move on to the next step
             self._releaseAllWorkerThreads(transitionFailed)                    
 
         if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate"
             logger.debug("Abnormal ending of main thraed")
+        elif workerTimeout:
+            logger.debug("Abnormal ending of main thread, due to worker timeout")
         else: # regular ending, workers waiting at "barrier"
             logger.debug("Regular ending, main thread waiting for all worker threads to stop...")
             self._syncAtBarrier()
@@ -561,6 +594,10 @@ class DbConn:
     def __init__(self):
         self.isOpen = False
         self._type = self.TYPE_INVALID
+        self._lastSql = None
+
+    def getLastSql(self):
+        return self._lastSql
 
     def open(self):
         if (self.isOpen):
@@ -569,9 +606,7 @@ class DbConn:
         # below implemented by child classes
         self.openByType()
 
-        logger.debug(
-            "[DB] data connection opened, type = {}".format(
-                self._type))
+        logger.debug("[DB] data connection opened, type = {}".format(self._type))
         self.isOpen = True
 
     def resetDb(self):  # reset the whole database, etc.
@@ -594,21 +629,29 @@ class DbConn:
 
     def _queryAny(self, sql):  # actual query result as an int
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot query database until connection is open")
+            raise RuntimeError("Cannot query database until connection is open")
         nRows = self.query(sql)
         if nRows != 1:
-            raise RuntimeError(
-                "Unexpected result for query: {}, rows = {}".format(
-                    sql, nRows))
+            raise RuntimeError("Unexpected result for query: {}, rows = {}".format(sql, nRows))
         if self.getResultRows() != 1 or self.getResultCols() != 1:
-            raise RuntimeError(
-                "Unexpected result set for query: {}".format(sql))
+            raise RuntimeError("Unexpected result set for query: {}".format(sql))
         return self.getQueryResult()[0][0]
 
+    def use(self, dbName):
+        self.execute("use {}".format(dbName))
+
+    def hasDatabases(self):
+        return self.query("show databases") > 0
+
+    def hasTables(self):
+        return self.query("show tables") > 0
+
     def execute(self, sql):
         raise RuntimeError("Unexpected execution, should be overriden")
 
+    def query(self, sql) -> int: # return num rows returned
+        raise RuntimeError("Unexpected execution, should be overriden")
+
     def openByType(self):
         raise RuntimeError("Unexpected execution, should be overriden")
 
@@ -643,10 +686,11 @@ class DbConnRest(DbConn):
         self.isOpen = False
 
     def _doSql(self, sql):
+        self._lastSql = sql # remember this, last SQL attempted
         try:
             r = requests.post(self._url, 
                 data = sql,
-                auth = HTTPBasicAuth('root', 'taosdata'))            
+                auth = HTTPBasicAuth('root', 'taosdata'))         
         except:
             print("REST API Failure (TODO: more info here)")
             raise
@@ -742,11 +786,16 @@ class MyTDSql:
 
 
 class DbConnNative(DbConn):
+    # Class variables
+    _lock = threading.Lock()
+    _connInfoDisplayed = False
+
     def __init__(self):
         super().__init__()
         self._type = self.TYPE_NATIVE
         self._conn = None
         self._cursor = None
+        
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -755,31 +804,32 @@ class DbConnNative(DbConn):
         else:
             projPath = selfPath[:selfPath.find("tests")]
 
+        buildPath = None
         for root, dirs, files in os.walk(projPath):
             if ("taosd" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
+        if buildPath == None:
+            raise RuntimeError("Failed to determine buildPath, selfPath={}".format(self_path))
         return buildPath
 
-    connInfoDisplayed = False
+    
     def openByType(self):  # Open connection
         cfgPath = self.getBuildPath() + "/test/cfg"
         hostAddr = "127.0.0.1"
-        if not self.connInfoDisplayed:
-            logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
-            self.connInfoDisplayed = True
-
-        self._conn = taos.connect(
-            host=hostAddr,
-            config=cfgPath)  # TODO: make configurable
-        self._cursor = self._conn.cursor()
 
-        # Get the connection/cursor ready
+        with self._lock: # force single threading for opening DB connections
+            if not self._connInfoDisplayed:
+                self.__class__._connInfoDisplayed = True # updating CLASS variable
+                logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
+            
+            self._conn = taos.connect(host=hostAddr, config=cfgPath)  # TODO: make configurable
+            self._cursor = self._conn.cursor()
+        
         self._cursor.execute('reset query cache')
         # self._cursor.execute('use db') # do this at the beginning of every
-        # step
 
         # Open connection
         self._tdSql = MyTDSql()
@@ -984,29 +1034,11 @@ class StateDbOnly(AnyState):
         if (not self.hasTask(tasks, TaskCreateDb)):
             # only if we don't create any more
             self.assertAtMostOneSuccess(tasks, TaskDropDb)
-        self.assertIfExistThenSuccess(tasks, TaskDropDb)
-        # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not true in massively parrallel cases
-        # Nothing to be said about adding data task
-        # if ( self.hasSuccess(tasks, DropDbTask) ): # dropped the DB
-        # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess
-        # self.assertAtMostOneSuccess(tasks, DropDbTask)
-        # self._state = self.STATE_EMPTY
-        # if ( self.hasSuccess(tasks, TaskCreateSuperTable) ): # did not drop db, create table success
-        #     # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table
-        #     if ( not self.hasTask(tasks, TaskDropSuperTable) ):
-        #         self.assertAtMostOneSuccess(tasks, TaskCreateSuperTable) # at most 1 attempt is successful, if we don't drop anything
-        # self.assertNoTask(tasks, DropDbTask) # should have have tried
-        # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet
-        #     # can't say there's add-data attempts, since they may all fail
-        #     self._state = self.STATE_TABLE_ONLY
-        # else:
-        #     self._state = self.STATE_HAS_DATA
-        # What about AddFixedData?
-        # elif ( self.hasSuccess(tasks, AddFixedDataTask) ):
-        #     self._state = self.STATE_HAS_DATA
-        # else: # no success in dropping db tasks, no success in create fixed table? read data should also fail
-        #     # raise RuntimeError("Unexpected no-success scenario")   # We might just landed all failure tasks,
-        #     self._state = self.STATE_DB_ONLY  # no change
+
+        # TODO: restore the below, the problem exists, although unlikely in real-world
+        # if (gSvcMgr!=None) and gSvcMgr.isRestarting():     
+        # if (gSvcMgr == None) or (not gSvcMgr.isRestarting()) : 
+        #     self.assertIfExistThenSuccess(tasks, TaskDropDb)       
 
 
 class StateSuperTableOnly(AnyState):
@@ -1082,7 +1114,7 @@ class StateMechine:
         self._curState = self._findCurrentState()  # starting state
         # transitition target probabilities, indexed with value of STATE_EMPTY,
         # STATE_DB_ONLY, etc.
-        self._stateWeights = [1, 3, 5, 15]
+        self._stateWeights = [1, 2, 10, 40]
 
     def getCurrentState(self):
         return self._curState
@@ -1128,33 +1160,22 @@ class StateMechine:
     def _findCurrentState(self):
         dbc = self._dbConn
         ts = time.time()  # we use this to debug how fast/slow it is to do the various queries to find the current DB state
-        if dbc.query("show databases") == 0:  # no database?!
-            # logger.debug("Found EMPTY state")
-            logger.debug(
-                "[STT] empty database found, between {} and {}".format(
-                    ts, time.time()))
+        if not dbc.hasDatabases():  # no database?!
+            logger.debug( "[STT] empty database found, between {} and {}".format(ts, time.time()))
             return StateEmpty()
         # did not do this when openning connection, and this is NOT the worker
         # thread, which does this on their own
-        dbc.execute("use db")
-        if dbc.query("show tables") == 0:  # no tables
-            # logger.debug("Found DB ONLY state")
-            logger.debug(
-                "[STT] DB_ONLY found, between {} and {}".format(
-                    ts, time.time()))
+        dbc.use("db")
+        if not dbc.hasTables():  # no tables
+            logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time()))
             return StateDbOnly()
-        if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName())
-                     ) == 0:  # no regular tables
-            # logger.debug("Found TABLE_ONLY state")
-            logger.debug(
-                "[STT] SUPER_TABLE_ONLY found, between {} and {}".format(
-                    ts, time.time()))
+
+        sTable = DbManager.getFixedSuperTable()
+        if sTable.hasRegTables(dbc):  # no regular tables
+            logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time()))
             return StateSuperTableOnly()
         else:  # has actual tables
-            # logger.debug("Found HAS_DATA state")
-            logger.debug(
-                "[STT] HAS_DATA found, between {} and {}".format(
-                    ts, time.time()))
+            logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time()))
             return StateHasData()
 
     def transition(self, tasks):
@@ -1172,7 +1193,8 @@ class StateMechine:
             # case of multiple creation and drops
 
         if self._curState.canDropDb():
-            self._curState.assertIfExistThenSuccess(tasks, TaskDropDb)
+            if gSvcMgr == None: # only if we are running as client-only
+                self._curState.assertIfExistThenSuccess(tasks, TaskDropDb)
             # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in
             # case of drop-create-drop
 
@@ -1300,13 +1322,17 @@ class DbManager():
     def getFixedSuperTableName(cls):
         return "fs_table"
 
+    @classmethod
+    def getFixedSuperTable(cls):
+        return TdSuperTable(cls.getFixedSuperTableName())
+
     def releaseTable(self, i):  # return the table back, so others can use it
         self.tableNumQueue.release(i)
 
     def getNextTick(self):
         with self._lock:  # prevent duplicate tick
-            if Dice.throw(10) == 0:  # 1 in 10 chance
-                return self._lastTick + datetime.timedelta(0, -100)
+            if Dice.throw(20) == 0:  # 1 in 20 chance
+                return self._lastTick + datetime.timedelta(0, -100) # Go back in time 100 seconds
             else:  # regular
                 # add one second to it
                 self._lastTick += datetime.timedelta(0, 1)
@@ -1322,7 +1348,9 @@ class DbManager():
             self.getNextInt())
 
     def getNextFloat(self):
-        return 0.9 + self.getNextInt()
+        ret = 0.9 + self.getNextInt()
+        # print("Float obtained: {}".format(ret))
+        return ret
 
     def getTableNameToDelete(self):
         tblNum = self.tableNumQueue.pop()  # TODO: race condition!
@@ -1340,33 +1368,35 @@ class TaskExecutor():
         def __init__(self, size=10):
             self._size = size
             self._list = []
+            self._lock = threading.Lock()
 
         def add(self, n: int):
-            if not self._list:  # empty
-                self._list.append(n)
-                return
-            # now we should insert
-            nItems = len(self._list)
-            insPos = 0
-            for i in range(nItems):
-                insPos = i
-                if n <= self._list[i]:  # smaller than this item, time to insert
-                    break  # found the insertion point
-                insPos += 1  # insert to the right
-
-            if insPos == 0:  # except for the 1st item, # TODO: elimiate first item as gating item
-                return  # do nothing
-
-            # print("Inserting at postion {}, value: {}".format(insPos, n))
-            self._list.insert(insPos, n)  # insert
-
-            newLen = len(self._list)
-            if newLen <= self._size:
-                return  # do nothing
-            elif newLen == (self._size + 1):
-                del self._list[0]  # remove the first item
-            else:
-                raise RuntimeError("Corrupt Bounded List")
+            with self._lock:
+                if not self._list:  # empty
+                    self._list.append(n)
+                    return
+                # now we should insert
+                nItems = len(self._list)
+                insPos = 0
+                for i in range(nItems):
+                    insPos = i
+                    if n <= self._list[i]:  # smaller than this item, time to insert
+                        break  # found the insertion point
+                    insPos += 1  # insert to the right
+
+                if insPos == 0:  # except for the 1st item, # TODO: elimiate first item as gating item
+                    return  # do nothing
+
+                # print("Inserting at postion {}, value: {}".format(insPos, n))
+                self._list.insert(insPos, n)  # insert
+
+                newLen = len(self._list)
+                if newLen <= self._size:
+                    return  # do nothing
+                elif newLen == (self._size + 1):
+                    del self._list[0]  # remove the first item
+                else:
+                    raise RuntimeError("Corrupt Bounded List")
 
         def __str__(self):
             return repr(self._list)
@@ -1419,7 +1449,6 @@ class Task():
         # logger.debug("Creating new task {}...".format(self._taskNum))
 
         self._execStats = execStats
-        self._lastSql = ""  # last SQL executed/attempted
 
     def isSuccess(self):
         return self._err is None
@@ -1446,6 +1475,39 @@ class Task():
             "To be implemeted by child classes, class name: {}".format(
                 self.__class__.__name__))
 
+    def _isErrAcceptable(self, errno, msg):
+        if errno in [
+                0x05,  # TSDB_CODE_RPC_NOT_READY
+                # 0x200, # invalid SQL， TODO: re-examine with TD-934
+                0x360, 0x362, 
+                0x369, # tag already exists
+                0x36A, 0x36B, 0x36D,
+                0x381, 
+                0x380, # "db not selected"
+                0x383,
+                0x386,  # DB is being dropped?!
+                0x503,
+                0x510,  # vnode not in ready state
+                0x600,
+                1000  # REST catch-all error
+            ]: 
+            return True # These are the ALWAYS-ACCEPTABLE ones
+        elif (errno in [ 0x0B ]) and gConfig.auto_start_service:
+            return True # We may get "network unavilable" when restarting service
+        elif errno == 0x200 : # invalid SQL, we need to div in a bit more
+            if msg.find("invalid column name") != -1:
+                return True 
+            elif msg.find("tags number not matched") != -1: # mismatched tags after modification
+                return True
+            elif msg.find("duplicated column names") != -1: # also alter table tag issues
+                return True
+        elif (gSvcMgr!=None) and gSvcMgr.isRestarting():
+            logger.info("Ignoring error when service is restarting: errno = {}, msg = {}".format(errno, msg))
+            return True
+        
+        return False # Not an acceptable error
+
+
     def execute(self, wt: WorkerThread):
         wt.verifyThreadSelf()
         self._workerThread = wt  # type: ignore
@@ -1456,36 +1518,25 @@ class Task():
             "[-] executing task {}...".format(self.__class__.__name__))
 
         self._err = None
-        self._execStats.beginTaskType(
-            self.__class__.__name__)  # mark beginning
+        self._execStats.beginTaskType(self.__class__.__name__)  # mark beginning
+        errno2 = None
         try:
             self._executeInternal(te, wt)  # TODO: no return value?
         except taos.error.ProgrammingError as err:
-            errno2 = err.errno if (
-                err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
             if (gConfig.continue_on_exception):  # user choose to continue
-                self.logDebug(
-                    "[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                        errno2, err, self._lastSql))
+                self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
+                        errno2, err, wt.getDbConn().getLastSql()))
                 self._err = err
-            elif (errno2 in [
-                0x05,  # TSDB_CODE_RPC_NOT_READY
-                0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D,
-                0x381, 0x380, 0x383,
-                0x386,  # DB is being dropped?!
-                0x503,
-                0x510,  # vnode not in ready state
-                0x600,
-                1000  # REST catch-all error
-            ]):  # allowed errors
-                self.logDebug(
-                    "[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                        errno2, err, self._lastSql))
+            elif self._isErrAcceptable(errno2, err.__str__()):
+                self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
+                        errno2, err, wt.getDbConn().getLastSql()))
                 print("_", end="", flush=True)
                 self._err = err
-            else:
-                errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                    errno2, err, self._lastSql)
+            else: # not an acceptable error
+                errMsg = "[=] Unexpected Taos library exception ({}): errno=0x{:X}, msg: {}, SQL: {}".format(
+                    self.__class__.__name__,
+                    errno2, err, wt.getDbConn().getLastSql())
                 self.logDebug(errMsg)
                 if gConfig.debug:
                     # raise # so that we see full stack
@@ -1509,25 +1560,22 @@ class Task():
         except BaseException:
             self.logDebug(
                 "[=] Unexpected exception, SQL: {}".format(
-                    self._lastSql))
+                    wt.getDbConn().getLastSql()))
             raise
         self._execStats.endTaskType(self.__class__.__name__, self.isSuccess())
 
         self.logDebug("[X] task execution completed, {}, status: {}".format(
             self.__class__.__name__, "Success" if self.isSuccess() else "Failure"))
         # TODO: merge with above.
-        self._execStats.incExecCount(self.__class__.__name__, self.isSuccess())
+        self._execStats.incExecCount(self.__class__.__name__, self.isSuccess(), errno2)
 
     def execSql(self, sql):
-        self._lastSql = sql
         return self._dbManager.execute(sql)
 
     def execWtSql(self, wt: WorkerThread, sql):  # execute an SQL on the worker thread
-        self._lastSql = sql
         return wt.execSql(sql)
 
     def queryWtSql(self, wt: WorkerThread, sql):  # execute an SQL on the worker thread
-        self._lastSql = sql
         return wt.querySql(sql)
 
     def getQueryResult(self, wt: WorkerThread):  # execute an SQL on the worker thread
@@ -1542,6 +1590,7 @@ class ExecutionStats:
         self._lock = threading.Lock()
         self._firstTaskStartTime = None
         self._execStartTime = None
+        self._errors = {}
         self._elapsedTime = 0.0  # total elapsed time
         self._accRunTime = 0.0  # accumulated run time
 
@@ -1561,13 +1610,18 @@ class ExecutionStats:
     def endExec(self):
         self._elapsedTime = time.time() - self._execStartTime
 
-    def incExecCount(self, klassName, isSuccess):  # TODO: add a lock here
+    def incExecCount(self, klassName, isSuccess, eno=None):  # TODO: add a lock here
         if klassName not in self._execTimes:
             self._execTimes[klassName] = [0, 0]
         t = self._execTimes[klassName]  # tuple for the data
         t[0] += 1  # index 0 has the "total" execution times
         if isSuccess:
             t[1] += 1  # index 1 has the "success" execution times
+        if eno != None:             
+            if klassName not in self._errors:
+                self._errors[klassName] = {}
+            errors = self._errors[klassName]
+            errors[eno] = errors[eno]+1 if eno in errors else 1
 
     def beginTaskType(self, klassName):
         with self._lock:
@@ -1597,7 +1651,14 @@ class ExecutionStats:
         execTimesAny = 0
         for k, n in self._execTimes.items():
             execTimesAny += n[0]
-            logger.info("|    {0:<24}: {1}/{2}".format(k, n[1], n[0]))
+            errStr = None
+            if k in self._errors:
+                errors = self._errors[k]
+                # print("errors = {}".format(errors))
+                errStrs = ["0x{:X}:{}".format(eno, n) for (eno, n) in errors.items()]
+                # print("error strings = {}".format(errStrs))
+                errStr = ", ".join(errStrs) 
+            logger.info("|    {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr))
 
         logger.info(
             "| Total Tasks Executed (success or not): {} ".format(execTimesAny))
@@ -1649,7 +1710,7 @@ class StateTransitionTask(Task):
 
     @classmethod
     def getRegTableName(cls, i):
-        return "db.reg_table_{}".format(i)
+        return "reg_table_{}".format(i)
 
     def execute(self, wt: WorkerThread):
         super().execute(wt)
@@ -1696,15 +1757,94 @@ class TaskCreateSuperTable(StateTransitionTask):
             logger.debug("Skipping task, no DB yet")
             return
 
-        tblName = self._dbManager.getFixedSuperTableName()
+        sTable = self._dbManager.getFixedSuperTable()
         # wt.execSql("use db")    # should always be in place
-        self.execWtSql(
-            wt,
-            "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
+        sTable.create(wt.getDbConn(), {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'})
+        # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
         # No need to create the regular tables, INSERT will do that
         # automatically
 
 
+class TdSuperTable:
+    def __init__(self, stName):
+        self._stName = stName
+
+    def create(self, dbc, cols: dict, tags: dict):
+        sql = "CREATE TABLE db.{} ({}) TAGS ({})".format(
+            self._stName,
+            ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]),
+            ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
+            )
+        dbc.execute(sql)        
+
+    def getRegTables(self, dbc: DbConn):
+        try:
+            dbc.query("select TBNAME from db.{}".format(self._stName))  # TODO: analyze result set later            
+        except taos.error.ProgrammingError as err:                    
+            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+            logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
+            raise
+
+        qr = dbc.getQueryResult()
+        return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation
+
+    def hasRegTables(self, dbc: DbConn):
+        return dbc.query("SELECT * FROM db.{}".format(self._stName)) > 0
+
+    def ensureTable(self, dbc: DbConn, regTableName: str):
+        sql = "select tbname from {} where tbname in ('{}')".format(self._stName, regTableName)
+        if dbc.query(sql) >= 1 : # reg table exists already
+            return
+        sql = "CREATE TABLE {} USING {} tags ({})".format(
+            regTableName, self._stName, self._getTagStrForSql(dbc)
+        )
+        dbc.execute(sql)
+
+    def _getTagStrForSql(self, dbc) :
+        tags = self._getTags(dbc)
+        tagStrs = []
+        for tagName in tags: 
+            tagType = tags[tagName]
+            if tagType == 'BINARY':
+                tagStrs.append("'Beijing-Shanghai-LosAngeles'")
+            elif tagType == 'FLOAT':
+                tagStrs.append('9.9')
+            elif tagType == 'INT':
+                tagStrs.append('88')
+            else:
+                raise RuntimeError("Unexpected tag type: {}".format(tagType))
+        return ", ".join(tagStrs)
+
+    def _getTags(self, dbc) -> dict:
+        dbc.query("DESCRIBE {}".format(self._stName))
+        stCols = dbc.getQueryResult()
+        # print(stCols)
+        ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type
+        # print("Tags retrieved: {}".format(ret))
+        return ret
+
+    def addTag(self, dbc, tagName, tagType):
+        if tagName in self._getTags(dbc): # already 
+            return
+        # sTable.addTag("extraTag", "int")
+        sql = "alter table db.{} add tag {} {}".format(self._stName, tagName, tagType)
+        dbc.execute(sql)
+
+    def dropTag(self, dbc, tagName):
+        if not tagName in self._getTags(dbc): # don't have this tag
+            return
+        sql = "alter table db.{} drop tag {}".format(self._stName, tagName)
+        dbc.execute(sql)
+
+    def changeTag(self, dbc, oldTag, newTag):
+        tags = self._getTags(dbc)
+        if not oldTag in tags: # don't have this tag
+            return
+        if newTag in tags: # already have this tag
+            return
+        sql = "alter table db.{} change tag {} {}".format(self._stName, oldTag, newTag)
+        dbc.execute(sql)
+
 class TaskReadData(StateTransitionTask):
     @classmethod
     def getEndState(cls):
@@ -1715,23 +1855,24 @@ class TaskReadData(StateTransitionTask):
         return state.canReadData()
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        sTbName = self._dbManager.getFixedSuperTableName()
-        self.queryWtSql(wt, "select TBNAME from db.{}".format(
-            sTbName))  # TODO: analyze result set later
+        sTable = self._dbManager.getFixedSuperTable()
 
         if random.randrange(
                 5) == 0:  # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations
             wt.getDbConn().close()
             wt.getDbConn().open()
-        else:
-            # wt.getDbConn().getQueryResult()
-            rTables = self.getQueryResult(wt)
-            # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0])))
-            for rTbName in rTables:  # regular tables
-                self.execWtSql(wt, "select * from db.{}".format(rTbName[0]))
-
-        # tdSql.query(" cars where tbname in ('carzero', 'carone')")
-
+        
+        for rTbName in sTable.getRegTables(wt.getDbConn()):  # regular tables
+            aggExpr = Dice.choice(['*', 'count(*)', 'avg(speed)', 
+                # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable
+                'sum(speed)', 'stddev(speed)', 
+                'min(speed)', 'max(speed)', 'first(speed)', 'last(speed)']) # TODO: add more from 'top'
+            try:
+                self.execWtSql(wt, "select {} from db.{}".format(aggExpr, rTbName))
+            except taos.error.ProgrammingError as err:                    
+                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+                logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, wt.getDbConn().getLastSql()))
+                raise
 
 class TaskDropSuperTable(StateTransitionTask):
     @classmethod
@@ -1789,20 +1930,55 @@ class TaskAlterTags(StateTransitionTask):
         return state.canDropFixedSuperTable()  # if we can drop it, we can alter tags
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        tblName = self._dbManager.getFixedSuperTableName()
+        # tblName = self._dbManager.getFixedSuperTableName()
+        dbc = wt.getDbConn()
+        sTable = self._dbManager.getFixedSuperTable()
         dice = Dice.throw(4)
         if dice == 0:
-            sql = "alter table db.{} add tag extraTag int".format(tblName)
+            sTable.addTag(dbc, "extraTag", "int")
+            # sql = "alter table db.{} add tag extraTag int".format(tblName)
         elif dice == 1:
-            sql = "alter table db.{} drop tag extraTag".format(tblName)
+            sTable.dropTag(dbc, "extraTag")
+            # sql = "alter table db.{} drop tag extraTag".format(tblName)
         elif dice == 2:
-            sql = "alter table db.{} drop tag newTag".format(tblName)
+            sTable.dropTag(dbc, "newTag")
+            # sql = "alter table db.{} drop tag newTag".format(tblName)
         else:  # dice == 3
-            sql = "alter table db.{} change tag extraTag newTag".format(
-                tblName)
+            sTable.changeTag(dbc, "extraTag", "newTag")
+            # sql = "alter table db.{} change tag extraTag newTag".format(tblName)
 
-        self.execWtSql(wt, sql)
+class TaskRestartService(StateTransitionTask):
+    _isRunning = False
+    _classLock = threading.Lock()
 
+    @classmethod
+    def getEndState(cls):
+        return None  # meaning doesn't affect state
+
+    @classmethod
+    def canBeginFrom(cls, state: AnyState):
+        if gConfig.auto_start_service:
+            return state.canDropFixedSuperTable()  # Basicallly when we have the super table
+        return False # don't run this otherwise
+
+    CHANCE_TO_RESTART_SERVICE = 100
+    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
+        if not gConfig.auto_start_service: # only execute when we are in -a mode
+            print("_a", end="", flush=True)
+            return
+
+        with self._classLock:
+            if self._isRunning:
+                print("Skipping restart task, another running already")
+                return
+            self._isRunning = True
+
+        if Dice.throw(self.CHANCE_TO_RESTART_SERVICE) == 0: # 1 in N chance
+            dbc = wt.getDbConn()
+            dbc.execute("show databases") # simple delay, align timing with other workers
+            gSvcMgr.restart()
+
+        self._isRunning = False
 
 class TaskAddData(StateTransitionTask):
     # Track which table is being actively worked on
@@ -1833,39 +2009,31 @@ class TaskAddData(StateTransitionTask):
         return state.canAddData()
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        ds = self._dbManager
-        # wt.execSql("use db") # TODO: seems to be an INSERT bug to require
-        # this
-        tblSeq = list(
-            range(
+        ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
+        tblSeq = list(range(
                 self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
         random.shuffle(tblSeq)
         for i in tblSeq:
             if (i in self.activeTable):  # wow already active
-                # logger.info("Concurrent data insertion into table: {}".format(i))
-                # print("ct({})".format(i), end="", flush=True) # Concurrent
-                # insertion into table
-                print("x", end="", flush=True)
+                print("x", end="", flush=True) # concurrent insertion
             else:
                 self.activeTable.add(i)  # marking it active
-            # No need to shuffle data sequence, unless later we decide to do
-            # non-increment insertion
-            regTableName = self.getRegTableName(
-                i)  # "db.reg_table_{}".format(i)
-            for j in range(
-                    self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS):  # number of records per table
+            
+            sTable = ds.getFixedSuperTable()
+            regTableName = self.getRegTableName(i)  # "db.reg_table_{}".format(i)
+            sTable.ensureTable(wt.getDbConn(), regTableName)  # Ensure the table exists           
+           
+            for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS):  # number of records per table
                 nextInt = ds.getNextInt()
                 if gConfig.record_ops:
                     self.prepToRecordOps()
-                    self.fAddLogReady.write(
-                        "Ready to write {} to {}\n".format(
-                            nextInt, regTableName))
+                    self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
                     self.fAddLogReady.flush()
                     os.fsync(self.fAddLogReady)
-                sql = "insert into {} using {} tags ('{}', {}) values ('{}', {});".format(
+                sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
                     regTableName,
-                    ds.getFixedSuperTableName(),
-                    ds.getNextBinary(), ds.getNextFloat(),
+                    # ds.getFixedSuperTableName(),
+                    # ds.getNextBinary(), ds.getNextFloat(),
                     ds.getNextTick(), nextInt)
                 self.execWtSql(wt, sql)
                 # Successfully wrote the data into the DB, let's record it
@@ -1912,6 +2080,10 @@ class Dice():
             raise RuntimeError("Cannot throw dice before seeding it")
         return random.randrange(start, stop)
 
+    @classmethod
+    def choice(cls, cList):
+        return random.choice(cList)
+
 
 class LoggingFilter(logging.Filter):
     def filter(self, record: logging.LogRecord):
@@ -1934,14 +2106,16 @@ class MyLoggingAdapter(logging.LoggerAdapter):
 class SvcManager:
     def __init__(self):
         print("Starting TDengine Service Manager")
-        signal.signal(signal.SIGTERM, self.sigIntHandler)
-        signal.signal(signal.SIGINT, self.sigIntHandler)
-        signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
+        # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec
+        # signal.signal(signal.SIGINT, self.sigIntHandler)
+        # signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
 
         self.inSigHandler = False
         # self._status = MainExec.STATUS_RUNNING # set inside
         # _startTaosService()
         self.svcMgrThread = None
+        self._lock = threading.Lock()
+        self._isRestarting = False
 
     def _doMenu(self):
         choice = ""
@@ -1976,23 +2150,22 @@ class SvcManager:
             self.sigHandlerResume()
         elif choice == "2":
             self.stopTaosService()
-        elif choice == "3":
-            self.stopTaosService()
-            self.startTaosService()
+        elif choice == "3": # Restart
+            self.restart()
         else:
             raise RuntimeError("Invalid menu choice: {}".format(choice))
 
         self.inSigHandler = False
 
     def sigIntHandler(self, signalNumber, frame):
-        print("Sig INT Handler starting...")
+        print("SvcManager: INT Signal Handler starting...")
         if self.inSigHandler:
             print("Ignoring repeated SIG_INT...")
             return
         self.inSigHandler = True
 
         self.stopTaosService()
-        print("INT signal handler returning...")
+        print("SvcManager: INT Signal Handler returning...")
         self.inSigHandler = False
 
     def sigHandlerResume(self):
@@ -2005,44 +2178,78 @@ class SvcManager:
                 self.svcMgrThread = None  # no more
 
     def _procIpcAll(self):
-        while self.svcMgrThread:  # for as long as the svc mgr thread is still here
-            self.svcMgrThread.procIpcBatch()  # regular processing,
+        while self.isRunning() or self.isRestarting() :  # for as long as the svc mgr thread is still here
+            if self.isRunning():
+                self.svcMgrThread.procIpcBatch()  # regular processing,
+                self._checkServiceManagerThread()
+            elif self.isRetarting():
+                print("Service restarting...")
             time.sleep(0.5)  # pause, before next round
-            self._checkServiceManagerThread()
         print(
             "Service Manager Thread (with subprocess) has ended, main thread now exiting...")
 
     def startTaosService(self):
-        if self.svcMgrThread:
-            raise RuntimeError(
-                "Cannot start TAOS service when one may already be running")
-        self.svcMgrThread = ServiceManagerThread()  # create the object
-        self.svcMgrThread.start()
-        print("TAOS service started, printing out output...")
-        self.svcMgrThread.procIpcBatch(
-            trimToTarget=10,
-            forceOutput=True)  # for printing 10 lines
-        print("TAOS service started")
+        with self._lock:
+            if self.svcMgrThread:
+                raise RuntimeError("Cannot start TAOS service when one may already be running")
+
+            # Find if there's already a taosd service, and then kill it
+            for proc in psutil.process_iter():
+                if proc.name() == 'taosd':
+                    print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupe")
+                    time.sleep(2.0)
+                    proc.kill()
+                # print("Process: {}".format(proc.name()))
+            
+            self.svcMgrThread = ServiceManagerThread()  # create the object
+            self.svcMgrThread.start()
+            print("Attempting to start TAOS service started, printing out output...")
+            self.svcMgrThread.procIpcBatch(
+                trimToTarget=10,
+                forceOutput=True)  # for printing 10 lines
+            print("TAOS service started")
 
     def stopTaosService(self, outputLines=20):
-        print("Terminating Service Manager Thread (SMT) execution...")
-        if not self.svcMgrThread:
-            raise RuntimeError("Unexpected empty svc mgr thread")
-        self.svcMgrThread.stop()
-        if self.svcMgrThread.isStopped():
-            self.svcMgrThread.procIpcBatch(outputLines)  # one last time
-            self.svcMgrThread = None
-            print("----- End of TDengine Service Output -----\n")
-            print("SMT execution terminated")
-        else:
-            print("WARNING: SMT did not terminate as expected")
+        with self._lock:
+            if not self.isRunning():
+                logger.warning("Cannot stop TAOS service, not running")
+                return
+
+            print("Terminating Service Manager Thread (SMT) execution...")
+            self.svcMgrThread.stop()
+            if self.svcMgrThread.isStopped():
+                self.svcMgrThread.procIpcBatch(outputLines)  # one last time
+                self.svcMgrThread = None
+                print("----- End of TDengine Service Output -----\n")
+                print("SMT execution terminated")
+            else:
+                print("WARNING: SMT did not terminate as expected")
 
     def run(self):
         self.startTaosService()
-        self._procIpcAll()  # pump/process all the messages
-        if self.svcMgrThread:  # if sig handler hasn't destroyed it by now
+        self._procIpcAll()  # pump/process all the messages, may encounter SIG + restart
+        if self.isRunning():  # if sig handler hasn't destroyed it by now
             self.stopTaosService()  # should have started already
 
+    def restart(self):
+        if self._isRestarting:
+            logger.warning("Cannot restart service when it's already restarting")
+            return
+
+        self._isRestarting = True
+        if self.isRunning():
+            self.stopTaosService()
+        else:
+            logger.warning("Service not running when restart requested")
+
+        self.startTaosService()
+        self._isRestarting = False
+
+    def isRunning(self):
+        return self.svcMgrThread != None
+
+    def isRestarting(self):
+        return self._isRestarting
 
 class ServiceManagerThread:
     MAX_QUEUE_SIZE = 10000
@@ -2094,6 +2301,7 @@ class ServiceManagerThread:
                 logger.info("[] TDengine service READY to process requests")
                 return  # now we've started
         # TODO: handle this better?
+        self.procIpcBatch(20, True) # display output before cronking out, trim to last 20 msgs, force output
         raise RuntimeError("TDengine service did not start successfully")
 
     def stop(self):
@@ -2196,7 +2404,10 @@ class ServiceManagerThread:
 
             if self._status == MainExec.STATUS_STARTING:  # we are starting, let's see if we have started
                 if line.find(self.TD_READY_MSG) != -1:  # found
-                    self._status = MainExec.STATUS_RUNNING
+                    logger.info("Waiting for the service to become FULLY READY")
+                    time.sleep(1.0) # wait for the server to truly start. TODO: remove this
+                    logger.info("Service is now FULLY READY")   
+                    self._status = MainExec.STATUS_RUNNING                 
 
             # Trim the queue if necessary: TODO: try this 1 out of 10 times
             self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10)  # trim to 90% size
@@ -2242,6 +2453,21 @@ class TdeSubProcess:
         taosdPath = self.getBuildPath() + "/build/bin/taosd"
         cfgPath = self.getBuildPath() + "/test/cfg"
 
+        # Delete the log files
+        logPath = self.getBuildPath() + "/test/log"
+        # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397
+        # filelist = [ f for f in os.listdir(logPath) ] # if f.endswith(".bak") ]
+        # for f in filelist:
+        #     filePath = os.path.join(logPath, f)
+        #     print("Removing log file: {}".format(filePath))
+        #     os.remove(filePath)        
+        if os.path.exists(logPath):
+            logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S')
+            logger.info("Saving old log files to: {}".format(logPathSaved))
+            os.rename(logPath, logPathSaved)
+        # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms
+            
+
         svcCmd = [taosdPath, '-c', cfgPath]
         # svcCmd = ['vmstat', '1']
         if self.subProcess:  # already there
@@ -2275,16 +2501,46 @@ class TdeSubProcess:
                 print("TDengine service process terminated successfully from SIG_INT")
                 self.subProcess = None
 
+class ThreadStacks: # stack info for all threads
+    def __init__(self):
+        self._allStacks = {}
+        allFrames = sys._current_frames()
+        for th in threading.enumerate():                        
+            stack = traceback.extract_stack(allFrames[th.ident])     
+            self._allStacks[th.native_id] = stack
+
+    def print(self, filteredEndName = None, filterInternal = False):
+        for thNid, stack in self._allStacks.items(): # for each thread            
+            lastFrame = stack[-1]
+            if filteredEndName: # we need to filter out stacks that match this name                
+                if lastFrame.name == filteredEndName : # end did not match
+                    continue
+            if filterInternal:
+                if lastFrame.name in ['wait', 'invoke_excepthook', 
+                    '_wait', # The Barrier exception
+                    'svcOutputReader', # the svcMgr thread
+                    '__init__']: # the thread that extracted the stack
+                    continue # ignore
+            # Now print
+            print("\n<----- Thread Info for ID: {}".format(thNid))
+            for frame in stack:
+                # print(frame)
+                print("File {filename}, line {lineno}, in {name}".format(
+                    filename=frame.filename, lineno=frame.lineno, name=frame.name))
+                print("    {}".format(frame.line))
+            print("-----> End of Thread Info\n")
 
 class ClientManager:
     def __init__(self):
         print("Starting service manager")
-        signal.signal(signal.SIGTERM, self.sigIntHandler)
-        signal.signal(signal.SIGINT, self.sigIntHandler)
+        # signal.signal(signal.SIGTERM, self.sigIntHandler)
+        # signal.signal(signal.SIGINT, self.sigIntHandler)
 
         self._status = MainExec.STATUS_RUNNING
         self.tc = None
 
+        self.inSigHandler = False
+
     def sigIntHandler(self, signalNumber, frame):
         if self._status != MainExec.STATUS_RUNNING:
             print("Repeated SIGINT received, forced exit...")
@@ -2292,9 +2548,50 @@ class ClientManager:
             sys.exit(-1)
         self._status = MainExec.STATUS_STOPPING  # immediately set our status
 
-        print("Terminating program...")
+        print("ClientManager: Terminating program...")
         self.tc.requestToStop()
 
+    def _doMenu(self):
+        choice = ""
+        while True:
+            print("\nInterrupting Client Program, Choose an Action: ")
+            print("1: Resume")
+            print("2: Terminate")
+            print("3: Show Threads")
+            # Remember to update the if range below
+            # print("Enter Choice: ", end="", flush=True)
+            while choice == "":
+                choice = input("Enter Choice: ")
+                if choice != "":
+                    break  # done with reading repeated input
+            if choice in ["1", "2", "3"]:
+                break  # we are done with whole method
+            print("Invalid choice, please try again.")
+            choice = ""  # reset
+        return choice
+
+    def sigUsrHandler(self, signalNumber, frame):
+        print("Interrupting main thread execution upon SIGUSR1")
+        if self.inSigHandler:  # already
+            print("Ignoring repeated SIG_USR1...")
+            return  # do nothing if it's already not running
+        self.inSigHandler = True
+
+        choice = self._doMenu()
+        if choice == "1":
+            print("Resuming execution...")
+            time.sleep(1.0)
+        elif choice == "2":
+            print("Not implemented yet")
+            time.sleep(1.0)
+        elif choice == "3":
+            ts = ThreadStacks()
+            ts.print()
+        else:
+            raise RuntimeError("Invalid menu choice: {}".format(choice))
+
+        self.inSigHandler = False
+
     def _printLastNumbers(self):  # to verify data durability
         dbManager = DbManager(resetDb=False)
         dbc = dbManager.getDbConn()
@@ -2327,21 +2624,17 @@ class ClientManager:
     def prepare(self):
         self._printLastNumbers()
 
-    def run(self):
-        if gConfig.auto_start_service:
-            svcMgr = SvcManager()
-            svcMgr.startTaosService()
-
+    def run(self, svcMgr):    
         self._printLastNumbers()
 
         dbManager = DbManager()  # Regular function
         thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
         self.tc = ThreadCoordinator(thPool, dbManager)
-
+        
         self.tc.run()
         # print("exec stats: {}".format(self.tc.getExecStats()))
         # print("TC failed = {}".format(self.tc.isFailed()))
-        if gConfig.auto_start_service:
+        if svcMgr: # gConfig.auto_start_service:
             svcMgr.stopTaosService()
         # Print exec status, etc., AFTER showing messages from the server
         self.conclude()
@@ -2353,25 +2646,58 @@ class ClientManager:
         self.tc.printStats()
         self.tc.getDbManager().cleanUp()
 
-
 class MainExec:
     STATUS_STARTING = 1
     STATUS_RUNNING = 2
     STATUS_STOPPING = 3
     STATUS_STOPPED = 4
 
-    @classmethod
-    def runClient(cls):
-        clientManager = ClientManager()
-        return clientManager.run()
+    def __init__(self):        
+        self._clientMgr = None
+        self._svcMgr = None
 
-    @classmethod
-    def runService(cls):
-        svcManager = SvcManager()
-        svcManager.run()
+        signal.signal(signal.SIGTERM, self.sigIntHandler)
+        signal.signal(signal.SIGINT,  self.sigIntHandler)
+        signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
 
-    @classmethod
-    def runTemp(cls):  # for debugging purposes
+    def sigUsrHandler(self, signalNumber, frame):
+        if self._clientMgr:
+            self._clientMgr.sigUsrHandler(signalNumber, frame)
+        elif self._svcMgr: # Only if no client mgr, we are running alone
+            self._svcMgr.sigUsrHandler(signalNumber, frame)
+        
+    def sigIntHandler(self, signalNumber, frame):
+        if self._svcMgr:
+            self._svcMgr.sigIntHandler(signalNumber, frame)
+        if self._clientMgr:
+            self._clientMgr.sigIntHandler(signalNumber, frame)
+
+    def runClient(self):
+        global gSvcMgr
+        if gConfig.auto_start_service:
+            self._svcMgr = SvcManager()
+            gSvcMgr = self._svcMgr # hack alert
+            self._svcMgr.startTaosService() # we start, don't run
+        
+        self._clientMgr = ClientManager()
+        ret = None
+        try: 
+            ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside
+        except requests.exceptions.ConnectionError as err:
+            logger.warning("Failed to open REST connection to DB")
+            # don't raise
+        return ret
+
+    def runService(self):
+        global gSvcMgr
+        self._svcMgr = SvcManager()
+        gSvcMgr = self._svcMgr # save it in a global variable TODO: hack alert
+
+        self._svcMgr.run() # run to some end state
+        self._svcMgr = None 
+        gSvcMgr = None        
+
+    def runTemp(self):  # for debugging purposes
         # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix
         # dbc = dbState.getDbConn()
         # sTbName = dbState.getFixedSuperTableName()
@@ -2527,10 +2853,11 @@ def main():
     Dice.seed(0)  # initial seeding of dice
 
     # Run server or client
+    mExec = MainExec()
     if gConfig.run_tdengine:  # run server
-        MainExec.runService()
+        mExec.runService()
     else:
-        return MainExec.runClient()
+        return mExec.runClient()
 
 
 if __name__ == "__main__":
diff --git a/tests/pytest/stable/query_after_reset.py b/tests/pytest/stable/query_after_reset.py
index 61f6558b83b95333bf37f1c0df13c3cf8c43d8cb..b66fc2eff1e9269411168ecd38cf459eae16e6d5 100644
--- a/tests/pytest/stable/query_after_reset.py
+++ b/tests/pytest/stable/query_after_reset.py
@@ -72,8 +72,12 @@ class Test:
 
     def query_stable(self):
         tdLog.info("query super table")
-        tdSql.query("select * from st")
-        tdSql.checkRows(1)
+        try:
+            tdSql.query("select * from st")
+        except Exception as e:
+            tdLog.info("Exception catched: %s" % repr(e))
+            if ('Table does not exist' not in repr(e)):
+                raise Exception(repr(e))
 
     def create_stable(self):
         tdLog.info("create a super table and sub-table and insert data")
@@ -162,10 +166,15 @@ class TDTestCase:
             12: test.stop_database,
         }
 
+        tdLog.info("create stable")
         switch.get(4, lambda: "ERROR")()
+        tdLog.info("stop database")
         switch.get(12, lambda: "ERROR")()
+        tdLog.info("delete datafiles")
         switch.get(10, lambda: "ERROR")()
+        tdLog.info("restart database")
         switch.get(5, lambda: "ERROR")()
+        tdLog.info("query stable")
         switch.get(11, lambda: "ERROR")()
 
     def stop(self):
diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim
index 46fc3a4ff7e78170c92c6b5944aced12f8bd5577..12a49c59b585e8d0da8cf53c85a4558cd43a9b65 100644
--- a/tests/script/general/db/alter_option.sim
+++ b/tests/script/general/db/alter_option.sim
@@ -24,16 +24,16 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 10 then 
+if $data06 != 10 then 
   return -1
 endi
-if $data06 != 20,20,20 then 
+if $data07 != 20,20,20 then 
   return -1
 endi
-if $data07 != 2 then 
+if $data08 != 2 then 
   return -1
 endi
-if $data08 != 4 then 
+if $data09 != 4 then 
   return -1
 endi
 
diff --git a/tests/script/general/db/basic.sim b/tests/script/general/db/basic.sim
index a7c4358affba250d861e9c738612ece8a896869b..1c4939256be82541f40e34d5c89ea4c1f67de288 100644
--- a/tests/script/general/db/basic.sim
+++ b/tests/script/general/db/basic.sim
@@ -32,10 +32,10 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 20 then 
+if $data06 != 20 then 
   return -1
 endi
-if $data07 != 16 then 
+if $data08 != 16 then 
   return -1
 endi
  
@@ -72,7 +72,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 15 then 
+if $data06 != 15 then 
   return -1
 endi
 
diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim
index bf2a3b3b58a9469c18ec17c3c27b4fa45ff9813f..d2c31a0e598249090c45695eafdc6067c27a76bd 100644
--- a/tests/script/general/http/grafana.sim
+++ b/tests/script/general/http/grafana.sim
@@ -66,7 +66,7 @@ endi
 
 system_content curl 127.0.0.1:6020/grafana/login/xx/xx/
 print 3-> $system_content
-if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then
+if $system_content != @{"status":"error","code":849,"desc":"Invalid user"}@ then
   return -1
 endi
 
@@ -78,7 +78,7 @@ endi
 
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:6020/grafana/login/1/root/1/
 print 5-> $system_content
-if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then
+if $system_content != @{"status":"error","code":849,"desc":"Invalid user"}@ then
   return -1
 endi
 
diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim
index 7f39a9c74824b2696c59d1b8c179e776bb87214d..a647399758f97572618c812004009bd1ddc0af8c 100644
--- a/tests/script/general/http/restful.sim
+++ b/tests/script/general/http/restful.sim
@@ -77,7 +77,7 @@ endi
 
 system_content curl 127.0.0.1:6020/rest/login/u2/aabcd_1234
 print curl 127.0.0.1:6020/rest/login/u2/abcd_1234 -----> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim
index 8545b8d2cd57c09996554a1b7d80ed4510c993f6..43511b31d8ec92d592b4ac6bb91559d70bf0c524 100644
--- a/tests/script/general/http/restful_full.sim
+++ b/tests/script/general/http/restful_full.sim
@@ -39,19 +39,19 @@ endi
 
 system_content curl 127.0.0.1:6020/rest/login/root/123
 print 5-> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
 system_content curl 127.0.0.1:6020/rest/login/root/123/1/1/3
 print 6-> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:6020/rest/login/root/1
 print 7-> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
@@ -81,7 +81,7 @@ print ===============  step2 - no db
 #11
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:6020/rest/sql
 print 11-> $system_content
-if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
+if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
   return -1
 endi
 
@@ -93,7 +93,7 @@ endi
 
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'create database d1' 127.0.0.1:6020/rest/sql
 print 13-> $system_content
-if $system_content != @{"status":"error","code":897,"desc":"mnode database aleady exist"}@ then
+if $system_content != @{"status":"error","code":897,"desc":"Database already exists"}@ then
   return -1
 endi
 
@@ -126,7 +126,7 @@ endi
 #18
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d ' show tables;' 127.0.0.1:6020/rest/sql
 print 18-> $system_content
-if $system_content != @{"status":"error","code":896,"desc":"mnode db not selected"}@ then
+if $system_content != @{"status":"error","code":896,"desc":"Database not specified or available"}@ then
   return -1
 endi
 
@@ -147,7 +147,7 @@ print ===============  step3 - db
 
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d ' select * from d1.t1;' 127.0.0.1:6020/rest/sql
 print 21-> $system_content
-if $system_content != @{"status":"error","code":866,"desc":"mnode invalid table name"}@ then
+if $system_content != @{"status":"error","code":866,"desc":"Table does not exist"}@ then
   return -1
 endi
 
diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim
index be8587f2aed9fda43507455eef84bf55bd8d4b43..ed6e427fe70fe5cfe65852352ac680f645a3a21f 100644
--- a/tests/script/general/parser/create_db.sim
+++ b/tests/script/general/parser/create_db.sim
@@ -122,18 +122,18 @@ endi
 if $data04 != $replica then 
  return -1
 endi
-if $data05 != $days then 
+if $data06 != $days then 
  return -1
 endi
-if $data06 != 365,365,365 then 
+if $data07 != 365,365,365 then 
  return -1
 endi
-print data07 = $data07
-if $data07 != $cache then
- print expect $cache, actual:$data07
+print data08 = $data07
+if $data08 != $cache then
+ print expect $cache, actual:$data08
  return -1
 endi
-if $data08 != 4 then
+if $data09 != 4 then
  return -1
 endi
 
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index c6981d29022dca56198d6655379472c905984285..1e73893793a9f011f6753e5745226e664ca253f2 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -93,6 +93,8 @@ run general/parser/groupby.sim
 sleep 2000
 run general/parser/tags_filter.sim
 sleep 2000
+run general/parser/topbot.sim
+sleep 2000
 run general/parser/union.sim
 sleep 2000
 run general/parser/sliding.sim
diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim
new file mode 100644
index 0000000000000000000000000000000000000000..a0c46dbc65bac39b33cd54f0c61ac3ba96bf5736
--- /dev/null
+++ b/tests/script/general/parser/topbot.sim
@@ -0,0 +1,74 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+$dbPrefix = tb_db
+$tbPrefix = tb_tb
+$stbPrefix = tb_stb
+$tbNum = 10
+$rowNum = 1000
+$totalNum = $tbNum * $rowNum
+$loops = 200000
+$log = 10000
+$ts0 = 1537146000000
+$delta = 600000
+print ========== topbot.sim
+$i = 0
+$db = $dbPrefix . $i
+$stb = $stbPrefix . $i
+
+sql drop database $db -x step1
+step1:
+sql create database $db cache 16 maxtables 200
+print ====== create tables
+sql use $db
+sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int)
+
+$i = 0
+$ts = $ts0
+$halfNum = $tbNum / 2
+while $i < $halfNum
+  $tbId = $i + $halfNum
+  $tb = $tbPrefix . $i
+  $tb1 = $tbPrefix . $tbId
+  sql create table $tb using $stb tags( $i )
+  sql create table $tb1 using $stb tags( $tbId )
+
+  $x = 0
+  while $x < $rowNum
+    $xs = $x * $delta
+    $ts = $ts0 + $xs
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+    $binary = 'binary . $c
+    $binary = $binary . '
+    $nchar = 'nchar . $c
+    $nchar = $nchar . '
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
+    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+endw
+print ====== tables created
+
+sql use $db
+##### select from table
+print ====== select top/bot from table and check num of rows returned
+sql select top(c1, 100) from tb_stb0
+if $row != 100 then
+  return -1
+endi
+
+sql select last(c2) from tb_tb9
+if $row != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh
index ac65237319cfb97a9e9100a980f6b840cf7bd20a..90ca14277b42447daa9667628a3e748d0e3cb1ac 100755
--- a/tests/script/sh/deploy.sh
+++ b/tests/script/sh/deploy.sh
@@ -104,48 +104,48 @@ elif [ $NODE -eq 8 ]; then
   NODE=7800    
 fi
 
-echo " "                                    >> $TAOS_CFG   
-echo "first               ${HOSTNAME}:7100" >> $TAOS_CFG
-echo "second              ${HOSTNAME}:7200" >> $TAOS_CFG
-echo "serverPort          ${NODE}"          >> $TAOS_CFG
-echo "dataDir             $DATA_DIR"      >> $TAOS_CFG
-echo "logDir              $LOG_DIR"       >> $TAOS_CFG
-echo "debugFlag           0"              >> $TAOS_CFG
-echo "mDebugFlag          135"            >> $TAOS_CFG
-echo "sdbDebugFlag        135"            >> $TAOS_CFG
-echo "dDebugFlag          135"            >> $TAOS_CFG
-echo "vDebugFlag          135"            >> $TAOS_CFG
-echo "tsdbDebugFlag       135"            >> $TAOS_CFG
-echo "cDebugFlag          135"            >> $TAOS_CFG
-echo "jnidebugFlag        135"            >> $TAOS_CFG
-echo "odbcdebugFlag       135"            >> $TAOS_CFG
-echo "httpDebugFlag       143"            >> $TAOS_CFG
-echo "monitorDebugFlag    135"            >> $TAOS_CFG
-echo "mqttDebugFlag       135"            >> $TAOS_CFG
-echo "qdebugFlag          135"            >> $TAOS_CFG
-echo "rpcDebugFlag        135"            >> $TAOS_CFG
-echo "tmrDebugFlag        131"            >> $TAOS_CFG
-echo "udebugFlag          135"            >> $TAOS_CFG
-echo "sdebugFlag          135"            >> $TAOS_CFG
-echo "wdebugFlag          135"            >> $TAOS_CFG
-echo "monitor             0"              >> $TAOS_CFG
-echo "monitorInterval     1"              >> $TAOS_CFG
-echo "http                0"              >> $TAOS_CFG
-echo "numOfThreadsPerCore 2.0"            >> $TAOS_CFG
-echo "defaultPass         taosdata"       >> $TAOS_CFG
-echo "numOfLogLines       20000000"       >> $TAOS_CFG
-echo "mnodeEqualVnodeNum  0"              >> $TAOS_CFG
-echo "clog                2"              >> $TAOS_CFG
-#echo "cache              1"              >> $TAOS_CFG
-#echo "block              2"              >> $TAOS_CFG
-echo "statusInterval      1"              >> $TAOS_CFG
-echo "maxVgroupsPerDb     4"              >> $TAOS_CFG
-echo "minTablesPerVnode   4"              >> $TAOS_CFG
-echo "maxTablesPerVnode   1000"           >> $TAOS_CFG
-echo "tableIncStepPerVnode 10000"         >> $TAOS_CFG
-echo "asyncLog            0"              >> $TAOS_CFG
-echo "numOfMnodes         1"              >> $TAOS_CFG
-echo "locale    en_US.UTF-8"              >> $TAOS_CFG
-echo "fsync               0"              >> $TAOS_CFG
-
+echo " "                                         >> $TAOS_CFG   
+echo "firstEp                ${HOSTNAME}:7100"   >> $TAOS_CFG
+echo "secondEp               ${HOSTNAME}:7200"   >> $TAOS_CFG
+echo "serverPort             ${NODE}"            >> $TAOS_CFG
+echo "dataDir                $DATA_DIR"          >> $TAOS_CFG
+echo "logDir                 $LOG_DIR"           >> $TAOS_CFG
+echo "debugFlag              0"                  >> $TAOS_CFG
+echo "mDebugFlag             135"                >> $TAOS_CFG
+echo "sdbDebugFlag           135"                >> $TAOS_CFG
+echo "dDebugFlag             135"                >> $TAOS_CFG
+echo "vDebugFlag             135"                >> $TAOS_CFG
+echo "tsdbDebugFlag          135"                >> $TAOS_CFG
+echo "cDebugFlag             135"                >> $TAOS_CFG
+echo "jnidebugFlag           135"                >> $TAOS_CFG
+echo "odbcdebugFlag          135"                >> $TAOS_CFG
+echo "httpDebugFlag          143"                >> $TAOS_CFG
+echo "monitorDebugFlag       135"                >> $TAOS_CFG
+echo "mqttDebugFlag          135"                >> $TAOS_CFG
+echo "qdebugFlag             135"                >> $TAOS_CFG
+echo "rpcDebugFlag           135"                >> $TAOS_CFG
+echo "tmrDebugFlag           131"                >> $TAOS_CFG
+echo "udebugFlag             135"                >> $TAOS_CFG
+echo "sdebugFlag             135"                >> $TAOS_CFG
+echo "wdebugFlag             135"                >> $TAOS_CFG
+echo "monitor                0"                  >> $TAOS_CFG
+echo "monitorInterval        1"                  >> $TAOS_CFG
+echo "http                   0"                  >> $TAOS_CFG
+echo "numOfThreadsPerCore    2.0"                >> $TAOS_CFG
+echo "defaultPass            taosdata"           >> $TAOS_CFG
+echo "numOfLogLines          20000000"           >> $TAOS_CFG
+echo "mnodeEqualVnodeNum     0"                  >> $TAOS_CFG
+echo "clog                   2"                  >> $TAOS_CFG
+#echo "cache                 1"                  >> $TAOS_CFG
+#echo "block                 2"                  >> $TAOS_CFG
+echo "statusInterval         1"                  >> $TAOS_CFG
+echo "maxVgroupsPerDb        4"                  >> $TAOS_CFG
+echo "minTablesPerVnode      4"                  >> $TAOS_CFG
+echo "maxTablesPerVnode      1000"               >> $TAOS_CFG
+echo "tableIncStepPerVnode   10000"              >> $TAOS_CFG
+echo "asyncLog               0"                  >> $TAOS_CFG
+echo "numOfMnodes            1"                  >> $TAOS_CFG
+echo "locale                 en_US.UTF-8"        >> $TAOS_CFG
+echo "fsync                  0"                  >> $TAOS_CFG
+echo " "                                         >> $TAOS_CFG  
 
diff --git a/tests/script/test.sh b/tests/script/test.sh
index d9a738749f6223bc7f2ec76384bf49fa0c19a98e..96e4ffe689e5098b36ca9993eed66fd7ba9cacef 100755
--- a/tests/script/test.sh
+++ b/tests/script/test.sh
@@ -101,28 +101,29 @@ TAOS_FLAG=$PRG_DIR/flag
 
 HOSTNAME=`hostname -f`
 
-echo " "                                     >> $TAOS_CFG
-echo "first            ${HOSTNAME}:7100"     >> $TAOS_CFG
-echo "second           ${HOSTNAME}:7200"     >> $TAOS_CFG
-echo "serverPort       7100"                 >> $TAOS_CFG
-echo "dataDir          $DATA_DIR"            >> $TAOS_CFG
-echo "logDir           $LOG_DIR"             >> $TAOS_CFG
-echo "scriptDir        ${CODE_DIR}/../script">> $TAOS_CFG
-echo "numOfLogLines    100000000"            >> $TAOS_CFG
-echo "dDebugFlag       135"                  >> $TAOS_CFG
-echo "mDebugFlag       135"                  >> $TAOS_CFG
-echo "sdbDebugFlag     135"                  >> $TAOS_CFG
-echo "rpcDebugFlag     135"                  >> $TAOS_CFG
-echo "tmrDebugFlag     131"                  >> $TAOS_CFG
-echo "cDebugFlag       135"                  >> $TAOS_CFG
-echo "httpDebugFlag    135"                  >> $TAOS_CFG
-echo "monitorDebugFlag 135"                  >> $TAOS_CFG
-echo "udebugFlag       135"                  >> $TAOS_CFG
-echo "tablemetakeeptimer 5"                  >> $TAOS_CFG
-echo "wal              0"                    >> $TAOS_CFG
-echo "asyncLog         0"                    >> $TAOS_CFG
-echo "locale           en_US.UTF-8"          >> $TAOS_CFG
-echo " "                                     >> $TAOS_CFG
+echo " "                                          >> $TAOS_CFG
+echo "firstEp            ${HOSTNAME}:7100"        >> $TAOS_CFG
+echo "secondEp           ${HOSTNAME}:7200"        >> $TAOS_CFG
+echo "serverPort         7100"                    >> $TAOS_CFG
+echo "dataDir            $DATA_DIR"               >> $TAOS_CFG
+echo "logDir             $LOG_DIR"                >> $TAOS_CFG
+echo "scriptDir          ${CODE_DIR}/../script"   >> $TAOS_CFG
+echo "numOfLogLines      100000000"               >> $TAOS_CFG
+echo "dDebugFlag         135"                     >> $TAOS_CFG
+echo "mDebugFlag         135"                     >> $TAOS_CFG
+echo "sdbDebugFlag       135"                     >> $TAOS_CFG
+echo "rpcDebugFlag       135"                     >> $TAOS_CFG
+echo "tmrDebugFlag       131"                     >> $TAOS_CFG
+echo "cDebugFlag         135"                     >> $TAOS_CFG
+echo "httpDebugFlag      135"                     >> $TAOS_CFG
+echo "monitorDebugFlag   135"                     >> $TAOS_CFG
+echo "udebugFlag         135"                     >> $TAOS_CFG
+echo "tablemetakeeptimer 5"                       >> $TAOS_CFG
+echo "wal                0"                       >> $TAOS_CFG
+echo "asyncLog           0"                       >> $TAOS_CFG
+echo "locale             en_US.UTF-8"             >> $TAOS_CFG
+echo "enableCoreFile     1"                       >> $TAOS_CFG
+echo " "                                          >> $TAOS_CFG  
 
 ulimit -n 600000
 ulimit -c unlimited
diff --git a/tests/script/unique/clusterSimCase/client-01.sim b/tests/script/unique/clusterSimCase/client-01.sim
new file mode 100644
index 0000000000000000000000000000000000000000..b35790d69ee3272caa83a28566b9d77be7d7b4e9
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-01.sim
@@ -0,0 +1,69 @@
+# create database db1
+# create table tb ()
+# loop insert seq data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-01-
+###############################################################
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-01 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-01 insert data into  $tb  error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-02.sim b/tests/script/unique/clusterSimCase/client-02.sim
new file mode 100644
index 0000000000000000000000000000000000000000..30a2ceb08942140b3b03dec7a1285b30f6fd99ac
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-02.sim
@@ -0,0 +1,70 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-02-
+###############################################################
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-02 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-02 insert data into  $tb  error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-03.sim b/tests/script/unique/clusterSimCase/client-03.sim
new file mode 100644
index 0000000000000000000000000000000000000000..78a25e26a5344bbfe3434dec61643bf5a9c26f19
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-03.sim
@@ -0,0 +1,83 @@
+# create database db1
+# create table stb ()
+# loop dynamic create sub table and insert disorder data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-03-
+###############################################################
+
+sql connect
+
+print ================ create db and super-table
+$db  = db
+$stb = stb
+sql create database if not exists $db replica 2 
+sql use $db
+sql create table if not exists $stb ( ts timestamp , c1 int , c2 binary(16) ) tags ( t1 int , t2 binary(16) )
+
+#print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+#$i = $tblStart
+#while $i < $tblEnd
+#  $tb = tb . $i
+##  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+#  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+#  $i = $i + 1
+#endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-03 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+
+  $t2Binary = $preBinary . $i
+  $t2Binary = $t2Binary . ' 
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb using $stb tags ( $i ,  $t2Binary ) values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb using $stb tags ( $i ,  $t2Binary ) values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ****** client-3 insert data into $tb error ***
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+sql select count(*) from $stb
+print data00 $data00 totalRows $totalRows
+if $data00 != $totalRows then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ***** client-3 insert data into $stbl error ********
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-04.sim b/tests/script/unique/clusterSimCase/client-04.sim
new file mode 100644
index 0000000000000000000000000000000000000000..5248e9d7bb9b339cdcd157c495b9bc8f73705d89
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-04.sim
@@ -0,0 +1,81 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop database, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-04-
+###############################################################
+
+$totalRows   = 0
+
+sql connect
+
+init_lable:
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:    
+print ================ client-04 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-04 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 100 then
+  sql drop database if exists $db
+  sleep 20000
+  $totalRows   = 0
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-05.sim b/tests/script/unique/clusterSimCase/client-05.sim
new file mode 100644
index 0000000000000000000000000000000000000000..a2deaba9b2943b7e3720ae6c6f9b9001d5e3b743
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-05.sim
@@ -0,0 +1,86 @@
+# create database db1
+# loop create table ,  insert disorder data, drop table, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-05-
+###############################################################
+
+$totalRows   = 0
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-05 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-05 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 100 then
+
+  $i = $tblStart
+  while $i < $tblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 20000
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-06.sim b/tests/script/unique/clusterSimCase/client-06.sim
new file mode 100644
index 0000000000000000000000000000000000000000..209b615e64a2742cac732bd22d0e25f7a8f5db4e
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-06.sim
@@ -0,0 +1,95 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop some tables and create new tables, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-06-
+$numPerUpdateTbl = 2000
+###############################################################
+$totalRows    = 0  
+$dropTblStart = $tblStart
+$dropTblEnd   = $dropTblStart
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table [ from $tblStart to $tblEnd ] (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$dropTblStart = $dropTblEnd
+$dropTblEnd   = $dropTblStart + $numPerUpdateTbl
+
+$tblStart = $tblEnd
+$tblEnd   = $tblEnd + $numPerUpdateTbl
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-06 start loop insert data from $dropTblStart to $tblStart ( loopCnt: $loopCnt )
+$i = $dropTblStart
+while $i < $tblStart
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $dropTblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-06 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 2 then
+  print ================ client-06 drop table from $dropTblStart to $dropTblEnd
+  $i = $dropTblStart
+  while $i < $dropTblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 10000
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-07.sim b/tests/script/unique/clusterSimCase/client-07.sim
new file mode 100644
index 0000000000000000000000000000000000000000..3bf358520511dc04cc33c5953f91bee237ccd576
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-07.sim
@@ -0,0 +1,101 @@
+# create database db1
+# loop create table ,  insert disorder data, drop table, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-07-
+###############################################################
+$alterCnt = 0
+
+$columnA = c . $alterCnt
+$columnB = d . $alterCnt
+
+$totalRows   = 0
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+
+print ================ create table $tb (ts timestamp, $columnA int, $columnB binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , $columnA int , $columnB binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerLoop = 160
+
+$loopCnt = 0
+$alterStep = 0
+ 
+loop_run:
+print ================ client-07 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-07 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $alterStep > 2 then
+  $alterStep = 0
+  
+  $alterCnt = $alterCnt + 1
+  $i = $tblStart
+  while $i < $tblEnd
+    $tb = tb . $i
+    $newColumnA = c . $alterCnt
+    $newcolumnB = d . $alterCnt
+
+    sql alter table $tb add column $newColumnA double
+    sql alter table $tb add column $newcolumnB binary( 16 )
+    sql alter table $tb drop column $columnA
+    sql alter table $tb drop column $columnB
+    $i = $i + 1
+  endw
+  $columnA = $newColumnA
+  $columnB = $newcolumnB
+    
+  sleep 10000
+  goto loop_run 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-08.sim b/tests/script/unique/clusterSimCase/client-08.sim
new file mode 100644
index 0000000000000000000000000000000000000000..d9d64834cb36693328ce721dab449cdacc2f026a
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-08.sim
@@ -0,0 +1,95 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop some tables and create new tables, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-08-
+$numPerUpdateTbl = 2000
+###############################################################
+$totalRows    = 0  
+$dropTblStart = $tblStart
+$dropTblEnd   = $dropTblStart
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table [ from $tblStart to $tblEnd ] (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$dropTblStart = $dropTblEnd
+$dropTblEnd   = $dropTblStart + $numPerUpdateTbl
+
+$tblStart = $tblEnd
+$tblEnd   = $tblEnd + $numPerUpdateTbl
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 40
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-08 start loop insert data from $dropTblStart to $tblStart ( loopCnt: $loopCnt )
+$i = $dropTblStart
+while $i < $tblStart
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $dropTblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl tb: $tb
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-08 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+#if $loopCnt > 3 then
+  print ================ client-08 drop table from $dropTblStart to $dropTblEnd
+  $i = $dropTblStart
+  while $i < $dropTblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 10000
+  goto init_lable 
+#endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/cluster_main.sim b/tests/script/unique/clusterSimCase/cluster_main.sim
new file mode 100644
index 0000000000000000000000000000000000000000..39607c2915da7594f7af3562ae3ccd7680b2b4aa
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/cluster_main.sim
@@ -0,0 +1,248 @@
+# scp -r root@114.116.252.220:/home/ubuntu/clusterSimCase unique/.
+
+#taos> show dnodes;
+#   id   |           end_point            | vnodes | cores  |     status     |   role   |       create_time       |
+#==================================================================================================================
+#      1 | ubuntu-OptiPlex-7060:7100      |      0 |     12 | ready          | mnode    | 2020-07-22 06:25:31.677 |
+#      2 | ubuntu-OptiPlex-7060:7200      |      4 |     12 | ready          | vnode    | 2020-07-22 06:25:34.795 |
+#      3 | ubuntu-OptiPlex-7060:7300      |      4 |     12 | ready          | vnode    | 2020-07-22 06:25:34.803 |
+#Query OK, 3 row(s) in set (0.002448s)
+#taos> show vgroups;
+#    vgId     |   tables    |   status   | onlineVnodes | dnode  |   vstatus   | dnode  |   vstatus   |
+#======================================================================================================
+#           2 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           3 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           4 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           5 |         718 | ready      |            2 |      3 | master      |      2 | slave       |
+#Query OK, 4 row(s) in set (0.002749s)
+#
+#taos> show mnodes
+#   -> ;
+#   id   |           end_point            |      role      |       create_time       |
+#=====================================================================================
+#      1 | ubuntu-OptiPlex-7060:7100      | master         | 2020-07-22 06:25:31.677 |
+#Query OK, 1 row(s) in set (0.002126s)
+
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256
+
+#system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1
+
+#system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 256
+
+system sh/cfg.sh -n dnode1 -c role -v 1
+system sh/cfg.sh -n dnode2 -c role -v 2
+system sh/cfg.sh -n dnode3 -c role -v 2
+system sh/cfg.sh -n dnode4 -c role -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator and dnode1
+system sh/exec_tarbitrator.sh -s start
+system sh/exec.sh -n dnode1 -s start
+
+print ============== step2: start dnode2/dnode3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep 1000
+sql connect
+
+print ============== step1: add dnode2/dnode3 into cluster
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+sleep 3000
+print ============== step3: start back client-01.sim
+#run_back unique/clusterSimCase/client-01.sim
+#run_back unique/clusterSimCase/client-02.sim
+#run_back unique/clusterSimCase/client-03.sim
+#run_back unique/clusterSimCase/client-04.sim
+#run_back unique/clusterSimCase/client-05.sim
+#run_back unique/clusterSimCase/client-06.sim
+#run_back unique/clusterSimCase/client-07.sim
+run_back unique/clusterSimCase/client-08.sim
+#run_back unique/clusterSimCase/client-01.sim
+#run_back unique/clusterSimCase/client-01.sim
+sleep 20000
+
+$db = db
+sql use $db
+
+$loop_cnt = 0
+loop_cluster_do:
+print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
+
+print ============== step4: stop dnode3
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+
+$checkCnt = 0
+wait_vgroup_chang_0:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_0
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != offline then
+  sleep 2000
+  goto wait_vgroup_chang_0
+endi
+if $vg2Dnode2Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_0
+endi
+
+
+sleep 3000
+print ============== step3: restart dnode3
+system sh/exec.sh -n dnode3 -s start
+
+$checkCnt = 0
+wait_vgroup_chang_1:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_1
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != slave then
+  sleep 2000
+  goto wait_vgroup_chang_1
+endi
+if $vg2Dnode2Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_1
+endi
+
+sleep 3000
+print ============== step4: stop dnode2
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+
+$checkCnt = 0
+wait_vgroup_chang_2:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_2
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_2
+endi
+if $vg2Dnode2Status != offline then
+  sleep 2000
+  goto wait_vgroup_chang_2
+endi
+
+
+sleep 3000
+print ============== step5: restart dnode2
+system sh/exec.sh -n dnode2 -s start
+
+$checkCnt = 0
+wait_vgroup_chang_3:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_3
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode2Status != slave then
+  sleep 2000
+  goto wait_vgroup_chang_3
+endi
+if $vg2Dnode3Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_3
+endi
+
+sleep 3000
+print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** ****
+$loop_cnt = $loop_cnt + 1
+if $loop_cnt == 50 then
+  return 0
+endi
+goto loop_cluster_do
diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim
index 10fad091b53e2012e42275e8b99734f2ad2c502e..aeead817c16b8f2a15fff49ef38d65fdb9fe4207 100644
--- a/tests/script/unique/http/admin.sim
+++ b/tests/script/unique/http/admin.sim
@@ -57,13 +57,13 @@ endi
 
 system_content curl 127.0.0.1:6020/admin/login/root/123
 print 5-> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
 system_content curl 127.0.0.1:6020/admin/login/root/123/1/1/3
 print 6-> $system_content
-if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then
+if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then
   return -1
 endi
 
diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim
index 7e2400bb156b86019ad7af600875d8532440a64c..4c7803c3dacfc821c933e33140af8e5dba912bff 100644
--- a/tests/script/unique/http/opentsdb.sim
+++ b/tests/script/unique/http/opentsdb.sim
@@ -74,7 +74,7 @@ endi
 
 system_content curl -u root:taosdata -d  '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put
 print $system_content
-if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":1547,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then
+if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":1547,"desc":"Timestamp data out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then
   return -1
 endi
 
diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 8686a23f65dbbc064dd0844e34b710656c745249..4736c1ebed80d899b4b31d4dce395b7cc278761a 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   #add_executable(insertPerTable insertPerTable.c)
diff --git a/tests/test/c/createTablePerformance.c b/tests/test/c/createTablePerformance.c
index 2e334fa9b280e9165f26978f3b100001ef271a3f..69022bdecf856d3b579330f3fbe188048edebac4 100644
--- a/tests/test/c/createTablePerformance.c
+++ b/tests/test/c/createTablePerformance.c
@@ -31,7 +31,6 @@ char    dbName[32] = "db";
 char    stableName[64] = "st";
 int32_t numOfThreads = 30;
 int32_t numOfTables = 100000;
-int32_t maxTables = 5000;
 int32_t replica = 1;
 int32_t numOfColumns = 2;
 
@@ -98,7 +97,7 @@ void createDbAndSTable() {
     exit(1);
   }
 
-  sprintf(qstr, "create database if not exists %s maxtables %d replica %d", dbName, maxTables, replica);
+  sprintf(qstr, "create database if not exists %s replica %d", dbName, replica);
   TAOS_RES *pSql = taos_query(con, qstr);
   int32_t code = taos_errno(pSql);
   if (code != 0) {
@@ -195,8 +194,6 @@ void printHelp() {
   printf("%s%s%s%d\n", indent, indent, "replica, default is ", replica);
   printf("%s%s\n", indent, "-columns");
   printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns);
-  printf("%s%s\n", indent, "-tables");
-  printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", maxTables);
   
   exit(EXIT_SUCCESS);
 }
@@ -218,8 +215,6 @@ void shellParseArgument(int argc, char *argv[]) {
       numOfTables = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-r") == 0) {
       replica = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-tables") == 0) {
-      maxTables = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-columns") == 0) {
       numOfColumns = atoi(argv[++i]);
     } else {
@@ -233,7 +228,6 @@ void shellParseArgument(int argc, char *argv[]) {
   pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC);
   pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC);
   pPrint("%s replica:%d %s", GREEN, replica, NC);
-  pPrint("%s dbPara maxTables:%d %s", GREEN, maxTables, NC);
   
   pPrint("%s start create table performace test %s", GREEN, NC);
 }
diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt
index 14ef791a4e8c5bc7ccc14641823ad6d269c2a6fe..0a11a8f14ed0a8855471a5875b2ba8668c5410cc 100644
--- a/tests/tsim/CMakeLists.txt
+++ b/tests/tsim/CMakeLists.txt
@@ -1,11 +1,11 @@
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF (TD_WINDOWS_64)