diff --git a/.travis.yml b/.travis.yml
index 4ae38f50325d6b691c848f66da944def59db5c71..fcb2e8f22f94bf137966b7a3dd849a673303e587 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,13 @@
#
# Build Matrix
#
+branches:
+ only:
+ - master
+ - develop
+ - coverity_scan
+ - /^.*ci-.*$/
+
matrix:
- os: linux
dist: bionic
@@ -260,19 +267,75 @@ matrix:
- cmake .. > /dev/null
- make > /dev/null
- # - os: osx
- # language: c
- # compiler: clang
- # env: DESC="mac/clang build"
- # git:
- # - depth: 1
- # addons:
- # homebrew:
- # - cmake
- #
- # script:
- # - cd ${TRAVIS_BUILD_DIR}
- # - mkdir debug
- # - cd debug
- # - cmake .. > /dev/null
- # - make > /dev/null
+ - os: linux
+ arch: arm64
+ dist: bionic
+ language: c
+ compiler: clang
+ env: DESC="linux/clang build"
+ git:
+ - depth: 1
+
+ addons:
+ apt:
+ packages:
+ - build-essential
+ - cmake
+
+ before_script:
+ - cd ${TRAVIS_BUILD_DIR}
+ - mkdir debug
+ - cd debug
+
+ script:
+ - if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+ cmake .. -DCPUTYPE=aarch64 > /dev/null;
+ else
+ cmake .. > /dev/null;
+ fi
+ - make > /dev/null
+
+ - os: linux
+ arch: arm64
+ dist: trusty
+ language: c
+ git:
+ - depth: 1
+
+ addons:
+ apt:
+ packages:
+ - build-essential
+ - cmake
+ env:
+ - DESC="trusty/gcc-4.8 build"
+
+ before_script:
+ - cd ${TRAVIS_BUILD_DIR}
+ - mkdir debug
+ - cd debug
+
+ script:
+ - if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+ cmake .. -DCPUTYPE=aarch64 > /dev/null;
+ else
+ cmake .. > /dev/null;
+ fi
+ - make > /dev/null
+
+# - os: osx
+# language: c
+# compiler: clang
+# env: DESC="mac/clang build"
+# git:
+# - depth: 1
+# addons:
+# homebrew:
+# - cmake
+#
+# script:
+# - cd ${TRAVIS_BUILD_DIR}
+# - mkdir debug
+# - cd debug
+# - cmake .. > /dev/null
+# - make > /dev/null
diff --git a/documentation/webdocs/markdowndocs/Super Table-ch.md b/documentation/webdocs/markdowndocs/Super Table-ch.md
index 626f695c27a128e48353d4c1f674c273107b1fbc..5f592ec3393e50b25316fb29edd6abb13364e06a 100644
--- a/documentation/webdocs/markdowndocs/Super Table-ch.md
+++ b/documentation/webdocs/markdowndocs/Super Table-ch.md
@@ -53,7 +53,7 @@ STable从属于库，一个STable只属于一个库，但一个库可以有一
说明：
- 1. TAGS列总长度不能超过64k bytes；
+ 1. TAGS列总长度不能超过16k bytes；
2. TAGS列的数据类型不能是timestamp；
3. TAGS列名不能与其他列名相同;
4. TAGS列名不能为预留关键字.
diff --git a/documentation/webdocs/markdowndocs/Super Table.md b/documentation/webdocs/markdowndocs/Super Table.md
index 6c80c2097c32aba7efdd5bcfad4c129a11cdb577..2c1093a3fafd143ada150dc1d743ed49cd665d8c 100644
--- a/documentation/webdocs/markdowndocs/Super Table.md
+++ b/documentation/webdocs/markdowndocs/Super Table.md
@@ -22,7 +22,7 @@ New keyword "tags" is introduced, where tag_name is the tag name, and tag_type i
Note：
-1. The bytes of all tags together shall be less than 64k
+1. The bytes of all tags together shall be less than 16k
2. Tag's data type can not be time stamp
3. Tag name shall be different from the field name
4. Tag name shall not be the same as system keywords
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md
index 7fe396e9e5e156c5528267e69beffd29f06a2f18..0be39ab8e46c2836bd804563f7401d8614c04c1e 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md
+++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md
@@ -63,7 +63,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
| 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63用于NULL |
| 4 | FLOAT | 4 | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38] |
| 5 | DOUBLE | 8 | 双精度浮点型，有效位数15-16，范围 [-1.7E308, 1.7E308] |
-| 6 | BINARY | 自定义 | 用于记录字符串，理论上，最长可以有65526字节，但由于每行数据最多64K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
+| 6 | BINARY | 自定义 | 用于记录字符串，理论上，最长可以有16374字节，但由于每行数据最多16K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
| 7 | SMALLINT | 2 | 短整型， 范围 [-32767, 32767], -32768用于NULL |
| 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128用于NULL |
| 9 | BOOL | 1 | 布尔型，{true, false} |
@@ -106,7 +106,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```mysql
CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...])
```
- 说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过64K字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
+ 说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过16K字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
- **删除数据表**
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL.md b/documentation/webdocs/markdowndocs/TAOS SQL.md
index 6c7f25b8f7c342d8e38ebdcd5f674534c340f6be..72e41dbec49c29b51515e5f1dd56030cf200edab 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL.md
+++ b/documentation/webdocs/markdowndocs/TAOS SQL.md
@@ -39,7 +39,7 @@ The full list of data types is listed below. For string types of data, we will
| 6 | DOUBLE | 8 | A standard nullable double float type with 15-16 significant digits and a range of [-1.7E308, 1.7E308] |
| 7 | BOOL | 1 | A nullable boolean type, [**`true`**, **`false`**] |
| 8 | TIMESTAMP | 8 | A nullable timestamp type with the same usage as the primary column timestamp |
-| 9 | BINARY(*M*) | *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 65526, but as maximum row size is 64K bytes, the actual upper limit will generally less than 65526. This type of string only supports ASCii encoded chars. |
+| 9 | BINARY(*M*) | *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 16374, but as maximum row size is 16K bytes, the actual upper limit will generally less than 16374. This type of string only supports ASCii encoded chars. |
| 10 | NCHAR(*M*) | 4 * *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars. The **`NCHAR`** type supports Unicode encoded chars. |
All the keywords in a SQL statement are case-insensitive, but strings values are case-sensitive and must be quoted by a pair of `'` or `"`. To quote a `'` or a `"` , you can use the escape character `\`.
@@ -86,7 +86,7 @@ All the keywords in a SQL statement are case-insensitive, but strings values are
1) The first column must be a `timestamp`, and the system will set it as the primary key.
- 2) The record size is limited to 64k bytes
+ 2) The record size is limited to 16k bytes
3) For `binary` or `nchar` data types, the length must be specified. For example, binary(20) means a binary data type with 20 bytes.
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index 45fc6459e9027b80846e7bc899359bde4458210e..40b4c43fdd133723b10df290ca73584bd0ed0832 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -6,10 +6,22 @@
########################################################
# first full-qualified domain name (FQDN) for TDengine system
-# first 127.0.0.1
+# first hostname1:6030
# second full-qualified domain name (FQDN) for TDengine system, for cluster edition only
-# second 127.0.0.1
+# second cluster_hostname2:6030
+
+# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only
+# arbitrator arbitrator_hostname:6030
+
+# the full-qualified domain name (FQDN) of dnode
+# fqdn hostname:6030
+
+# port for MNode connect to Client, default udp[6030-6055] tcp[6030]
+# serverPort 6030
+
+# http service port, default tcp[6020]
+# httpPort 6020
# set socket type ("udp" and "tcp")
# the server and client should have the same socket type. Otherwise, connect will fail
@@ -30,42 +42,38 @@
# log file's directory
# logDir /var/log/taos
-# http service port, default tcp[6020]
-# httpPort 6020
+# number of management nodes in the system
+# numOfMnodes 3
-# port for MNode connect to Client, default udp[6030-6055] tcp[6030]
-# serverPort 6030
+# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode
+# alternativeRole 0
# number of threads per CPU core
-# numOfThreadsPerCore 1
+# numOfThreadsPerCore 1.0
# number of vnodes per core in DNode
# numOfVnodesPerCore 8
+# the ratio of threads responsible for querying in the total thread
+# ratioOfQueryThreads 0.5
+
# number of total vnodes in DNode
# numOfTotalVnodes 0
# max number of tables per vnode
# maxtablesPerVnode 1000
-# cache block size
-# cache 16384
+# interval of check load balance when the management node is in normal operation
+# balanceInterval 300
-# row of records in file block
-# minRows 100
-# maxRows 4096
+# interval of DNode report status to MNode, unit is Second, for cluster version only
+# statusInterval 1
-# number of cache blocks per vnode
-# blocks 2
+# interval of Shell send HB to MNode, unit is Second
+# shellActivityTimer 3
# interval of system monitor
-# monitorInterval 60
-
-# the compressed rpc message, option:
-# -1 (no compression)
-# 0 (all message compressed),
-# > 0 (rpc message body which larger than this value will be compressed)
-# compressMsgSize -1
+# monitorInterval 30
# RPC re-try timer, millisecond
# rpcTimer 300
@@ -73,38 +81,20 @@
# RPC maximum time for ack, seconds
# rpcMaxTime 600
-# commit interval，unit is second
-# ctime 3600
-
-# interval of DNode report status to MNode, unit is Second, for cluster version only
-# statusInterval 1
-
-# interval of Shell send HB to MNode, unit is Second
-# shellActivityTimer 3
-
-# interval of DNode send HB to DNode, unit is Second, for cluster version only
-# vnodePeerHBTimer 1
-
-# interval of MNode send HB to MNode, unit is Second, for cluster version only
-# mgmtPeerHBTimer 1
-
# duration of to keep tableMeta kept in Cache, seconds
# tableMetaKeepTimer 7200
-# max number of users
-# maxUsers 1000
-
-# max number of databases
-# maxDbs 1000
+# Minimum sliding window time
+# minSlidingTime 10
-# max number of tables
-# maxTables 650000
+# Time window minimum
+# minIntervalTime 10
-# max number of Dnodes, for cluster version only
-# maxDnodes 1000
+# max length of an SQL
+# maxSQLLength 65380
-# Max number of VGroups, for cluster version only
-# maxVGroups 1000
+# Support the maximum number of records allowed for super table time sorting
+# maxNumOfOrderedRes 100000
# system locale
# locale en_US.UTF-8
@@ -121,6 +111,24 @@
# enable/disable async log
# asyncLog 1
+# the compressed rpc message, option:
+# -1 (no compression)
+# 0 (all message compressed),
+# > 0 (rpc message body which larger than this value will be compressed)
+# compressMsgSize -1
+
+# cache block size (Mbyte)
+# cache 16
+
+# number of cache blocks per vnode
+# blocks 4
+
+# min row of records in file block
+# minRows 100
+
+# max row of records in file block
+# maxRows 4096
+
# enable/disable compression
# comp 1
@@ -131,31 +139,25 @@
# keep 3650
# number of replications, for cluster version only
-# replications 1
+# replica 1
-# client default database(database should be created)
-# defaultDB
+# max number of connections from client for dnode
+# maxShellConns 5000
-# client default username
-# defaultUser root
+# max numerber of connections to one database
+# maxConnections 5000
-# client default password
-# defaultPass taosdata
+# Stop writing logs when the disk size of the log folder is less than this value
+# minimalLogDirGB 0.1
-# max number of connections from client for mgmt node
-# maxShellConns 2000
+# Stop writing temporary files when the disk size of the log folder is less than this value
+# minimalTmpDirGB 0.1
-# max numerber of meter Connections
-# maxMeterConnections 10000
-
-# max connection to management node
-# maxMgmtConnections 2000
-
-# max connection to Vnode
-# maxVnodeConnections 10000
+# Stop writing data when the disk size of the log folder is less than this value
+# minimalDataDirGB 0.1
# mnode take into account while balance, for cluster version only
-# mnodeEqualVnodeNum 4
+# mnodeEqualVnodeNum 4
# number of seconds allowed for a dnode to be offline, for cluster version only
# offlineThreshold 864000
@@ -170,10 +172,10 @@
# mqtt 0
# mqtt uri
-# mqttBrokerAddress mqtt://username:password@hostname:1883/taos/
+# mqttBrokerAddress mqtt://username:password@hostname:1883/taos/
# mqtt client name
-# mqttBrokerClientId taos_mqtt
+# mqttBrokerClientId taos_mqtt
# maximum number of rows returned by the restful interface
# restfulRowLimit 10240
@@ -181,41 +183,17 @@
# number of threads used to process http requests
# httpMaxThreads 2
-# pre-allocated number of http sessions
-# httpCacheSessions 100
+# The minimum time to wait before the first stream execution
+# maxFirstStreamCompDelay 10000
-# whether to enable HTTP compression transmission
-# httpEnableCompress 0
+# Retry wait time benchmark
+# retryStreamCompDelay 10
# the delayed time for launching each continuous query. 10% of the whole computing time window by default.
# streamCompDelayRatio 0.1
# the max allowed delayed time for launching continuous query. 20ms by default
-# tsMaxStreamComputDelay 20000
-
-# whether the telegraf table name contains the number of tags and the number of fields
-# telegrafUseFieldNum 0
-
-# enable table names generated by telegraf can be configured
-# use /etc/taos/taos.telegraf.cfg as the configuration file, such as
-# {
-# "metrics": [
-# {
-# "name" : "system",
-# "tbname" : "system_uptime",
-# "fields": [
-# "uptime"
-# ]
-# },
-# {
-# "name": "system",
-# "tbname" : "system_uptime_format",
-# "fields": [
-# "uptime_format"
-# ]
-# }
-# ]
-# }
+# maxStreamCompDelay 20000
# The following parameter is used to limit the maximum number of lines in log files.
# max number of rows per log filters
@@ -226,35 +204,62 @@
# 131: output warning and error，135: output info, warning and error to log.
# 199: output debug, info, warning and error to both screen and file
+# debug flag for basic utils
+# debugFlag 131
+
# debug flag for meta management messages
# mDebugFlag 135
# debug flag for dnode messages
# dDebugFlag 131
+# debug flag for TDengine SDB
+# sDebugFlag 135
+
# debug flag for TDengine SDB
# sdbDebugFlag 135
# debug flag for RPC
# rpcDebugFlag 131
-# debug flag for basic utils
-# debugFlag 131
+# debug flag for TAOS TIMER
+# tmrDebugFlag 131
# debug flag for TDengine client
# cDebugFlag 131
-# debug flag for query
-# qDebugflag 131
+# debug flag for JNI
+# jniDebugflag 131
+
+# debug flag for ODBC
+# odbcDebugflag 131
+
+# debug flag for storage
+# uDebugflag 131
# debug flag for http server
# httpDebugFlag 131
-# debug flag for system monitor
+# debug flag for mqtt
+# mqttDebugFlag 131
+
+# debug flag for monitor
# monitorDebugFlag 131
-#debug flag for mqtt client
-# mqttDebugFlag 131
+# debug flag for query
+# qDebugflag 131
-# debug flag for TAOS TIMER
-# tmrDebugFlag 131
+# debug flag for http server
+# tsdbDebugFlag 131
+
+# Record the SQL through restful interface
+# httpEnableRecordSql 0
+
+# Record the SQL in taos client
+# tscEnableRecordSql 0
+
+# if generate core file when service crash
+# enableCoreFile 1
+
+# The maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
+# maxBinaryDisplayWidth 30
\ No newline at end of file
diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscLocalMerge.h
similarity index 87%
rename from src/client/inc/tscSecondaryMerge.h
rename to src/client/inc/tscLocalMerge.h
index 19230e34f1f8769229d0f386a8276a6c2395b279..c073f40546c6a5ceaeabeafd3d731fdf211402df 100644
--- a/src/client/inc/tscSecondaryMerge.h
+++ b/src/client/inc/tscLocalMerge.h
@@ -13,8 +13,8 @@
* along with this program. If not, see .
*/
-#ifndef TDENGINE_TSCSECONARYMERGE_H
-#define TDENGINE_TSCSECONARYMERGE_H
+#ifndef TDENGINE_TSCLOCALMERGE_H
+#define TDENGINE_TSCLOCALMERGE_H
#ifdef __cplusplus
extern "C" {
@@ -27,14 +27,7 @@ extern "C" {
#include "tsclient.h"
#define MAX_NUM_OF_SUBQUERY_RETRY 3
-
-/*
- * @version 0.1
- * @date 2018/01/05
- * @author liaohj
- * management of client-side reducer for metric query
- */
-
+
struct SQLFunctionCtx;
typedef struct SLocalDataSource {
@@ -60,7 +53,6 @@ typedef struct SLocalReducer {
char * prevRowOfInput;
tFilePage * pResultBuf;
int32_t nResultBufSize;
-// char * pBufForInterpo; // intermediate buffer for interpolation
tFilePage * pTempBuffer;
struct SQLFunctionCtx *pCtx;
int32_t rowSize; // size of each intermediate result.
@@ -81,13 +73,8 @@ typedef struct SLocalReducer {
} SLocalReducer;
typedef struct SSubqueryState {
- /*
- * the number of completed retrieval subquery, once this value equals to numOfVnodes,
- * all retrieval are completed.Local merge is launched.
- */
- int32_t numOfCompleted;
- int32_t numOfTotal; // number of total sub-queries
- int32_t code; // code from subqueries
+ int32_t numOfRemain; // the number of remain unfinished subquery
+ int32_t numOfTotal; // the number of total sub-queries
uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
@@ -128,4 +115,4 @@ int32_t tscDoLocalMerge(SSqlObj *pSql);
}
#endif
-#endif // TDENGINE_TSCSECONARYMERGE_H
+#endif // TDENGINE_TSCLOCALMERGE_H
diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h
index 368fe2250a37e3ef226174b80a39e63e0c990fda..82d490376aa8de18814c6cf9a1f3fbfb5be6ce75 100644
--- a/src/client/inc/tscSubquery.h
+++ b/src/client/inc/tscSubquery.h
@@ -26,11 +26,9 @@ extern "C" {
void tscFetchDatablockFromSubquery(SSqlObj* pSql);
void tscSetupOutputColumnIndex(SSqlObj* pSql);
-int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql);
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index);
-void tscDestroyJoinSupporter(SJoinSupporter* pSupporter);
int32_t tscHandleMasterJoinQuery(SSqlObj* pSql);
diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index b3e05cb0cafcd2bc68b9887e44272bf063acbbb7..f954667287c02c951c71e7fd3d11162209fa9890 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -28,7 +28,7 @@ extern "C" {
#include "exception.h"
#include "qextbuffer.h"
#include "taosdef.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
#include "tsclient.h"
#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
@@ -64,7 +64,8 @@ typedef struct SJoinSupporter {
SSubqueryState* pState;
SSqlObj* pObj; // parent SqlObj
int32_t subqueryIndex; // index of sub query
- int64_t interval; // interval time
+ int64_t intervalTime; // interval time
+ int64_t slidingTime; // sliding time
SLimitVal limit; // limit info
uint64_t uid; // query meter uid
SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution
@@ -122,15 +123,13 @@ bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableI
bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
-bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo);
+bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
-bool tscQueryOnSTable(SSqlCmd* pCmd);
bool tscQueryTags(SQueryInfo* pQueryInfo);
-bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd);
void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex,
- SSchema* pColSchema, int16_t isTag);
+ SSchema* pColSchema, int16_t colType);
int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql);
void tscClearInterpInfo(SQueryInfo* pQueryInfo);
@@ -139,7 +138,7 @@ bool tscIsInsertData(char* sqlstr);
/* use for keep current db info temporarily, for handle table with db prefix */
// todo remove it
-void tscGetDBInfoFromMeterId(char* tableId, char* db);
+void tscGetDBInfoFromTableFullName(char* tableId, char* db);
int tscAllocPayload(SSqlCmd* pCmd, int size);
@@ -248,12 +247,14 @@ void tscDoQuery(SSqlObj* pSql);
* @param pPrevSql
* @return
*/
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd);
+
SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql);
void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
-int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h
index 2fc77d69bbe0f276aa5b963da95ef8e6c98dcdae..99ffa4e766c42b094ff29517c70640caca674518 100644
--- a/src/client/inc/tschemautil.h
+++ b/src/client/inc/tschemautil.h
@@ -64,12 +64,20 @@ SSchema* tscGetTableSchema(const STableMeta* pTableMeta);
SSchema *tscGetTableTagSchema(const STableMeta *pMeta);
/**
- *
+ * get the column schema according to the column index
* @param pMeta
- * @param startCol
+ * @param colIndex
+ * @return
+ */
+SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
+
+/**
+ * get the column schema according to the column id
+ * @param pTableMeta
+ * @param colId
* @return
*/
-SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t startCol);
+SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
/**
* check if the schema is valid or not, including following aspects:
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index ed3b481d0e7d694eec2591345cd72d884592563d..a0eb1bd63c7767b774e58301625611e58817ec55 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -85,7 +85,7 @@ typedef struct SSqlExpr {
int16_t functionId; // function id in aAgg array
int16_t resType; // return value type
int16_t resBytes; // length of return value
- int32_t interBytes; // inter result buffer size
+ int32_t interBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function
tVariant param[3]; // parameters are not more than 3
int32_t offset; // sub result column value of arithmetic expression.
@@ -123,7 +123,7 @@ typedef struct SCond {
typedef struct SJoinNode {
char tableId[TSDB_TABLE_ID_LEN];
uint64_t uid;
- int16_t tagCol;
+ int16_t tagColId;
} SJoinNode;
typedef struct SJoinInfo {
@@ -155,20 +155,19 @@ typedef struct SParamInfo {
} SParamInfo;
typedef struct STableDataBlocks {
- char tableId[TSDB_TABLE_ID_LEN];
- int8_t tsSource; // where does the UNIX timestamp come from, server or client
- bool ordered; // if current rows are ordered or not
- int64_t vgId; // virtual group id
- int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending
- int32_t numOfTables; // number of tables in current submit block
-
- int32_t rowSize; // row size for current table
+ char tableId[TSDB_TABLE_ID_LEN];
+ int8_t tsSource; // where does the UNIX timestamp come from, server or client
+ bool ordered; // if current rows are ordered or not
+ int64_t vgId; // virtual group id
+ int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending
+ int32_t numOfTables; // number of tables in current submit block
+ int32_t rowSize; // row size for current table
uint32_t nAllocSize;
- uint32_t headerSize; // header for metadata (submit metadata)
+ uint32_t headerSize; // header for table info (uid, tid, submit metadata)
uint32_t size;
/*
- * the metermeta for current table, the metermeta will be used during submit stage, keep a ref
+ * the table meta of table, the table meta will be used during submit, keep a ref
* to avoid it to be removed from cache
*/
STableMeta *pTableMeta;
@@ -191,32 +190,28 @@ typedef struct SDataBlockList { // todo remove
} SDataBlockList;
typedef struct SQueryInfo {
- int16_t command; // the command may be different for each subclause, so keep it seperately.
- uint32_t type; // query/insert/import type
+ int16_t command; // the command may be different for each subclause, so keep it seperately.
+ uint32_t type; // query/insert/import type
char slidingTimeUnit;
-
STimeWindow window;
int64_t intervalTime; // aggregation time interval
int64_t slidingTime; // sliding window in mseconds
SSqlGroupbyExpr groupbyExpr; // group by tags info
-
- SArray * colList; // SArray
+ SArray * colList; // SArray
SFieldInfo fieldsInfo;
- SArray * exprList; // SArray
+ SArray * exprList; // SArray
SLimitVal limit;
SLimitVal slimit;
STagCond tagCond;
SOrderVal order;
- int16_t fillType; // final result fill type
+ int16_t fillType; // final result fill type
int16_t numOfTables;
STableMetaInfo **pTableMetaInfo;
struct STSBuf * tsBuf;
- int64_t * fillVal; // default value for fill
- char * msg; // pointer to the pCmd->payload to keep error message temporarily
- int64_t clauseLimit; // limit for current sub clause
-
- // offset value in the original sql expression, NOT sent to virtual node, only applied at client side
- int64_t prjOffset;
+ int64_t * fillVal; // default value for fill
+ char * msg; // pointer to the pCmd->payload to keep error message temporarily
+ int64_t clauseLimit; // limit for current sub clause
+ int64_t prjOffset; // offset value in the original sql expression, only applied at client side
} SQueryInfo;
typedef struct {
@@ -299,11 +294,12 @@ typedef struct STscObj {
} STscObj;
typedef struct SSqlObj {
- void * signature;
- STscObj *pTscObj;
- void (*fp)();
- void (*fetchFp)();
- void * param;
+ void *signature;
+ STscObj *pTscObj;
+ void *pRpcCtx;
+ void (*fp)();
+ void (*fetchFp)();
+ void *param;
int64_t stime;
uint32_t queryId;
void * pStream;
@@ -312,8 +308,7 @@ typedef struct SSqlObj {
char retry;
char maxRetry;
SRpcIpSet ipList;
- char freed : 4;
- char listed : 4;
+ char listed;
tsem_t rspSem;
SSqlCmd cmd;
SSqlRes res;
@@ -353,7 +348,7 @@ typedef struct SSqlStream {
int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
void tscInitMsgsFp();
-int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
+int tsParseSql(SSqlObj *pSql, bool initial);
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
int tscProcessSql(SSqlObj *pSql);
@@ -409,6 +404,7 @@ void tscKillSTableQuery(SSqlObj *pSql);
void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
bool tscIsUpdateQuery(SSqlObj* pSql);
bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
+bool tscResultsetFetchCompleted(TAOS_RES *result);
char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
@@ -431,7 +427,7 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
int32_t tscCompareTidTags(const void* p1, const void* p2);
-void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables);
+void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
#ifdef __cplusplus
}
diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c
index 459b0f2b9877e46e8428c15f30c2d246726a51e9..287e061da06aebd44ff7779f568ac04308ed1058 100644
--- a/src/client/src/TSDBJNIConnector.c
+++ b/src/client/src/TSDBJNIConnector.c
@@ -151,8 +151,8 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) {
if (jconfigDir != NULL) {
const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL);
- if (confDir && strlen(configDir) != 0) {
- strcpy(configDir, confDir);
+ if (confDir && strlen(confDir) != 0) {
+ tstrncpy(configDir, confDir, TSDB_FILENAME_LEN);
}
(*env)->ReleaseStringUTFChars(env, jconfigDir, confDir);
}
@@ -192,7 +192,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv
const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL);
if (tz1 && strlen(tz1) != 0) {
res = taos_options(TSDB_OPTION_TIMEZONE, tz1);
- jniTrace("set timezone to %s, result:%d", timezone, res);
+ jniTrace("set timezone to %s, result:%d", tz1, res);
} else {
jniTrace("input timezone is empty");
}
@@ -227,12 +227,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn
}
if (user == NULL) {
- jniTrace("jobj:%p, user is null, use tsDefaultUser", jobj);
- user = tsDefaultUser;
+ jniTrace("jobj:%p, user is null, use default user %s", jobj, TSDB_DEFAULT_USER);
}
if (pass == NULL) {
- jniTrace("jobj:%p, pass is null, use tsDefaultPass", jobj);
- pass = tsDefaultPass;
+ jniTrace("jobj:%p, pass is null, use default password", jobj);
}
/*
@@ -252,8 +250,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn
if (host != NULL) (*env)->ReleaseStringUTFChars(env, jhost, host);
if (dbname != NULL) (*env)->ReleaseStringUTFChars(env, jdbName, dbname);
- if (user != NULL && user != tsDefaultUser) (*env)->ReleaseStringUTFChars(env, juser, user);
- if (pass != NULL && pass != tsDefaultPass) (*env)->ReleaseStringUTFChars(env, jpass, pass);
+ if (user != NULL) (*env)->ReleaseStringUTFChars(env, juser, user);
+ if (pass != NULL) (*env)->ReleaseStringUTFChars(env, jpass, pass);
return ret;
}
@@ -344,7 +342,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(
STscObj *pObj = pSql->pTscObj;
if (tscIsUpdateQuery(pSql)) {
- taos_free_result(pSql); // free result here
+ // taos_free_result(pSql); // free result here
jniTrace("jobj:%p, conn:%p, no resultset, %p", jobj, pObj, (void *)tres);
return 0;
} else {
@@ -385,7 +383,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsIm
}
jint ret = taos_affected_rows((SSqlObj *)res);
- jniTrace("jobj:%p, conn:%p, sql:%p, affect rows:%d", jobj, tscon, (void *)con, res, ret);
+ jniTrace("jobj:%p, conn:%p, sql:%p, res: %p, affect rows:%d", jobj, tscon, (void *)con, (void *)res, ret);
return ret;
}
@@ -411,10 +409,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData
// jobject arrayListObj = (*env)->NewObject(env, g_arrayListClass, g_arrayListConstructFp, "");
if (num_fields == 0) {
- jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields);
+ jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, (void *)res, num_fields);
return JNI_NUM_OF_FIELDS_0;
} else {
- jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields);
+ jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, (void *)res, num_fields);
for (int i = 0; i < num_fields; ++i) {
jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp);
(*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type);
@@ -465,7 +463,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
int num_fields = taos_num_fields(result);
if (num_fields == 0) {
- jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields);
+ jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, (void*)res, num_fields);
return JNI_NUM_OF_FIELDS_0;
}
@@ -473,7 +471,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
if (row == NULL) {
int tserrno = taos_errno(result);
if (tserrno == 0) {
- jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, res, num_fields);
+ jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, (void*)res, num_fields);
return JNI_FETCH_END;
} else {
jniTrace("jobj:%p, conn:%p, interruptted query", jobj, tscon);
@@ -571,9 +569,9 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
sub = (jlong)tsub;
if (sub == 0) {
- jniTrace("jobj:%p, failed to subscribe: topic:%s", jobj, jtopic);
+ jniTrace("jobj:%p, failed to subscribe: topic:%s", jobj, topic);
} else {
- jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, jtopic);
+ jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, topic);
}
if (topic != NULL) (*env)->ReleaseStringUTFChars(env, jtopic, topic);
@@ -583,7 +581,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
}
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) {
- jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub);
+ jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub);
jniGetGlobalMethod(env);
TAOS_SUB *tsub = (TAOS_SUB *)sub;
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index f4fad427192f155b957ad339471ea2bd19bc0db1..2b99d230993472b130e662a1191c0ac3e706c413 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -19,9 +19,8 @@
#include "tnote.h"
#include "trpc.h"
#include "tscLog.h"
-#include "tscProfile.h"
#include "tscSubquery.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
#include "tscUtil.h"
#include "tsched.h"
#include "tschemautil.h"
@@ -40,38 +39,26 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
-int doAsyncParseSql(SSqlObj* pSql) {
- SSqlCmd* pCmd = &pSql->cmd;
- SSqlRes* pRes = &pSql->res;
- int32_t code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
- if (code != TSDB_CODE_SUCCESS) {
- tscError("failed to malloc payload");
- tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
- return code;
- }
-
- pRes->qhandle = 0;
- pRes->numOfRows = 1;
-
- tscDump("%p SQL: %s", pSql, pSql->sqlstr);
- return tsParseSql(pSql, true);
-}
-
void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
pSql->signature = pSql;
pSql->param = param;
pSql->pTscObj = pObj;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
pSql->fp = fp;
+
pSql->sqlstr = calloc(1, sqlLen + 1);
if (pSql->sqlstr == NULL) {
tscError("%p failed to malloc sql string buffer", pSql);
tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
return;
}
+
strtolower(pSql->sqlstr, sqlstr);
-
- int32_t code = doAsyncParseSql(pSql);
+
+ tscDump("%p SQL: %s", pSql, pSql->sqlstr);
+ pSql->cmd.curSql = pSql->sqlstr;
+
+ int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
if (code != TSDB_CODE_SUCCESS) {
@@ -211,7 +198,8 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
if (pRes->qhandle == 0) {
tscError("qhandle is NULL");
- tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_QHANDLE);
+ pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
+ tscQueueAsyncRes(pSql);
return;
}
@@ -285,7 +273,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
tscProcessSql(pSql);
} else {
- SSchedMsg schedMsg;
+ SSchedMsg schedMsg = { 0 };
schedMsg.fp = tscProcessFetchRow;
schedMsg.ahandle = pSql;
schedMsg.thandle = pRes->tsrow;
@@ -387,7 +375,7 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) {
int32_t* c = malloc(sizeof(int32_t));
*c = code;
- SSchedMsg schedMsg;
+ SSchedMsg schedMsg = { 0 };
schedMsg.fp = tscProcessAsyncError;
schedMsg.ahandle = fp;
schedMsg.thandle = param;
@@ -403,7 +391,7 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
}
- SSchedMsg schedMsg;
+ SSchedMsg schedMsg = { 0 };
schedMsg.fp = tscProcessAsyncRes;
schedMsg.ahandle = pSql;
schedMsg.thandle = (void *)1;
@@ -420,7 +408,7 @@ void tscProcessAsyncFree(SSchedMsg *pMsg) {
void tscQueueAsyncFreeResult(SSqlObj *pSql) {
tscTrace("%p sqlObj put in queue to async free", pSql);
- SSchedMsg schedMsg;
+ SSchedMsg schedMsg = { 0 };
schedMsg.fp = tscProcessAsyncFree;
schedMsg.ahandle = pSql;
schedMsg.thandle = (void *)1;
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 457e187971605a94e34cff67376657ad5a880a1a..d5e3126f71230241e1614642c6118d92e6a0d03d 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -153,7 +153,7 @@ typedef struct SRateInfo {
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
- if (!isValidDataType(dataType, dataBytes)) {
+ if (!isValidDataType(dataType)) {
tscError("Illegal data type %d or data type length %d", dataType, dataBytes);
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -1853,26 +1853,14 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
static void last_row_finalizer(SQLFunctionCtx *pCtx) {
// do nothing at the first stage
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
- if (pResInfo->hasResult != DATA_SET_FLAG) {
- if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
- } else {
- setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
- }
-
- return;
- }
- } else {
- if (pResInfo->hasResult != DATA_SET_FLAG) {
- if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
- } else {
- setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
- }
-
- return;
+ if (pResInfo->hasResult != DATA_SET_FLAG) {
+ if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+ setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+ } else {
+ setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
}
+
+ return;
}
GET_RES_INFO(pCtx)->numOfRes = 1;
@@ -2989,12 +2977,12 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
*/
static void tag_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, 1, 1);
- tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
+ tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
}
static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
- tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
+ tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
}
static void copy_function(SQLFunctionCtx *pCtx) {
@@ -3903,7 +3891,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
STSCompInfo *pInfo = pResInfo->interResultBuf;
- pInfo->pTSBuf = tsBufCreate(false);
+ pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
pInfo->pTSBuf->tsOrder = pCtx->order;
return true;
}
@@ -3925,7 +3913,6 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) {
}
SET_VAL(pCtx, pCtx->size, 1);
-
pResInfo->hasResult = DATA_SET_FLAG;
}
diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index a880f4487ef73199a0c6198894a7d4d647d7dbe1..83700ce0a573ccf15a474a58ec4ebfda2634e2fd 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -22,7 +22,6 @@
#include "taosdef.h"
#include "tscLog.h"
#include "qextbuffer.h"
-#include "tscSecondaryMerge.h"
#include "tschemautil.h"
#include "tname.h"
@@ -183,8 +182,12 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
// type length
int32_t bytes = pSchema[i].bytes;
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
- if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
- bytes = bytes / TSDB_NCHAR_SIZE;
+ if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+ bytes -= VARSTR_HEADER_SIZE;
+
+ if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+ bytes = bytes / TSDB_NCHAR_SIZE;
+ }
}
*(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;
diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscLocalMerge.c
similarity index 98%
rename from src/client/src/tscSecondaryMerge.c
rename to src/client/src/tscLocalMerge.c
index 52a06277e360a583f8e56f27fed38db741815db0..bc3c33423f85fcc9c81e122b9a4e3904f598410a 100644
--- a/src/client/src/tscSecondaryMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -13,7 +13,6 @@
* along with this program. If not, see .
*/
-#include "tscSecondaryMerge.h"
#include "os.h"
#include "tlosertree.h"
#include "tscUtil.h"
@@ -21,6 +20,7 @@
#include "tsclient.h"
#include "tutil.h"
#include "tscLog.h"
+#include "tscLocalMerge.h"
typedef struct SCompareParam {
SLocalDataSource **pLocalData;
@@ -230,6 +230,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds == NULL) {
tscError("%p failed to create merge structure", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+ tfree(pReducer);
return;
}
@@ -266,6 +267,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
// no data actually, no need to merge result.
if (idx == 0) {
+ tfree(pReducer);
return;
}
@@ -282,6 +284,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
if (pReducer->pLoserTree == NULL || pRes->code != 0) {
+ tfree(pReducer);
return;
}
@@ -325,7 +328,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes);
tfree(pReducer->prevRowOfInput);
-
+ tfree(pReducer);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return;
}
@@ -353,7 +356,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->numOfGroups = 0;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
- STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);;
+ STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
int64_t revisedSTime =
@@ -410,13 +413,13 @@ static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor
}
int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType) {
- int32_t ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType);
- if (ret != 0) {
- return -1;
+ int32_t ret = 0;
+ if ((ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType)) != 0) {
+ return ret;
}
- if (!tExtMemBufferFlush(pMemoryBuf)) {
- return -1;
+ if ((ret = tExtMemBufferFlush(pMemoryBuf)) != 0) {
+ return ret;
}
return 0;
@@ -437,9 +440,9 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
// current buffer is full, need to flushed to disk
assert(pPage->num == pModel->capacity);
- int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
- if (ret != 0) {
- return -1;
+ int32_t code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
+ if (code != 0) {
+ return code;
}
int32_t remain = numOfRows - numOfRemainEntries;
@@ -455,8 +458,8 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows);
if (pPage->num == pModel->capacity) {
- if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) {
- return -1;
+ if ((code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType)) != TSDB_CODE_SUCCESS) {
+ return code;
}
} else {
pPage->num = numOfWriteElems;
@@ -685,6 +688,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+ tfree(pSchema);
return pRes->code;
}
@@ -1092,14 +1096,6 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t j = 0; j < size; ++j) {
- // SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j];
- // if (pExpr == NULL) {
- // assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL);
- //
- // maxOutput = 1;
- // continue;
- // }
-
/*
* ts, tag, tagprj function can not decide the output number of current query
* the number of output result is decided by main output
@@ -1109,8 +1105,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
continue;
}
- if (maxOutput < GET_RES_INFO(&pCtx[j])->numOfRes) {
- maxOutput = GET_RES_INFO(&pCtx[j])->numOfRes;
+ SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
+ if (maxOutput < pResInfo->numOfRes) {
+ maxOutput = pResInfo->numOfRes;
}
}
@@ -1260,7 +1257,6 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
#ifdef _DEBUG_VIEW
printf("final result before interpo:\n");
- assert(0);
// tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
#endif
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index afd1a3e8c881b6f89224a9b7744dbb2b179e36fa..69bc4a3e20049aaa520df99393dcb17a80950ab4 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -252,7 +252,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
numType = tscToInteger(pToken, &iv, &endptr);
if (TK_ILLEGAL == numType) {
return tscInvalidSQLErrMsg(msg, "invalid bigint data", pToken->z);
- } else if (errno == ERANGE || iv > INT64_MAX || iv <= INT64_MIN) {
+ } else if (errno == ERANGE || iv == INT64_MIN) {
return tscInvalidSQLErrMsg(msg, "bigint data overflow", pToken->z);
}
@@ -594,7 +594,6 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
size_t remain = pDataBlock->nAllocSize - pDataBlock->size;
const int factor = 5;
uint32_t nAllocSizeOld = pDataBlock->nAllocSize;
- assert(pDataBlock->headerSize >= 0);
// expand the allocated size
if (remain < rowSize * factor) {
@@ -987,14 +986,16 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return code;
}
-int validateTableName(char *tblName, int len) {
+int validateTableName(char *tblName, int len, SSQLToken* psTblToken) {
char buf[TSDB_TABLE_ID_LEN] = {0};
tstrncpy(buf, tblName, sizeof(buf));
- SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
- tSQLGetToken(buf, &token.type);
+ psTblToken->n = len;
+ psTblToken->type = TK_ID;
+ psTblToken->z = buf;
+ tSQLGetToken(buf, &psTblToken->type);
- return tscValidateName(&token);
+ return tscValidateName(psTblToken);
}
static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
@@ -1015,46 +1016,41 @@ static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
* @param pSql
* @return
*/
-int doParseInsertSql(SSqlObj *pSql, char *str) {
+int tsParseInsertSql(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
+ char* str = pCmd->curSql;
int32_t totalNum = 0;
int32_t code = TSDB_CODE_SUCCESS;
- STableMetaInfo *pTableMetaInfo = NULL;
-
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
assert(pQueryInfo != NULL);
+ STableMetaInfo *pTableMetaInfo = NULL;
if (pQueryInfo->numOfTables == 0) {
pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo);
} else {
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
}
- // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536
- // but TSDB_PAYLOAD_SIZE is 65380
+ // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536, but TSDB_PAYLOAD_SIZE is 65380
if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
return code;
}
- assert(((NULL == pCmd->curSql) && (NULL == pCmd->pTableList))
- || ((NULL != pCmd->curSql) && (NULL != pCmd->pTableList)));
-
- if ((NULL == pCmd->curSql) && (NULL == pCmd->pTableList)) {
+ if (NULL == pCmd->pTableList) {
pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
pSql->cmd.pDataBlocks = tscCreateBlockArrayList();
if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- goto _error_clean;
+ goto _error;
}
} else {
- assert((NULL != pCmd->curSql) && (NULL != pCmd->pTableList));
str = pCmd->curSql;
}
- tscTrace("%p create data block list for submit data:%p, curSql:%p, pTableList:%p", pSql, pSql->cmd.pDataBlocks, pCmd->curSql, pCmd->pTableList);
+ tscTrace("%p create data block list for submit data:%p, pTableList:%p", pSql, pCmd->pDataBlocks, pCmd->pTableList);
while (1) {
int32_t index = 0;
@@ -1076,48 +1072,41 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
*/
if (totalNum == 0) {
code = TSDB_CODE_TSC_INVALID_SQL;
- goto _error_clean;
+ goto _error;
} else {
break;
}
}
pCmd->curSql = sToken.z;
-
+ SSQLToken sTblToken;
// Check if the table name available or not
- if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) {
+ if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);
- goto _error_clean;
+ goto _error;
}
- if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) {
+ goto _error;
}
- ptrdiff_t pos = pCmd->curSql - pSql->sqlstr;
-
if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) {
/*
- * For async insert, after get the table meta from server, the sql string will not be
- * parsed using the new table meta to avoid the overhead cause by get table meta data information.
- * And during the getMeterMetaCallback function, the sql string will be parsed from the
- * interrupted position.
+ * After retrieving the table meta from server, the sql string will be parsed from the paused position.
+ * And during the getTableMetaCallback function, the sql string will be parsed from the paused position.
*/
if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
- tscTrace("%p waiting for get table meta during insert, then resume from offset: %" PRId64 " , %s", pSql,
- pos, pCmd->curSql);
return code;
}
- // todo add to return
- tscError("%p async insert parse error, code:%d, %s", pSql, code, tstrerror(code));
+ tscError("%p async insert parse error, code:%s", pSql, tstrerror(code));
pCmd->curSql = NULL;
- goto _error_clean; // TODO: should _clean or _error_clean to async flow ????
+ goto _error;
}
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL);
- goto _error_clean;
+ goto _error;
}
index = 0;
@@ -1126,7 +1115,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
if (sToken.n == 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z);
- goto _error_clean;
+ goto _error;
}
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
@@ -1138,7 +1127,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns);
if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
/*
@@ -1147,11 +1136,11 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
*/
code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum);
if (code != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
} else if (sToken.type == TK_FILE) {
if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
index = 0;
@@ -1159,7 +1148,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
str += index;
if (sToken.n == 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
- goto _error_clean;
+ goto _error;
}
char fname[PATH_MAX] = {0};
@@ -1169,7 +1158,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
wordexp_t full_path;
if (wordexp(fname, &full_path, 0) != 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z);
- goto _error_clean;
+ goto _error;
}
strcpy(fname, full_path.we_wordv[0]);
wordfree(&full_path);
@@ -1180,7 +1169,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name,
pTableMeta, &pDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock);
@@ -1191,7 +1180,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
SSchema * pSchema = tscGetTableSchema(pTableMeta);
if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
SParsedDataColInfo spd = {0};
@@ -1227,7 +1216,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
if (spd.hasVal[t] == true) {
code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z);
- goto _error_clean;
+ goto _error;
}
spd.hasVal[t] = true;
@@ -1238,13 +1227,13 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
if (!findColumnIndex) {
code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column name", sToken.z);
- goto _error_clean;
+ goto _error;
}
}
if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) {
code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z);
- goto _error_clean;
+ goto _error;
}
index = 0;
@@ -1253,16 +1242,16 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
if (sToken.type != TK_VALUES) {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z);
- goto _error_clean;
+ goto _error;
}
code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum);
if (code != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
} else {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE are required", sToken.z);
- goto _error_clean;
+ goto _error;
}
}
@@ -1273,7 +1262,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
- goto _error_clean;
+ goto _error;
}
} else {
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
@@ -1282,7 +1271,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
code = TSDB_CODE_SUCCESS;
goto _clean;
-_error_clean:
+_error:
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
_clean:
@@ -1295,7 +1284,7 @@ _clean:
return code;
}
-int tsParseInsertSql(SSqlObj *pSql) {
+int tsInsertInitialCheck(SSqlObj *pSql) {
if (!pSql->pTscObj->writeAuth) {
return TSDB_CODE_TSC_NO_WRITE_AUTH;
}
@@ -1313,28 +1302,23 @@ int tsParseInsertSql(SSqlObj *pSql) {
SQueryInfo *pQueryInfo = NULL;
tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
- TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
- TSDB_QUERY_SET_TYPE(pQueryInfo->type, pCmd->insertType);
+ TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType);
sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
if (sToken.type != TK_INTO) {
return tscInvalidSQLErrMsg(pCmd->payload, "keyword INTO is expected", sToken.z);
}
- return doParseInsertSql(pSql, pSql->sqlstr + index);
+ pCmd->curSql = sToken.z + sToken.n;
+ return TSDB_CODE_SUCCESS;
}
-int tsParseSql(SSqlObj *pSql, bool initialParse) {
+int tsParseSql(SSqlObj *pSql, bool initial) {
int32_t ret = TSDB_CODE_SUCCESS;
-
- if (initialParse) {
- char* p = pSql->sqlstr;
- pSql->sqlstr = NULL;
-
- tscPartiallyFreeSqlObj(pSql);
- pSql->sqlstr = p;
- } else {
- tscTrace("continue parse sql: %s", pSql->cmd.curSql);
+ SSqlCmd* pCmd = &pSql->cmd;
+
+ if ((!pCmd->parseFinished) && (!initial)) {
+ tscTrace("%p resume to parse sql: %s", pSql, pCmd->curSql);
}
if (tscIsInsertData(pSql->sqlstr)) {
@@ -1342,18 +1326,22 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
* Set the fp before parse the sql string, in case of getTableMeta failed, in which
* the error handle callback function can rightfully restore the user-defined callback function (fp).
*/
- if (initialParse && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
+ if (initial && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
pSql->fetchFp = pSql->fp;
pSql->fp = (void(*)())tscHandleMultivnodeInsert;
}
-
+
+ if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) {
+ return ret;
+ }
+
ret = tsParseInsertSql(pSql);
} else {
ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
}
-
+
SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr);
ret = tscToSQLCmd(pSql, &SQLInfo);
SQLInfoDestroy(&SQLInfo);
@@ -1361,11 +1349,9 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
/*
* the pRes->code may be modified or released by another thread in tscTableMetaCallBack function,
- * so do NOT use pRes->code to determine if the getTableMeta/getMetricMeta function
- * invokes new threads to get data from mnode or simply retrieves data from cache.
- *
- * do NOT assign return code to pRes->code for the same reason since it may be released by another thread
- * pRes->code = ret;
+ * so do NOT use pRes->code to determine if the getTableMeta function
+ * invokes new threads to get data from mgmt node or simply retrieves data from cache.
+ * do NOT assign return code to pRes->code for the same reason since it may be released by another thread already.
*/
return ret;
}
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index db12693c2925228b9947bd650a673c9b3de994e0..f241e317cdc83b432a1a1d964dff80ea84b663bb 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -98,8 +98,6 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy
static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
-static void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex);
-
static int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql);
static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql);
static int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
@@ -640,17 +638,11 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
return TSDB_CODE_TSC_INVALID_SQL;
}
- SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
- SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
- TSDB_KEYSIZE, false);
-
- SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX);
+ SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+ tstrncpy(s.name, aAggs[TSDB_FUNC_TS].aName, sizeof(s.name));
- int32_t ret =
- insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName, pExpr);
- if (ret != TSDB_CODE_SUCCESS) {
- return ret;
- }
+ SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
+ tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL);
if (parseSlidingClause(pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
@@ -760,6 +752,10 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
int32_t nLen = 0;
for (int32_t i = 0; i < pFieldList->nField; ++i) {
+ if (pFieldList->p[i].bytes == 0) {
+ invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
+ return false;
+ }
nLen += pFieldList->p[i].bytes;
}
@@ -816,6 +812,10 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
int32_t nLen = 0;
for (int32_t i = 0; i < pTagsList->nField; ++i) {
+ if (pTagsList->p[i].bytes == 0) {
+ invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
+ return false;
+ }
nLen += pTagsList->p[i].bytes;
}
@@ -1241,11 +1241,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
return invalidSqlErrMsg(pQueryInfo->msg, msg2);
}
+ /*
+ * transfer sql functions that need secondary merge into another format
+ * in dealing with metric queries such as: count/first/last
+ */
if (isSTable) {
- /*
- * transfer sql functions that need secondary merge into another format
- * in dealing with metric queries such as: count/first/last
- */
tscTansformSQLFuncForSTableQuery(pQueryInfo);
if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) {
@@ -1272,7 +1272,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi
}
TAOS_FIELD f = tscCreateField(type, fieldName, bytes);
- SFieldSupInfo* pInfo =tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
+ SFieldSupInfo* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
pInfo->pSqlExpr = pSqlExpr;
return TSDB_CODE_SUCCESS;
@@ -1324,8 +1324,9 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) {
- SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type,
+ SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type,
pColSchema->bytes, pColSchema->bytes, flag);
+ tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName));
SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex);
if (TSDB_COL_IS_TAG(flag)) {
@@ -1403,7 +1404,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
SSchema colSchema = tGetTableNameColumnSchema();
- tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true);
+ tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG);
} else {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@@ -1445,7 +1446,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
}
if (aliasName != NULL) {
- strcpy(columnName, aliasName);
+ tstrncpy(columnName, aliasName, sizeof(columnName));
} else {
getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
}
@@ -1481,7 +1482,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
const char* msg5 = "parameter is out of range [0, 100]";
const char* msg6 = "function applied to tags not allowed";
const char* msg7 = "normal table can not apply this function";
-
+ const char* msg8 = "multi-columns selection does not support alias column name";
+
switch (optr) {
case TK_COUNT: {
if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) {
@@ -1688,6 +1690,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
return invalidSqlErrMsg(pQueryInfo->msg, msg3);
}
+ if (pItem->pNode->pParam->nExpr > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) {
+ return invalidSqlErrMsg(pQueryInfo->msg, msg8);
+ }
+
/* in first/last function, multiple columns can be add to resultset */
for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) {
tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]);
@@ -1754,6 +1760,11 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
} else { // select * from xxx
int32_t numOfFields = 0;
+ // multicolumn selection does not support alias name
+ if (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0) {
+ return invalidSqlErrMsg(pQueryInfo->msg, msg8);
+ }
+
for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) {
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j);
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
@@ -1846,7 +1857,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
} else {
tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true);
- int64_t nTop = *((int32_t*)val);
+ int64_t nTop = GET_INT32_VAL(val);
if (nTop <= 0 || nTop > 100) { // todo use macro
return invalidSqlErrMsg(pQueryInfo->msg, msg5);
}
@@ -1856,12 +1867,14 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
return TSDB_CODE_TSC_INVALID_SQL;
}
+ // todo REFACTOR
// set the first column ts for top/bottom query
SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
TSDB_KEYSIZE, false);
+ tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName));
- const int32_t TS_COLUMN_INDEX = 0;
+ const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX;
SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX);
insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP,
aAggs[TSDB_FUNC_TS].aName, pExpr);
@@ -2221,7 +2234,6 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// db prefix in tagCond, show table conds in payload
SSQLToken* pDbPrefixToken = &pShowInfo->prefix;
if (pDbPrefixToken->type != 0) {
- assert(pDbPrefixToken->n >= 0);
if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
@@ -2470,62 +2482,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
return true;
}
-void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) {
-// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-//
-// // update tags column index for expression
-// size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-// for (int32_t i = 0; i < size; ++i) {
-// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-//
-// if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue
-// continue;
-// }
-//
-// // not belongs to this table
-// if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) {
-// continue;
-// }
-
-// for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) {
-// if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) {
-// pExpr->colInfo.colIndex = j;
-// break;
-// }
-// }
-// }
-
- // update join condition tag column index
-// SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo;
-// if (!pJoinInfo->hasJoin) { // not join query
-// return;
-// }
-//
-// assert(pJoinInfo->left.uid != pJoinInfo->right.uid);
-//
-// // the join condition expression node belongs to this table(super table)
-// assert(0);
-// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) {
-// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) {
-// if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) {
-// pJoinInfo->left.tagCol = i;
-// }
-// }
-// }
-//
-// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) {
-// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) {
-// if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) {
-// pJoinInfo->right.tagCol = i;
-// }
-// }
-// }
-}
-
int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
const char* msg1 = "too many columns in group by clause";
const char* msg2 = "invalid column name in group by clause";
- const char* msg3 = "group by columns must belong to one table";
+// const char* msg3 = "group by columns must belong to one table";
const char* msg7 = "not support group by expression";
const char* msg8 = "not allowed column type for group by";
const char* msg9 = "tags not allowed for table query";
@@ -2561,10 +2521,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
return invalidSqlErrMsg(pQueryInfo->msg, msg2);
}
- if (tableIndex != index.tableIndex && tableIndex >= 0) {
- return invalidSqlErrMsg(pQueryInfo->msg, msg3);
- }
-
tableIndex = index.tableIndex;
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
@@ -2621,7 +2577,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
}
pQueryInfo->groupbyExpr.tableIndex = tableIndex;
-
return TSDB_CODE_SUCCESS;
}
@@ -2677,7 +2632,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn
tVariantDump(&pRight->val, (char*)&pColumnFilter->upperBndd, colType, false);
} else { // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd
if (colType == TSDB_DATA_TYPE_BINARY) {
- pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + 1);
+ pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + TSDB_NCHAR_SIZE);
pColumnFilter->len = pRight->val.nLen;
tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false);
@@ -2687,7 +2642,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn
tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false);
- size_t len = wcslen((wchar_t*)pColumnFilter->pz);
+ size_t len = twcslen((wchar_t*)pColumnFilter->pz);
pColumnFilter->len = len * TSDB_NCHAR_SIZE;
} else {
tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType, false);
@@ -3051,14 +3006,17 @@ static int32_t getColumnQueryCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, i
}
static int32_t getJoinCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) {
- const char* msg = "invalid join query condition";
+ const char* msg1 = "invalid join query condition";
+ const char* msg2 = "join on binary/nchar not supported";
+ const char* msg3 = "type of join columns must be identical";
+ const char* msg4 = "invalid column name in join condition";
if (pExpr == NULL) {
return TSDB_CODE_SUCCESS;
}
if (!isExprDirectParentOfLeaftNode(pExpr)) {
- return invalidSqlErrMsg(pQueryInfo->msg, msg);
+ return invalidSqlErrMsg(pQueryInfo->msg, msg1);
}
STagCond* pTagCond = &pQueryInfo->tagCond;
@@ -3067,28 +3025,36 @@ static int32_t getJoinCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) {
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (getColumnIndexByName(&pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_TSC_INVALID_SQL;
+ return invalidSqlErrMsg(pQueryInfo->msg, msg4);
}
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
- int16_t tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+ SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
pLeft->uid = pTableMetaInfo->pTableMeta->uid;
- pLeft->tagCol = tagColIndex;
+ pLeft->tagColId = pTagSchema1->colId;
strcpy(pLeft->tableId, pTableMetaInfo->name);
index = (SColumnIndex)COLUMN_INDEX_INITIALIZER;
if (getColumnIndexByName(&pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_TSC_INVALID_SQL;
+ return invalidSqlErrMsg(pQueryInfo->msg, msg4);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
- tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+ SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
pRight->uid = pTableMetaInfo->pTableMeta->uid;
- pRight->tagCol = tagColIndex;
+ pRight->tagColId = pTagSchema2->colId;
strcpy(pRight->tableId, pTableMetaInfo->name);
+ if (pTagSchema1->type != pTagSchema2->type) {
+ return invalidSqlErrMsg(pQueryInfo->msg, msg3);
+ }
+
+ if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) {
+ return invalidSqlErrMsg(pQueryInfo->msg, msg2);
+ }
+
pTagCond->joinInfo.hasJoin = true;
return TSDB_CODE_SUCCESS;
}
@@ -3816,6 +3782,10 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i);
+ if (p1 == NULL) { // no query condition on this table
+ continue;
+ }
+
tExprNode* p = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
@@ -4765,7 +4735,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
pQueryInfo->slimit = pQuerySql->slimit;
- tscTrace("%p limit:%d, offset:%" PRId64 " slimit:%d, soffset:%" PRId64, pSql, pQueryInfo->limit.limit,
+ tscTrace("%p limit:%" PRId64 ", offset:%" PRId64 " slimit:%" PRId64 ", soffset:%" PRId64, pSql, pQueryInfo->limit.limit,
pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset);
if (pQueryInfo->slimit.offset < 0 || pQueryInfo->limit.offset < 0) {
@@ -4949,25 +4919,25 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
return TSDB_CODE_SUCCESS;
}
-void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) {
- // the first column not timestamp column, add it
- SSqlExpr* pExpr = NULL;
- if (tscSqlExprNumOfExprs(pQueryInfo) > 0) {
- pExpr = tscSqlExprGet(pQueryInfo, 0);
- }
-
- if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) {
- SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-
- pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false);
- pExpr->colInfo.flag = TSDB_COL_NORMAL;
-
- // NOTE: tag column does not add to source column list
- SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX);
-
- insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr);
- }
-}
+//void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) {
+// // the first column not timestamp column, add it
+// SSqlExpr* pExpr = NULL;
+// if (tscSqlExprNumOfExprs(pQueryInfo) > 0) {
+// pExpr = tscSqlExprGet(pQueryInfo, 0);
+// }
+//
+// if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) {
+// SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
+//
+// pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false);
+// pExpr->colInfo.flag = TSDB_COL_NORMAL;
+//
+// // NOTE: tag column does not add to source column list
+// SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX);
+//
+// insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr);
+// }
+//}
void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) {
SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentObj->cmd, subClauseIndex);
@@ -4980,7 +4950,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
if (pExpr->functionId != TSDB_FUNC_TAG) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
- int16_t columnInfo = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+ int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo};
SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
@@ -5016,27 +4986,17 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) {
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) {
SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex);
- int32_t index = pColIndex->colIndex;
-
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
- SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index);
- SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index};
-
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
- SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes,
- pSchema->bytes, false);
- pExpr->colInfo.flag = TSDB_COL_NORMAL;
- doLimitOutputNormalColOfGroupby(pExpr);
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex);
+ SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex};
- // NOTE: tag column does not add to source column list
- SColumnList list = {0};
- list.num = 1;
- list.ids[0] = colIndex;
+ tscAddSpecialColumnForSelect(pQueryInfo, size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
- insertResultField(pQueryInfo, size, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr);
- SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1);
+ SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size);
+ doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr);
pInfo->visible = false;
}
@@ -5248,6 +5208,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ SSchema s = tGetTableNameColumnSchema();
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
int16_t bytes = 0;
int16_t type = 0;
@@ -5255,10 +5216,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i);
-
int16_t colIndex = pColIndex->colIndex;
if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
- SSchema s = tGetTableNameColumnSchema();
type = s.type;
bytes = s.bytes;
name = s.name;
@@ -5796,6 +5755,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
const char* msg7 = "illegal number of tables in from clause";
const char* msg8 = "too many columns in selection clause";
const char* msg9 = "TWA query requires both the start and end time";
+ const char* msg10= "too many tables in from clause";
int32_t code = TSDB_CODE_SUCCESS;
@@ -5831,6 +5791,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pQueryInfo->command = TSDB_SQL_SELECT;
+ if (pQuerySql->from->nExpr > 2) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
+ }
+
// set all query tables, which are maybe more than one.
for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) {
tVariant* pTableItem = &pQuerySql->from->a[i].pVar;
@@ -5955,10 +5919,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
setColumnOffsetValueInResultset(pQueryInfo);
- for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
- updateTagColumnIndex(pQueryInfo, i);
- }
-
/*
* fill options are set at the end position, when all columns are set properly
* the columns may be increased due to group by operation
diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c
index 439aa7c1dedc1f9b301d3aeb08ccb7d68ac9ab00..da06e3e5e22c7ead926339a36e79aba4317ba757 100644
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@@ -32,7 +32,6 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) {
}
if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) {
- assert(tinfo.numOfTags >= 0);
return tinfo.numOfTags;
}
@@ -64,14 +63,6 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
-
-#if 0
- if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
- assert (pTableMeta->pSTable != NULL);
- return pTableMeta->pSTable->tableInfo;
- }
-#endif
-
return pTableMeta->tableInfo;
}
@@ -119,11 +110,24 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
return (rowLen <= TSDB_MAX_BYTES_PER_ROW);
}
-SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) {
+SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
assert(pTableMeta != NULL);
SSchema* pSchema = (SSchema*) pTableMeta->schema;
- return &pSchema[startCol];
+ return &pSchema[colIndex];
+}
+
+// TODO for large number of columns, employ the binary search method
+SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+ STableComInfo tinfo = tscGetTableInfo(pTableMeta);
+
+ for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
+ if (pTableMeta->schema[i].colId == colId) {
+ return &pTableMeta->schema[i];
+ }
+ }
+
+ return NULL;
}
struct SSchema tscGetTbnameColumnSchema() {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 09a3f6767b2fdd6a48be26db428d396984d13661..b2e0c0107ea89a800550839055bb0b338fbf8160 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -14,20 +14,18 @@
*/
#include "os.h"
+#include "qsqltype.h"
#include "tcache.h"
#include "trpc.h"
+#include "tscLocalMerge.h"
+#include "tscLog.h"
#include "tscProfile.h"
-#include "tscSecondaryMerge.h"
-#include "tscSubquery.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "tsclient.h"
-#include "tsocket.h"
#include "ttime.h"
#include "ttimer.h"
#include "tutil.h"
-#include "tscLog.h"
-#include "qsqltype.h"
#define TSC_MGMT_VNODE 999
@@ -198,30 +196,35 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.handle = pSql,
.code = 0
};
- rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+ pSql->pRpcCtx = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
- if (pSql == NULL) {
+ if (pSql == NULL || pSql->signature != pSql) {
tscError("%p sql is already released", pSql->signature);
return;
}
- if (pSql->signature != pSql) {
- tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
- return;
- }
-
+ STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
- STscObj *pObj = pSql->pTscObj;
- if (pObj->signature != pObj || pSql->freed == 1) {
- tscTrace("%p sqlObj needs to be released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
+ if (pObj->signature != pObj) {
+ tscTrace("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
+
+ tscFreeSqlObj(pSql);
+ rpcFreeCont(rpcMsg->pCont);
+ return;
+ }
+
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+ if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
+ tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command,
pObj, pObj->signature);
+
tscFreeSqlObj(pSql);
rpcFreeCont(rpcMsg->pCont);
return;
@@ -274,7 +277,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) {
pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL;
} else {
- tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code));
+ tscTrace("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code));
}
if (pRes->code == TSDB_CODE_SUCCESS) {
@@ -414,7 +417,6 @@ void tscKillSTableQuery(SSqlObj *pSql) {
for (int i = 0; i < pSql->numOfSubs; ++i) {
SSqlObj *pSub = pSql->pSubs[i];
-
if (pSub == NULL) {
continue;
}
@@ -424,7 +426,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
* sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
*/
pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-// taosStopRpcConn(pSql->pSubs[i]->);
+ rpcCancelRequest(pSql->pSubs[i]->pRpcCtx);
}
/*
@@ -644,14 +646,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
- pQueryMsg->queryType = htons(pQueryInfo->type);
+ pQueryMsg->queryType = htonl(pQueryInfo->type);
size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
pQueryMsg->numOfOutput = htons(numOfOutput);
- if (numOfOutput < 0) {
- tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput);
- return -1;
- }
// set column list ids
size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
@@ -665,7 +663,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
- pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex,
+ pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
pColSchema->name);
return TSDB_CODE_TSC_INVALID_SQL;
@@ -723,6 +721,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += sizeof(SSqlFuncMsg);
for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
+ // todo add log
pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
pSqlFuncExpr->arg[j].argBytes = htons(pExpr->param[j].nLen);
@@ -784,7 +783,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
(pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) {
tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns,
- pCol->colIndex, pColSchema->name);
+ pCol->colIndex.columnIndex, pColSchema->name);
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -800,6 +799,27 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
}
+ // serialize tag column query condition
+ if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
+ STagCond* pTagCond = &pQueryInfo->tagCond;
+
+ SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
+ if (pCond != NULL && pCond->cond != NULL) {
+ pQueryMsg->tagCondLen = htons(pCond->len);
+ memcpy(pMsg, pCond->cond, pCond->len);
+
+ pMsg += pCond->len;
+ }
+ }
+
+ if (pQueryInfo->tagCond.tbnameCond.cond == NULL) {
+ *pMsg = 0;
+ pMsg++;
+ } else {
+ strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond);
+ pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1;
+ }
+
// compressed ts block
pQueryMsg->tsOffset = htonl(pMsg - pStart);
int32_t tsLen = 0;
@@ -824,27 +844,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
}
- // serialize tag column query condition
- if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
- STagCond* pTagCond = &pQueryInfo->tagCond;
-
- SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
- if (pCond != NULL && pCond->cond != NULL) {
- pQueryMsg->tagCondLen = htons(pCond->len);
- memcpy(pMsg, pCond->cond, pCond->len);
-
- pMsg += pCond->len;
- }
- }
-
- if (pQueryInfo->tagCond.tbnameCond.cond == NULL) {
- *pMsg = 0;
- pMsg++;
- } else {
- strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond);
- pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1;
- }
-
int32_t msgLen = pMsg - pStart;
tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
@@ -983,7 +982,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
- strncpy(pDropDbMsg->db, pTableMetaInfo->name, tListLen(pDropDbMsg->db));
+ tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
pDropDbMsg->ignoreNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0;
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB;
@@ -1053,7 +1052,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
- strcpy(pDropMsg->user, pTableMetaInfo->name);
+ tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
return TSDB_CODE_SUCCESS;
}
@@ -1175,7 +1174,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name);
// use dbinfo from table id without modifying current db info
- tscGetDBInfoFromMeterId(pTableMetaInfo->name, pCreateTableMsg->db);
+ tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateTableMsg->db);
SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo;
@@ -1252,7 +1251,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
- tscGetDBInfoFromMeterId(pTableMetaInfo->name, pAlterTableMsg->db);
+ tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);
strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
pAlterTableMsg->type = htons(pAlterInfo->type);
@@ -1473,7 +1472,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += len;
}
- pCmd->payloadLen = pMsg - (char*)pInfoMsg;;
+ pCmd->payloadLen = pMsg - (char*)pInfoMsg;
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META;
tfree(tmpData);
@@ -1577,7 +1576,7 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg = pStart;
SMgmtHead *pMgmt = (SMgmtHead *)pMsg;
- tscGetDBInfoFromMeterId(pTableMetaInfo->name, pMgmt->db);
+ tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pMgmt->db);
pMsg += sizeof(SMgmtHead);
@@ -1813,6 +1812,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
// todo handle out of memory case
if (pTableMetaInfo->pTableMeta == NULL) {
+ free(pTableMeta);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@@ -1932,113 +1932,6 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
}
int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
-#if 0
- void ** metricMetaList = NULL;
- int32_t * sizes = NULL;
-
- int32_t num = htons(*(int16_t *)rsp);
- rsp += sizeof(int16_t);
-
- metricMetaList = calloc(1, POINTER_BYTES * num);
- sizes = calloc(1, sizeof(int32_t) * num);
-
- // return with error code
- if (metricMetaList == NULL || sizes == NULL) {
- tfree(metricMetaList);
- tfree(sizes);
- pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-
- return pSql->res.code;
- }
-
- for (int32_t k = 0; k < num; ++k) {
- pMeta = (SSuperTableMeta *)rsp;
-
- size_t size = (size_t)pSql->res.rspLen - 1;
- rsp = rsp + sizeof(SSuperTableMeta);
-
- pMeta->numOfTables = htonl(pMeta->numOfTables);
- pMeta->numOfVnodes = htonl(pMeta->numOfVnodes);
- pMeta->tagLen = htons(pMeta->tagLen);
-
- size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfTables * sizeof(STableIdInfo *);
-
- char *pBuf = calloc(1, size);
- if (pBuf == NULL) {
- pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- goto _error_clean;
- }
-
- SSuperTableMeta *pNewMetricMeta = (SSuperTableMeta *)pBuf;
- metricMetaList[k] = pNewMetricMeta;
-
- pNewMetricMeta->numOfTables = pMeta->numOfTables;
- pNewMetricMeta->numOfVnodes = pMeta->numOfVnodes;
- pNewMetricMeta->tagLen = pMeta->tagLen;
-
- pBuf = pBuf + sizeof(SSuperTableMeta) + pNewMetricMeta->numOfVnodes * sizeof(SVnodeSidList *);
-
- for (int32_t i = 0; i < pMeta->numOfVnodes; ++i) {
- SVnodeSidList *pSidLists = (SVnodeSidList *)rsp;
- memcpy(pBuf, pSidLists, sizeof(SVnodeSidList));
-
- pNewMetricMeta->list[i] = pBuf - (char *)pNewMetricMeta; // offset value
- SVnodeSidList *pLists = (SVnodeSidList *)pBuf;
-
- tscTrace("%p metricmeta:vid:%d,numOfTables:%d", pSql, i, pLists->numOfSids);
-
- pBuf += sizeof(SVnodeSidList) + sizeof(STableIdInfo *) * pSidLists->numOfSids;
- rsp += sizeof(SVnodeSidList);
-
- size_t elemSize = sizeof(STableIdInfo) + pNewMetricMeta->tagLen;
- for (int32_t j = 0; j < pSidLists->numOfSids; ++j) {
- pLists->pSidExtInfoList[j] = pBuf - (char *)pLists;
- memcpy(pBuf, rsp, elemSize);
-
- ((STableIdInfo *)pBuf)->uid = htobe64(((STableIdInfo *)pBuf)->uid);
- ((STableIdInfo *)pBuf)->sid = htonl(((STableIdInfo *)pBuf)->sid);
-
- rsp += elemSize;
- pBuf += elemSize;
- }
- }
-
- sizes[k] = pBuf - (char *)pNewMetricMeta;
- }
-
- SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
- for (int32_t i = 0; i < num; ++i) {
- char name[TSDB_MAX_TAGS_LEN + 1] = {0};
-
- STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
- tscGetMetricMetaCacheKey(pQueryInfo, name, pTableMetaInfo->pTableMeta->uid);
-
-#ifdef _DEBUG_VIEW
- printf("generate the metric key:%s, index:%d\n", name, i);
-#endif
-
- // release the used metricmeta
- taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
- pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i],
- sizes[i], tsMetricMetaKeepTimer);
- tfree(metricMetaList[i]);
-
- // failed to put into cache
- if (pTableMetaInfo->pMetricMeta == NULL) {
- pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- goto _error_clean;
- }
- }
-
-_error_clean:
- // free allocated resource
- for (int32_t i = 0; i < num; ++i) {
- tfree(metricMetaList[i]);
- }
-
- free(sizes);
- free(metricMetaList);
-#endif
SSqlRes* pRes = &pSql->res;
// NOTE: the order of several table must be preserved.
@@ -2291,7 +2184,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
}
pRes->row = 0;
- tscTrace("%p numOfRows:%d, offset:%d, complete:%d", pSql, pRes->numOfRows, pRes->offset, pRes->completed);
+ tscTrace("%p numOfRows:%" PRId64 ", offset:%" PRId64 ", complete:%d", pSql, pRes->numOfRows, pRes->offset, pRes->completed);
return 0;
}
@@ -2432,6 +2325,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
SQueryInfo *pNewQueryInfo = NULL;
if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) {
+ tscFreeSqlObj(pNew);
return code;
}
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 6f043f186a9673840d6deffd57639bc76a74ff67..51b49304a8e72adb980ccbcf9368f68051f02745 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -20,14 +20,9 @@
#include "tnote.h"
#include "trpc.h"
#include "tscLog.h"
-#include "tscProfile.h"
-#include "tscSecondaryMerge.h"
#include "tscSubquery.h"
#include "tscUtil.h"
#include "tsclient.h"
-#include "tscompression.h"
-#include "tsocket.h"
-#include "ttimer.h"
#include "ttokendef.h"
#include "tutil.h"
@@ -152,6 +147,8 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db);
+ if (user == NULL) user = TSDB_DEFAULT_USER;
+ if (pass == NULL) pass = TSDB_DEFAULT_PASS;
STscObj* pObj = NULL;
SSqlObj *pSql = taosConnectImpl(ip, user, pass, db, port, syncConnCallback, NULL, (void**) &pObj);
@@ -491,8 +488,7 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
(pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
- tscTrace("%p start to send msg to free qhandle in dnode, command:%s", pSql, sqlCmd[pCmd->command]);
- pSql->freed = 1;
+ tscTrace("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]);
tscProcessSql(pSql);
// in case of sync model query, waits for response and then goes on
@@ -513,18 +509,17 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
void taos_free_result(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res;
- tscTrace("%p start to free result", res);
-
+
if (pSql == NULL || pSql->signature != pSql) {
- tscTrace("%p result has been freed", pSql);
+ tscTrace("%p sqlObj has been freed", pSql);
return;
}
// The semaphore can not be changed while freeing async sub query objects.
SSqlRes *pRes = &pSql->res;
if (pRes == NULL || pRes->qhandle == 0) {
- tscTrace("%p SqlObj is freed by app, qhandle is null", pSql);
tscFreeSqlObj(pSql);
+ tscTrace("%p SqlObj is freed by app, qhandle is null", pSql);
return;
}
@@ -532,6 +527,7 @@ void taos_free_result(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pQueryInfo == NULL) {
tscFreeSqlObj(pSql);
+ tscTrace("%p SqlObj is freed by app", pSql);
return;
}
@@ -589,7 +585,7 @@ char *taos_errstr(TAOS_RES *tres) {
void taos_config(int debug, char *log_path) {
uDebugFlag = debug;
- strcpy(tsLogDir, log_path);
+ tstrncpy(tsLogDir, log_path, TSDB_FILENAME_LEN);
}
char *taos_get_server_info(TAOS *taos) {
@@ -612,7 +608,9 @@ int* taos_fetch_lengths(TAOS_RES *res) {
char *taos_get_client_info() { return version; }
void taos_stop_query(TAOS_RES *res) {
- if (res == NULL) return;
+ if (res == NULL) {
+ return;
+ }
SSqlObj *pSql = (SSqlObj *)res;
SSqlCmd *pCmd = &pSql->cmd;
@@ -632,7 +630,7 @@ void taos_stop_query(TAOS_RES *res) {
return;
}
- //taosStopRpcConn(pSql->thandle);
+ rpcCancelRequest(pSql->pRpcCtx);
tscTrace("%p query is cancelled", res);
}
@@ -714,7 +712,6 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
- pRes->numOfRows = 1;
pRes->numOfTotal = 0;
pRes->numOfClauseTotal = 0;
@@ -724,6 +721,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
if (sqlLen > tsMaxSQLStringLen) {
tscError("%p sql too long", pSql);
pRes->code = TSDB_CODE_TSC_INVALID_SQL;
+ tfree(pSql);
return pRes->code;
}
@@ -732,6 +730,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql);
tscTrace("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+ tfree(pSql);
return pRes->code;
}
@@ -856,6 +855,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
if (tblListLen > MAX_TABLE_NAME_LENGTH) {
tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH);
pRes->code = TSDB_CODE_TSC_INVALID_SQL;
+ tfree(pSql);
return pRes->code;
}
@@ -863,6 +863,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
if (str == NULL) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql);
+ tfree(pSql);
return pRes->code;
}
@@ -878,6 +879,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
free(str);
if (pRes->code != TSDB_CODE_SUCCESS) {
+ tscFreeSqlObj(pSql);
return pRes->code;
}
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index c4413f85414f67d8db3cb5422acc5e34e5eb87c3..86a41b7ba46d22b193304792ddf14cc131f505b9 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -125,7 +125,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
}
// launch stream computing in a new thread
- SSchedMsg schedMsg;
+ SSchedMsg schedMsg = { 0 };
schedMsg.fp = tscProcessStreamLaunchQuery;
schedMsg.ahandle = pStream;
schedMsg.thandle = (void *)1;
@@ -239,7 +239,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
/* no resuls in the query range, retry */
// todo set retry dynamic time
int32_t retry = tsProjectExecInterval;
- tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retry);
+ tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId32 "ms", pSql, pStream, numOfRows, retry);
tscSetRetryTimer(pStream, pStream->pSql, retry);
return;
@@ -250,7 +250,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
}
}
- tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d", pSql, pStream, pTableMetaInfo->name,
+ tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name,
pStream->numOfRes);
// release the metric/meter meta information reference, so data in cache can be updated
@@ -499,12 +499,14 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
if (pSql->sqlstr == NULL) {
tscError("%p failed to malloc sql string buffer", pSql);
tscFreeSqlObj(pSql);
- return NULL;;
+ return NULL;
}
strtolower(pSql->sqlstr, sqlstr);
+ tscDump("%p SQL: %s", pSql, pSql->sqlstr);
tsem_init(&pSql->rspSem, 0, 0);
- int32_t code = doAsyncParseSql(pSql);
+
+ int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
sem_wait(&pSql->rspSem);
}
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 1a1305825bb32cd3a300e990a33cca52ec7633b9..30a0de76fc690bd12b34f3d6b76acc5903503867 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -182,21 +182,25 @@ static SArray* getTableList( SSqlObj* pSql ) {
char* sql = alloca(strlen(p) + 32);
sprintf(sql, "select tbid(tbname)%s", p);
- SSqlObj* pSql1 = taos_query(pSql->pTscObj, sql);
- if (terrno != TSDB_CODE_SUCCESS) {
- tscError("failed to retrieve table id: %s", tstrerror(terrno));
+ SSqlObj* pNew = taos_query(pSql->pTscObj, sql);
+ if (pNew == NULL) {
+ tscError("failed to retrieve table id: cannot create new sql object.");
+ return NULL;
+
+ } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) {
+ tscError("failed to retrieve table id: %s", tstrerror(taos_errno(pNew)));
return NULL;
}
TAOS_ROW row;
SArray* result = taosArrayInit( 128, sizeof(STidTags) );
- while ((row = taos_fetch_row(pSql1))) {
+ while ((row = taos_fetch_row(pNew))) {
STidTags tags;
memcpy(&tags, row[0], sizeof(tags));
taosArrayPush(result, &tags);
}
- taos_free_result(pSql1);
+ taos_free_result(pNew);
return result;
}
@@ -222,6 +226,9 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
}
SArray* tables = getTableList(pSql);
+ if (tables == NULL) {
+ return 0;
+ }
size_t numOfTables = taosArrayGetSize(tables);
SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
@@ -238,10 +245,11 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
taosArraySort( tables, tscCompareTidTags );
- tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+ tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables);
}
taosArrayDestroy(tables);
+ TSDB_QUERY_SET_TYPE(tscGetQueryInfoDetail(pCmd, 0)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
return 1;
}
@@ -291,7 +299,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
fclose(fp);
taosArraySort(progress, tscCompareSubscriptionProgress);
- tscTrace("subscription progress loaded, %d tables: %s", taosArrayGetSize(progress), pSub->topic);
+ tscTrace("subscription progress loaded, %zu tables: %s", taosArrayGetSize(progress), pSub->topic);
return 1;
}
@@ -350,7 +358,7 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
pSub->interval = interval;
if (fp != NULL) {
- tscTrace("asynchronize subscription, create new timer", topic);
+ tscTrace("asynchronize subscription, create new timer: %s", topic);
pSub->fp = fp;
pSub->param = param;
taosTmrReset(tscProcessSubscriptionTimer, interval, pSub, tscTmr, &pSub->pTimer);
@@ -413,7 +421,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
}
if (pRes->code != TSDB_CODE_SUCCESS) {
- tscError("failed to query data, error code=%d", pRes->code);
+ tscError("failed to query data: %s", tstrerror(pRes->code));
tscRemoveFromSqlList(pSql);
return NULL;
}
@@ -435,7 +443,9 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
} else {
char path[256];
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
- remove(path);
+ if (remove(path) != 0) {
+ tscError("failed to remove progress file, topic = %s, error = %s", pSub->topic, strerror(errno));
+ }
}
tscFreeSqlObj(pSub->pSql);
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 3bc931a855876fff6e7630f6324d0e1a7261bbb8..87c75bd7e684ae942a4d7b49dd33efedf0c0d388 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -12,12 +12,12 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+#include "os.h"
#include "tscSubquery.h"
-#include
-#include
-#include
-#include "os.h"
+#include "qast.h"
+#include "tcompare.h"
+#include "tschemautil.h"
#include "qtsbuf.h"
#include "tscLog.h"
#include "tsclient.h"
@@ -28,7 +28,7 @@ typedef struct SInsertSupporter {
} SInsertSupporter;
static void freeJoinSubqueryObj(SSqlObj* pSql);
-static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql);
+static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
static bool tsCompare(int32_t order, int64_t left, int64_t right) {
if (order == TSDB_ORDER_ASC) {
@@ -38,16 +38,15 @@ static bool tsCompare(int32_t order, int64_t left, int64_t right) {
}
}
-static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1,
- SJoinSupporter* pSupporter2, TSKEY* st, TSKEY* et) {
- STSBuf* output1 = tsBufCreate(true);
- STSBuf* output2 = tsBufCreate(true);
+static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJoinSupporter* pSupporter2, STimeWindow * win) {
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
- *st = INT64_MAX;
- *et = INT64_MIN;
+ STSBuf* output1 = tsBufCreate(true, pQueryInfo->order.order);
+ STSBuf* output2 = tsBufCreate(true, pQueryInfo->order.order);
+
+ win->skey = INT64_MAX;
+ win->ekey = INT64_MIN;
- SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
-
SLimitVal* pLimit = &pQueryInfo->limit;
int32_t order = pQueryInfo->order.order;
@@ -106,12 +105,12 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1,
* final results which is acquired after the secondry merge of in the client.
*/
if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
- if (*st > elem1.ts) {
- *st = elem1.ts;
+ if (win->skey > elem1.ts) {
+ win->skey = elem1.ts;
}
- if (*et < elem1.ts) {
- *et = elem1.ts;
+ if (win->ekey < elem1.ts) {
+ win->ekey = elem1.ts;
}
tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
@@ -151,8 +150,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1,
tsBufDestory(pSupporter2->pTSBuf);
tscTrace("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks "
- "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql,
- numOfInput1, numOfInput2, output1->numOfTotal, *st, *et);
+ "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal,
+ win->skey, win->ekey);
return output1->numOfTotal;
}
@@ -170,7 +169,8 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
pSupporter->subqueryIndex = index;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
- pSupporter->interval = pQueryInfo->intervalTime;
+ pSupporter->intervalTime = pQueryInfo->intervalTime;
+ pSupporter->slidingTime = pQueryInfo->slidingTime;
pSupporter->limit = pQueryInfo->limit;
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index);
@@ -187,7 +187,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
return pSupporter;
}
-void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
+static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
if (pSupporter == NULL) {
return;
}
@@ -218,7 +218,7 @@ void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
* primary timestamp column , the secondary query is not necessary
*
*/
-bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
+static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
for (int32_t i = 0; i < numOfCols; ++i) {
@@ -234,14 +234,11 @@ bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
/*
* launch secondary stage query to fetch the result that contains timestamp in set
*/
-int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
+static int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
int32_t numOfSub = 0;
SJoinSupporter* pSupporter = NULL;
- /*
- * If the columns are not involved in the final select clause,
- * the corresponding query will not be issued.
- */
+ //If the columns are not involved in the final select clause, the corresponding query will not be issued.
for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
pSupporter = pSql->pSubs[i]->param;
if (taosArrayGetSize(pSupporter->exprList) > 0) {
@@ -252,13 +249,12 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
assert(numOfSub > 0);
// scan all subquery, if one sub query has only ts, ignore it
- tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query, others are not retrieve in "
- "select clause", pSql, pSql->numOfSubs, numOfSub);
+ tscTrace("%p start to launch secondary subquery, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
//the subqueries that do not actually launch the secondary query to virtual node is set as completed.
SSubqueryState* pState = pSupporter->pState;
pState->numOfTotal = pSql->numOfSubs;
- pState->numOfCompleted = (pSql->numOfSubs - numOfSub);
+ pState->numOfRemain = numOfSub;
bool success = true;
@@ -301,8 +297,9 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
// set the second stage sub query for join process
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE);
-
- pQueryInfo->intervalTime = pSupporter->interval;
+
+ pQueryInfo->intervalTime = pSupporter->intervalTime;
+ pQueryInfo->slidingTime = pSupporter->slidingTime;
pQueryInfo->groupbyExpr = pSupporter->groupbyExpr;
tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
@@ -328,22 +325,38 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
*/
pSupporter->limit = pQueryInfo->limit;
pNewQueryInfo->limit = pSupporter->limit;
-
- // fetch the join tag column
+
+ SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+ SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0);
+
+ SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0);
+ int16_t funcId = pExpr->functionId;
+
+ if ((pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) ||
+ (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) {
+
+ int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS;
+
+ tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
+ tscPrintSelectClause(pNew, 0);
+ tscFieldInfoUpdateOffset(pNewQueryInfo);
+
+ pExpr = tscSqlExprGet(pQueryInfo, 0);
+ }
+
+ // set the join condition tag column info, to do extract method
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
- SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, 0);
assert(pQueryInfo->tagCond.joinInfo.hasJoin);
-
- int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
- pExpr->param[0].i64Key = tagColIndex;
+ int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+
+ pExpr->param[0].i64Key = colId;
pExpr->numOfParams = 1;
}
-
+
size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
- tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
- pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type,
- taosArrayGetSize(pNewQueryInfo->exprList), numOfCols,
- pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name);
+ tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%zu, colList:%zu, fieldsInfo:%d, name:%s",
+ pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList),
+ numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
}
//prepare the subqueries object failed, abort
@@ -357,12 +370,11 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
}
for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
- SSqlObj* pSub = pSql->pSubs[i];
- if (pSub == NULL) {
+ if (pSql->pSubs[i] == NULL) {
continue;
}
-
- tscProcessSql(pSub);
+
+ tscDoQuery(pSql->pSubs[i]);
}
return TSDB_CODE_SUCCESS;
@@ -392,25 +404,21 @@ void freeJoinSubqueryObj(SSqlObj* pSql) {
}
static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
- int32_t numOfTotal = pSupporter->pState->numOfTotal;
- int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1);
-
- pSqlObj->res.code = pSupporter->pState->code;
- if (finished >= numOfTotal) {
+ assert(pSupporter->pState->numOfRemain > 0);
+
+ if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) <= 0) {
tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
freeJoinSubqueryObj(pSqlObj);
}
}
// update the query time range according to the join results on timestamp
-static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) {
- assert(pQueryInfo->window.skey <= st && pQueryInfo->window.ekey >= et);
-
- pQueryInfo->window.skey = st;
- pQueryInfo->window.ekey = et;
+static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
+ assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey);
+ pQueryInfo->window = *win;
}
-static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) {
+static UNUSED_FUNC void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) {
SSqlObj* pParentSql = pSupporter->pObj;
SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
@@ -432,40 +440,24 @@ static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj*
// return;
// }
// }
-
- int32_t numOfTotal = pSupporter->pState->numOfTotal;
- int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1);
-
- if (finished >= numOfTotal) {
- assert(finished == numOfTotal);
-
- if (pSupporter->pState->code != TSDB_CODE_SUCCESS) {
- tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, pSql,
- pSupporter->subqueryIndex);
- freeJoinSubqueryObj(pParentSql);
- return;
- }
-
- tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql);
-
+
SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
- TSKEY st, et;
- int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et);
+ STimeWindow win = TSWINDOW_INITIALIZER;
+ int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &win);
if (num <= 0) { // no result during ts intersect
tscTrace("%p free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql);
} else {
- updateQueryTimeRange(pParentQueryInfo, st, et);
+ updateQueryTimeRange(pParentQueryInfo, &win);
tscLaunchSecondPhaseSubqueries(pParentSql);
}
- }
}
int32_t tscCompareTidTags(const void* p1, const void* p2) {
- const STidTags* t1 = (const STidTags*) p1;
- const STidTags* t2 = (const STidTags*) p2;
+ const STidTags* t1 = (const STidTags*) varDataVal(p1);
+ const STidTags* t2 = (const STidTags*) varDataVal(p2);
if (t1->vgId != t2->vgId) {
return (t1->vgId > t2->vgId) ? 1 : -1;
@@ -476,38 +468,40 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) {
return 0;
}
-void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) {
- SArray* result = taosArrayInit( 4, sizeof(SVgroupTableInfo) );
- SArray* vgTables = NULL;
+void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
+ SArray* result = taosArrayInit(4, sizeof(SVgroupTableInfo));
+ SArray* vgTables = NULL;
STidTags* prev = NULL;
- size_t numOfTables = taosArrayGetSize( tables );
- for( size_t i = 0; i < numOfTables; i++ ) {
- STidTags* tt = taosArrayGet( tables, i );
+ size_t numOfTables = taosArrayGetSize(tables);
+ for (size_t i = 0; i < numOfTables; i++) {
+ STidTags* tt = taosArrayGet(tables, i);
- if( prev == NULL || tt->vgId != prev->vgId ) {
+ if (prev == NULL || tt->vgId != prev->vgId) {
SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
- SVgroupTableInfo info = {{ 0 }};
- for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) {
- if( tt->vgId == pvg->vgroups[m].vgId ) {
+ SVgroupTableInfo info = {{0}};
+ for (int32_t m = 0; m < pvg->numOfVgroups; ++m) {
+ if (tt->vgId == pvg->vgroups[m].vgId) {
info.vgInfo = pvg->vgroups[m];
break;
}
}
- assert( info.vgInfo.numOfIps != 0 );
+ assert(info.vgInfo.numOfIps != 0);
- vgTables = taosArrayInit( 4, sizeof(STableIdInfo) );
+ vgTables = taosArrayInit(4, sizeof(STableIdInfo));
info.itemList = vgTables;
- taosArrayPush( result, &info );
+ taosArrayPush(result, &info);
}
- STableIdInfo item = { .uid = tt->uid, .tid = tt->tid, .key = INT64_MIN };
- taosArrayPush( vgTables, &item );
+ tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId)
+ STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN};
+ taosArrayPush(vgTables, &item);
prev = tt;
}
pTableMetaInfo->pVgroupTables = result;
+ pTableMetaInfo->vgroupIndex = 0;
}
static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@@ -532,12 +526,13 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL);
// set the tags value for ts_comp function
- SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0);
- int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
-
- pExpr->param->i64Key = tagColIndex;
- pExpr->numOfParams = 1;
-
+ if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+ SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0);
+ int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+ pExpr->param->i64Key = tagColId;
+ pExpr->numOfParams = 1;
+ }
+
// add the filter tag column
if (pSupporter->colList != NULL) {
size_t s = taosArrayGetSize(pSupporter->colList);
@@ -555,151 +550,233 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
tscTrace(
- "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to ts_comp query to retrieve timestamps, "
- "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
- pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, tscSqlExprNumOfExprs(pQueryInfo),
- numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
+ "%p subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, "
+ "numOfExpr:%zu, colList:%zu, numOfOutputFields:%d, name:%s",
+ pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type,
+ tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
tscProcessSql(pSql);
}
-static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
+static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, SSqlObj* pPSqlObj) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed
+ SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0);
+ SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex];
+
+ for(int32_t i = 1; i < p1->num; ++i) {
+ STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize);
+ STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize);
+
+ if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) {
+ tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj);
+ pPSqlObj->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
+ tscTrace("%p all subqueries retrieve complete, do tags match", pParentSql);
+
+ SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
+ SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
+
+ qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
+ qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
+
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+
+ SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+
+ *s1 = taosArrayInit(p1->num, p1->tagSize);
+ *s2 = taosArrayInit(p2->num, p2->tagSize);
+
+ if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) {
+ freeJoinSubqueryObj(pParentSql);
+ pParentSql->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY;
+ tscQueueAsyncRes(pParentSql);
+ return;
+ }
+
+ int32_t i = 0, j = 0;
+ while(i < p1->num && j < p2->num) {
+ STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize);
+ STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize);
+
+ int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes);
+ if (ret == 0) {
+ tscTrace("%p tag matched, vgId:%d, val:%d, tid:%d, uid:%"PRIu64", tid:%d, uid:%"PRIu64, pParentSql, pp1->vgId,
+ *(int*) pp1->tag, pp1->tid, pp1->uid, pp2->tid, pp2->uid);
+
+ taosArrayPush(*s1, pp1);
+ taosArrayPush(*s2, pp2);
+ j++;
+ i++;
+ } else if (ret > 0) {
+ j++;
+ } else {
+ i++;
+ }
+ }
+}
+
+static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) {
SJoinSupporter* pSupporter = (SJoinSupporter*)param;
-
+
SSqlObj* pParentSql = pSupporter->pObj;
+
SSqlObj* pSql = (SSqlObj*)tres;
SSqlCmd* pCmd = &pSql->cmd;
-
- SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-
- // response of tag retrieve
- if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
- if (numOfRows == 0 || pSql->res.completed) {
-
- if (numOfRows > 0) {
- size_t length = pSupporter->totalLen + pSql->res.rspLen;
- char* tmp = realloc(pSupporter->pIdTagList, length);
- assert(tmp != NULL);
- pSupporter->pIdTagList = tmp;
-
- memcpy(pSupporter->pIdTagList, pSql->res.data, pSql->res.rspLen);
- pSupporter->totalLen += pSql->res.rspLen;
- pSupporter->num += pSql->res.numOfRows;
- }
-
- int32_t numOfTotal = pSupporter->pState->numOfTotal;
- int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1);
-
- if (finished < numOfTotal) {
- return;
- }
-
- // all subqueries are returned, start to compare the tags
- assert(finished == numOfTotal);
- tscTrace("%p all subqueries retrieve tags complete, do tags match", pParentSql);
-
- SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
- SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
-
- qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
- qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
-
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed
-
- SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0);
- SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex];
-
- SArray* s1 = taosArrayInit(p1->num, p1->tagSize);
- SArray* s2 = taosArrayInit(p2->num, p2->tagSize);
-
- int32_t i = 0, j = 0;
- while(i < p1->num && j < p2->num) {
- STidTags* pp1 = (STidTags*) p1->pIdTagList + i * p1->tagSize;
- STidTags* pp2 = (STidTags*) p2->pIdTagList + j * p2->tagSize;
-
- int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes);
- if (ret == 0) {
- taosArrayPush(s1, pp1);
- taosArrayPush(s2, pp2);
- j++;
- i++;
- } else if (ret > 0) {
- j++;
- } else {
- i++;
- }
- }
-
- if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) {// no results,return.
- tscTrace("%p free all sub SqlObj and quit", pParentSql);
- freeJoinSubqueryObj(pParentSql);
- return;
- } else {
- SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd;
- SSqlCmd* pSubCmd2 = &pParentSql->pSubs[1]->cmd;
-
- SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0);
- STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0);
- tscBuildVgroupTableInfo(pTableMetaInfo1, s1);
-
- SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0);
- STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
- tscBuildVgroupTableInfo(pTableMetaInfo2, s2);
-
- pSupporter->pState->numOfCompleted = 0;
- pSupporter->pState->code = 0;
- pSupporter->pState->numOfTotal = 2;
-
- for(int32_t m = 0; m < pParentSql->numOfSubs; ++m) {
- SSqlObj* psub = pParentSql->pSubs[m];
- issueTSCompQuery(psub, psub->param, pParentSql);
- }
- }
-
- } else {
- size_t length = pSupporter->totalLen + pSql->res.rspLen;
- char* tmp = realloc(pSupporter->pIdTagList, length);
- assert(tmp != NULL);
-
- pSupporter->pIdTagList = tmp;
-
- memcpy(pSupporter->pIdTagList, pSql->res.data, pSql->res.rspLen);
- pSupporter->totalLen += pSql->res.rspLen;
- pSupporter->num += pSql->res.numOfRows;
-
- // continue retrieve data from vnode
- taos_fetch_rows_a(tres, joinRetrieveCallback, param);
- }
-
+ SSqlRes* pRes = &pSql->res;
+
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+ assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY));
+
+ // check for the error code firstly
+ if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
+ // todo retry if other subqueries are not failed
+
+ assert(numOfRows < 0 && numOfRows == taos_errno(pSql));
+ tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
+
+ pParentSql->res.code = numOfRows;
+ quitAllSubquery(pParentSql, pSupporter);
+
+ tscQueueAsyncRes(pParentSql);
return;
}
-
- if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
- if (numOfRows < 0) {
- tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
- pSupporter->pState->code = numOfRows;
+
+ // keep the results in memory
+ if (numOfRows > 0) {
+ size_t validLen = pSupporter->tagSize * pRes->numOfRows;
+ size_t length = pSupporter->totalLen + validLen;
+
+ // todo handle memory error
+ char* tmp = realloc(pSupporter->pIdTagList, length);
+ if (tmp == NULL) {
+ tscError("%p failed to malloc memory", pSql);
+
+ pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
quitAllSubquery(pParentSql, pSupporter);
+
+ tscQueueAsyncRes(pParentSql);
return;
}
- if (numOfRows == 0) {
- tSIntersectionAndLaunchSecQuery(pSupporter, pSql);
+ pSupporter->pIdTagList = tmp;
+
+ memcpy(pSupporter->pIdTagList + pSupporter->totalLen, pRes->data, validLen);
+ pSupporter->totalLen += validLen;
+ pSupporter->num += pRes->numOfRows;
+
+ // query not completed, continue to retrieve tid + tag tuples
+ if (!pRes->completed) {
+ taos_fetch_rows_a(tres, tidTagRetrieveCallback, param);
return;
}
+ }
+
+ // data in current vnode has all returned to client, try next vnode if exits
+ // tuples have been retrieved to client, try tuples from the next vnode
+ if (hasMoreVnodesToTry(pSql)) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+ pTableMetaInfo->vgroupIndex += 1;
+ assert(pTableMetaInfo->vgroupIndex < totalVgroups);
+
+ tscTrace("%p tid_tag from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%d",
+ pSql, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pSupporter->num);
+
+ pCmd->command = TSDB_SQL_SELECT;
+ tscResetForNextRetrieve(&pSql->res);
+
+ // set the callback function
+ pSql->fp = tscJoinQueryCallback;
+ tscProcessSql(pSql);
+ return;
+ }
+
+ // no data exists in next vnode, mark the query completed
+ // only when there is no subquery exits any more, proceeds to get the intersect of the tuple sets.
+ if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+ return;
+ }
+
+ SArray *s1 = NULL, *s2 = NULL;
+ getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2);
+ if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
+ tscTrace("%p free all sub SqlObj and quit", pParentSql);
+ freeJoinSubqueryObj(pParentSql);
+
+ } else {
+ // proceed to for ts_comp query
+ SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd;
+ SSqlCmd* pSubCmd2 = &pParentSql->pSubs[1]->cmd;
+
+ SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0);
+ STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0);
+ tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo1, s1);
+
+ SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0);
+ STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
+ tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);
+
+ pSupporter->pState->numOfTotal = 2;
+ pSupporter->pState->numOfRemain = pSupporter->pState->numOfTotal;
+
+ for (int32_t m = 0; m < pParentSql->numOfSubs; ++m) {
+ SSqlObj* sub = pParentSql->pSubs[m];
+ issueTSCompQuery(sub, sub->param, pParentSql);
+ }
+ }
+
+ taosArrayDestroy(s1);
+ taosArrayDestroy(s2);
+}
+
+static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) {
+ SJoinSupporter* pSupporter = (SJoinSupporter*)param;
+
+ SSqlObj* pParentSql = pSupporter->pObj;
+
+ SSqlObj* pSql = (SSqlObj*)tres;
+ SSqlCmd* pCmd = &pSql->cmd;
+ SSqlRes* pRes = &pSql->res;
- // write the compressed timestamp to disk file
- fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f);
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+ assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE));
+
+ // check for the error code firstly
+ if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
+ // todo retry if other subqueries are not failed yet
+ assert(numOfRows < 0 && numOfRows == taos_errno(pSql));
+ tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
+
+ pParentSql->res.code = numOfRows;
+ quitAllSubquery(pParentSql, pSupporter);
+
+ tscQueueAsyncRes(pParentSql);
+ return;
+ }
+
+ if (numOfRows > 0) { // write the compressed timestamp to disk file
+ fwrite(pRes->data, pRes->numOfRows, 1, pSupporter->f);
fclose(pSupporter->f);
pSupporter->f = NULL;
-
+
STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true);
- if (pBuf == NULL) {
+ if (pBuf == NULL) { // in error process, close the fd
tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows);
- pSupporter->pState->code = TSDB_CODE_TSC_APP_ERROR; // todo set the informative code
- quitAllSubquery(pParentSql, pSupporter);
+ pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
+ tscQueueAsyncRes(pParentSql);
+
return;
}
@@ -713,73 +790,135 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex);
tsBufDestory(pBuf);
}
-
- if (pSql->res.completed) {
- tSIntersectionAndLaunchSecQuery(pSupporter, pSql);
- } else { // open a new file to save the incoming result
+
+ // continue to retrieve ts-comp data from vnode
+ if (!pRes->completed) {
getTmpfilePath("ts-join", pSupporter->path);
pSupporter->f = fopen(pSupporter->path, "w");
- pSql->res.row = pSql->res.numOfRows;
+ pRes->row = pRes->numOfRows;
- taos_fetch_rows_a(tres, joinRetrieveCallback, param);
- }
- } else { // secondary stage retrieve, driven by taos_fetch_row or other functions
- if (numOfRows < 0) {
- pSupporter->pState->code = numOfRows;
- tscError("%p retrieve failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
+ taos_fetch_rows_a(tres, tsCompRetrieveCallback, param);
+ return;
}
+ }
- if (numOfRows >= 0) {
- pSql->res.numOfTotal += pSql->res.numOfRows;
- }
-
- if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) {
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- assert(pQueryInfo->numOfTables == 1);
-
- // for projection query, need to try next vnode if current vnode is exhausted
- if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) {
- pSupporter->pState->numOfCompleted = 0;
- pSupporter->pState->numOfTotal = 1;
-
- pSql->cmd.command = TSDB_SQL_SELECT;
- pSql->fp = tscJoinQueryCallback;
- tscProcessSql(pSql);
-
- return;
- }
- }
-
- int32_t numOfTotal = pSupporter->pState->numOfTotal;
- int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1);
-
- if (finished >= numOfTotal) {
- assert(finished == numOfTotal);
- tscTrace("%p all %d secondary subquery retrieves completed, global code:%d", tres, numOfTotal,
- pParentSql->res.code);
+ if (hasMoreVnodesToTry(pSql)) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- if (pSupporter->pState->code != TSDB_CODE_SUCCESS) {
- pParentSql->res.code = pSupporter->pState->code;
- freeJoinSubqueryObj(pParentSql);
- pParentSql->res.completed = true;
- }
-
- // update the records for each subquery in parent sql object.
- for(int32_t i = 0; i < pParentSql->numOfSubs; ++i) {
- if (pParentSql->pSubs[i] == NULL) {
- continue;
- }
+ int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+ pTableMetaInfo->vgroupIndex += 1;
+ assert(pTableMetaInfo->vgroupIndex < totalVgroups);
+
+ tscTrace("%p results from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%" PRId64,
+ pSql, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups,
+ pRes->numOfClauseTotal);
+
+ pCmd->command = TSDB_SQL_SELECT;
+ tscResetForNextRetrieve(&pSql->res);
+
+ assert(pSupporter->f == NULL);
+ getTmpfilePath("ts-join", pSupporter->path);
- SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
- pRes1->numOfClauseTotal += pRes1->numOfRows;
- }
-
- // data has retrieved to client, build the join results
- tscBuildResFromSubqueries(pParentSql);
- } else {
- tscTrace("%p sub:%p completed, completed:%d, total:%d", pParentSql, tres, finished, numOfTotal);
+ // TODO check for failure
+ pSupporter->f = fopen(pSupporter->path, "w");
+ pRes->row = pRes->numOfRows;
+
+ // set the callback function
+ pSql->fp = tscJoinQueryCallback;
+ tscProcessSql(pSql);
+ return;
+ }
+
+ if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+ return;
+ }
+
+ tscTrace("%p all subquery retrieve ts complete, do ts block intersect", pParentSql);
+
+ // proceeds to launched secondary query to retrieve final data
+ SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
+ SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
+
+ STimeWindow win = TSWINDOW_INITIALIZER;
+ int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &win);
+ if (num <= 0) { // no result during ts intersect
+ tscTrace("%p no results generated in ts intersection, free all sub SqlObj and quit", pParentSql);
+ freeJoinSubqueryObj(pParentSql);
+
+ return;
+ }
+
+ // launch the query the retrieve actual results from vnode along with the filtered timestamp
+ SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
+ updateQueryTimeRange(pPQueryInfo, &win);
+ tscLaunchSecondPhaseSubqueries(pParentSql);
+}
+
+static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfRows) {
+ SJoinSupporter* pSupporter = (SJoinSupporter*)param;
+
+ SSqlObj* pParentSql = pSupporter->pObj;
+ SSubqueryState* pState = pSupporter->pState;
+
+ SSqlObj* pSql = (SSqlObj*)tres;
+ SSqlCmd* pCmd = &pSql->cmd;
+ SSqlRes* pRes = &pSql->res;
+
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+
+ // TODO put to async res?
+ if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
+ assert(numOfRows == taos_errno(pSql));
+
+ pParentSql->res.code = numOfRows;
+ tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows));
+ }
+
+ if (numOfRows >= 0) {
+ pRes->numOfTotal += pRes->numOfRows;
+ }
+
+ if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ assert(pQueryInfo->numOfTables == 1);
+
+ // for projection query, need to try next vnode if current vnode is exhausted
+ if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) {
+ pState->numOfRemain = 1;
+ pState->numOfTotal = 1;
+
+ pSql->cmd.command = TSDB_SQL_SELECT;
+ pSql->fp = tscJoinQueryCallback;
+ tscProcessSql(pSql);
+
+ return;
+ }
+ }
+
+ if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
+ tscTrace("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfTotal);
+ return;
+ }
+
+ tscTrace("%p all %d secondary subqueries retrieval completed, code:%d", tres, pState->numOfTotal, pParentSql->res.code);
+
+ if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
+ freeJoinSubqueryObj(pParentSql);
+ pParentSql->res.completed = true;
+ }
+
+ // update the records for each subquery in parent sql object.
+ for (int32_t i = 0; i < pParentSql->numOfSubs; ++i) {
+ if (pParentSql->pSubs[i] == NULL) {
+ continue;
}
+
+ SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
+ pRes1->numOfClauseTotal += pRes1->numOfRows;
}
+
+ // data has retrieved to client, build the join results
+ tscBuildResFromSubqueries(pParentSql);
}
static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) {
@@ -797,7 +936,7 @@ static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch
}
pState->numOfTotal = pSql->numOfSubs;
- pState->numOfCompleted = pSql->numOfSubs - numOfFetch;
+ pState->numOfRemain = numOfFetch;
return pSupporter;
}
@@ -833,7 +972,10 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
}
}
} else { // has reach the limitation, no data anymore
- hasData = false;
+ if (pRes->row >= pRes->numOfRows) {
+ hasData = false;
+ break;
+ }
}
}
@@ -882,7 +1024,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
pSupporter->subqueryIndex, pTableMetaInfo->vgroupIndex);
tscResetForNextRetrieve(pRes1);
- pSql1->fp = joinRetrieveCallback;
+ pSql1->fp = joinRetrieveFinalResCallback;
if (pCmd1->command < TSDB_SQL_LOCAL) {
pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
@@ -900,8 +1042,9 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
tscTrace("%p all subquery response, retrieve data", pSql);
+ // the column transfer support struct has been built
if (pRes->pColumnIndex != NULL) {
- return; // the column transfer support struct has been built
+ return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@@ -937,68 +1080,76 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
SSqlObj* pSql = (SSqlObj*)tres;
-
+
SJoinSupporter* pSupporter = (SJoinSupporter*)param;
-
+ SSqlObj* pParentSql = pSupporter->pObj;
+
// There is only one subquery and table for each subquery.
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
assert(pQueryInfo->numOfTables == 1 && pSql->cmd.numOfClause == 1);
-
- if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
- if (code != TSDB_CODE_SUCCESS) { // direct call joinRetrieveCallback and set the error code
- joinRetrieveCallback(param, pSql, code);
- } else { // first stage query, continue to retrieve compressed time stamp data
- pSql->fp = joinRetrieveCallback;
- pSql->cmd.command = TSDB_SQL_FETCH;
- tscProcessSql(pSql);
- }
- } else { // second stage join subquery
- SSqlObj* pParentSql = pSupporter->pObj;
+ // retrieve actual query results from vnode during the second stage join subquery
+ if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
+ tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code);
+ quitAllSubquery(pParentSql, pSupporter);
+ tscQueueAsyncRes(pParentSql);
- if (pSupporter->pState->code != TSDB_CODE_SUCCESS) {
- tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code,
- pSupporter->pState->code);
- quitAllSubquery(pParentSql, pSupporter);
+ return;
+ }
- return;
- }
+ // TODO here retry is required, not directly returns to client
+ if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
+ assert(taos_errno(pSql) == code);
- if (code != TSDB_CODE_SUCCESS) {
- tscError("%p sub query failed, code:%s, set global code:%s, index:%d", pSql, tstrerror(code), tstrerror(code),
- pSupporter->subqueryIndex);
-
- pSupporter->pState->code = code;
- quitAllSubquery(pParentSql, pSupporter);
+ tscError("%p abort query, code:%d, global code:%d", pSql, code, pParentSql->res.code);
+ pParentSql->res.code = code;
+
+ quitAllSubquery(pParentSql, pSupporter);
+ tscQueueAsyncRes(pParentSql);
+
+ return;
+ }
+
+ // retrieve tuples from vnode
+ if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
+ pSql->fp = tidTagRetrieveCallback;
+ pSql->cmd.command = TSDB_SQL_FETCH;
+ tscProcessSql(pSql);
+ return;
+ }
+
+ // retrieve ts_comp info from vnode
+ if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
+ pSql->fp = tsCompRetrieveCallback;
+ pSql->cmd.command = TSDB_SQL_FETCH;
+ tscProcessSql(pSql);
+ return;
+ }
+
+ // wait for the other subqueries response from vnode
+ if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+ return;
+ }
+
+ tscSetupOutputColumnIndex(pParentSql);
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ /**
+ * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
+ * data instead of returning to its invoker
+ */
+ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+ pSupporter->pState->numOfRemain = pSupporter->pState->numOfTotal; // reset the record value
+
+ pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data
+ pSql->cmd.command = TSDB_SQL_FETCH;
+ tscProcessSql(pSql);
+ } else { // first retrieve from vnode during the secondary stage sub-query
+ // set the command flag must be after the semaphore been correctly set.
+ if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
+ (*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
- int32_t numOfTotal = pSupporter->pState->numOfTotal;
- int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1);
-
- if (finished >= numOfTotal) {
- assert(finished == numOfTotal);
-
- tscSetupOutputColumnIndex(pParentSql);
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
- /**
- * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
- * data instead of returning to its invoker
- */
- if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
- pSupporter->pState->numOfCompleted = 0; // reset the record value
-
- pSql->fp = joinRetrieveCallback; // continue retrieve data
- pSql->cmd.command = TSDB_SQL_FETCH;
- tscProcessSql(pSql);
- } else { // first retrieve from vnode during the secondary stage sub-query
- // set the command flag must be after the semaphore been correctly set.
- if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
- (*pParentSql->fp)(pParentSql->param, pParentSql, 0);
- } else {
- tscQueueAsyncRes(pParentSql);
- }
- }
- }
+ tscQueueAsyncRes(pParentSql);
}
}
}
@@ -1014,8 +1165,8 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
pSql->res.qhandle = 0x1;
- pSql->res.numOfRows = 0;
-
+ assert(pSql->res.numOfRows == 0);
+
if (pSql->pSubs == NULL) {
pSql->pSubs = calloc(pSupporter->pState->numOfTotal, POINTER_BYTES);
if (pSql->pSubs == NULL) {
@@ -1055,10 +1206,8 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
// this data needs to be transfer to support struct
memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo));
-
- pSupporter->tagCond = pNewQueryInfo->tagCond;
- memset(&pNewQueryInfo->tagCond, 0, sizeof(STagCond));
-
+ tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond);//pNewQueryInfo->tagCond;
+
pNew->cmd.numOfCols = 0;
pNewQueryInfo->intervalTime = 0;
memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal));
@@ -1071,39 +1220,34 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
- SSchema s = {0};
SColumnIndex index = {0};
-
- size_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList);
- for(int32_t i = 0; i < numOfTags; ++i) {
- SColumn* c = taosArrayGetP(pTableMetaInfo->tagColList, i);
- index = (SColumnIndex) {.tableIndex = 0, .columnIndex = c->colIndex.columnIndex};
-
- SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
- s = pTagSchema[c->colIndex.columnIndex];
-
- int16_t bytes = 0;
- int16_t type = 0;
- int32_t inter = 0;
-
- getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
-
- s.type = type;
- s.bytes = bytes;
- pSupporter->tagSize = s.bytes;
- }
-
+
+ STagCond* pTagCond = &pSupporter->tagCond;
+ assert(pTagCond->joinInfo.hasJoin);
+
+ int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->uid);
+ SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+
+ int16_t bytes = 0;
+ int16_t type = 0;
+ int32_t inter = 0;
+
+ getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
+
+ SSchema s1 = {.colId = s->colId, .type = type, .bytes = bytes};
+ pSupporter->tagSize = s1.bytes;
+ assert(isValidDataType(s1.type) && s1.bytes > 0);
+
// set get tags query type
TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
- tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG);
+ tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s1, TSDB_COL_TAG);
size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
tscTrace(
"%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), "
- "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
+ "exprInfo:%zu, colList:%zu, fieldsInfo:%d, tagIndex:%d, name:%s",
pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo),
- numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name);
-
+ numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, index.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name);
} else {
SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1};
SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
@@ -1112,10 +1256,11 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
// set the tags value for ts_comp function
SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0);
- int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
-
- pExpr->param->i64Key = tagColIndex;
- pExpr->numOfParams = 1;
+ if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+ int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+ pExpr->param->i64Key = tagColId;
+ pExpr->numOfParams = 1;
+ }
// add the filter tag column
if (pSupporter->colList != NULL) {
@@ -1134,8 +1279,8 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
tscTrace(
- "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to ts_comp query to retrieve timestamps, "
- "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
+ "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, "
+ "exprInfo:%zu, colList:%zu, fieldsInfo:%d, name:%s",
pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo),
numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name);
}
@@ -1155,14 +1300,15 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pQueryInfo->numOfTables;
-
+ pState->numOfRemain = pState->numOfTotal;
+
tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i);
if (pSupporter == NULL) { // failed to create support struct, abort current query
tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i);
- pState->numOfCompleted = pQueryInfo->numOfTables - i - 1;
+ pState->numOfRemain = i;
pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pSql->res.code;
@@ -1218,7 +1364,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
tOrderDescriptor *pDesc = NULL;
SColumnModel * pModel = NULL;
- pRes->qhandle = 1; // hack the qhandle check
+ pRes->qhandle = 0x1; // hack the qhandle check
const uint32_t nBufferSize = (1u << 16); // 64KB
@@ -1240,6 +1386,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
tscTrace("%p retrieved query data from %d vnode(s)", pSql, pSql->numOfSubs);
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pSql->numOfSubs;
+ pState->numOfRemain = pSql->numOfSubs;
+
pRes->code = TSDB_CODE_SUCCESS;
int32_t i = 0;
@@ -1267,7 +1415,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
pthread_mutexattr_t mutexattr;
memset(&mutexattr, 0, sizeof(pthread_mutexattr_t));
-
+
pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP);
pthread_mutex_init(&trs->queryMutex, &mutexattr);
pthread_mutexattr_destroy(&mutexattr);
@@ -1284,6 +1432,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
if (pQueryInfo->tsBuf) {
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf);
+ assert(pNewQueryInfo->tsBuf != NULL);
}
tscTrace("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex);
@@ -1333,7 +1482,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows);
-static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) {
+static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) {
// set no disk space error info
#ifdef WINDOWS
LPVOID lpMsgBuf;
@@ -1343,46 +1492,45 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES
tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf);
LocalFree(lpMsgBuf);
#else
- tscError("sub:%p failed to flush data to disk:reason:%s", tres, strerror(errno));
+ tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code));
#endif
-
- trsupport->pState->code = -errCode;
+
+ SSqlObj* pParentSql = trsupport->pParentSqlObj;
+
+ pParentSql->res.code = code;
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
pthread_mutex_unlock(&trsupport->queryMutex);
-
- tscHandleSubqueryError(trsupport, tres, trsupport->pState->code);
+ tscHandleSubqueryError(trsupport, tres, pParentSql->res.code);
}
void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
- SSqlObj *pPObj = trsupport->pParentSqlObj;
+ SSqlObj *pParentSql = trsupport->pParentSqlObj;
int32_t subqueryIndex = trsupport->subqueryIndex;
assert(pSql != NULL);
SSubqueryState* pState = trsupport->pState;
- assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 &&
- pPObj->numOfSubs == pState->numOfTotal);
+ assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
// retrieved in subquery failed. OR query cancelled in retrieve phase.
- if (pState->code == TSDB_CODE_SUCCESS && pPObj->res.code != TSDB_CODE_SUCCESS) {
- pState->code = pPObj->res.code;
-
+ if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) {
+
/*
* kill current sub-query connection, which may retrieve data from vnodes;
* Here we get: pPObj->res.code == TSDB_CODE_TSC_QUERY_CANCELLED
*/
pSql->res.numOfRows = 0;
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts
- tscTrace("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", trsupport->pParentSqlObj, pSql,
- subqueryIndex, pState->code);
+ tscTrace("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql,
+ subqueryIndex, pParentSql->res.code);
}
if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query.
- tscTrace("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pPObj, pSql, numOfRows, subqueryIndex);
- tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pPObj, pSql,
- subqueryIndex, pState->code);
+ tscTrace("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex);
+ tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql,
+ subqueryIndex, pParentSql->res.code);
} else {
- if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pState->code == TSDB_CODE_SUCCESS) {
+ if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) {
/*
* current query failed, and the retry count is less than the available
* count, retry query clear previous retrieved data, then launch a new sub query
@@ -1400,8 +1548,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
if (pNew == NULL) {
tscError("%p sub:%p failed to create new subquery sqlObj due to out of memory, abort retry",
trsupport->pParentSqlObj, pSql);
-
- pState->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+
+ pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
return;
}
@@ -1409,24 +1557,24 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
tscProcessSql(pNew);
return;
} else { // reach the maximum retry count, abort
- atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows);
- tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pPObj, pSql,
- tstrerror(numOfRows), subqueryIndex, tstrerror(pState->code));
+ atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows);
+ tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pParentSql, pSql,
+ tstrerror(numOfRows), subqueryIndex, tstrerror(pParentSql->res.code));
}
}
-
- int32_t numOfTotal = pState->numOfTotal;
-
- int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
- if (finished < numOfTotal) {
- tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
+
+ int32_t remain = -1;
+ if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
+ tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
+ pState->numOfTotal - remain);
+
return tscFreeSubSqlObj(trsupport, pSql);
}
// all subqueries are failed
- tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pPObj, pState->numOfTotal, tstrerror(pState->code));
- pPObj->res.code = pState->code;
-
+ tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfTotal,
+ tstrerror(pParentSql->res.code));
+
// release allocated resource
tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel,
pState->numOfTotal);
@@ -1435,13 +1583,13 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
tscFreeSubSqlObj(trsupport, pSql);
// in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes
- SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
+ SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
- if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) {
- (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code);
+ if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
+ (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code);
} else { // regular super table query
- if (pPObj->res.code != TSDB_CODE_SUCCESS) {
- tscQueueAsyncRes(pPObj);
+ if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
+ tscQueueAsyncRes(pParentSql);
}
}
}
@@ -1458,7 +1606,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num;
- tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
+ tscTrace("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
numOfRowsFromSubquery, idx);
@@ -1481,27 +1629,23 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
// each result for a vnode is ordered as an independant list,
// then used as an input of loser tree for disk-based merge routine
- int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
- pQueryInfo->groupbyExpr.orderType);
- if (ret != 0) { // set no disk space error info, and abort retry
- return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE);
+ int32_t code = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pQueryInfo->groupbyExpr.orderType);
+ if (code != 0) { // set no disk space error info, and abort retry
+ return tscAbortFurtherRetryRetrieval(trsupport, pSql, code);
}
- // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
- // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
- // In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
- int32_t numOfTotal = pState->numOfTotal;
-
- int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
- if (finished < numOfTotal) {
- tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
+ int32_t remain = -1;
+ if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
+ tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex,
+ pState->numOfTotal - remain);
+
return tscFreeSubSqlObj(trsupport, pSql);
}
// all sub-queries are returned, start to local merge process
pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
- tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
+ tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", pPObj,
pState->numOfTotal, pState->numOfRetrievedRows);
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
@@ -1529,10 +1673,10 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
- int32_t idx = trsupport->subqueryIndex;
- SSqlObj * pPObj = trsupport->pParentSqlObj;
tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
-
+ int32_t idx = trsupport->subqueryIndex;
+ SSqlObj * pPObj = trsupport->pParentSqlObj;
+
SSqlObj *pSql = (SSqlObj *)tres;
if (pSql == NULL) { // sql object has been released in error process, return immediately
tscTrace("%p subquery has been released, idx:%d, abort", pPObj, idx);
@@ -1540,13 +1684,12 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
}
SSubqueryState* pState = trsupport->pState;
- assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 &&
- pPObj->numOfSubs == pState->numOfTotal);
+ assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pPObj->numOfSubs == pState->numOfTotal);
// query process and cancel query process may execute at the same time
pthread_mutex_lock(&trsupport->queryMutex);
- if (numOfRows < 0 || pState->code < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) {
+ if (numOfRows < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) {
return tscHandleSubqueryError(trsupport, pSql, numOfRows);
}
@@ -1557,11 +1700,11 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(pRes->numOfRows == numOfRows);
int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
- tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%s, orderOfSub:%d", pPObj, pSql,
+ tscTrace("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pPObj, pSql,
pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx);
-
+
if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
- tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64,
+ tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64,
pPObj, pSql, tsMaxNumOfOrderedResults, num);
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY);
return;
@@ -1584,8 +1727,9 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data,
pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
- if (ret < 0) { // set no disk space error info, and abort retry
+ if (ret != 0) { // set no disk space error info, and abort retry
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE);
+ pthread_mutex_unlock(&trsupport->queryMutex);
} else if (pRes->completed) {
tscAllDataRetrievedFromDnode(trsupport, pSql);
@@ -1596,7 +1740,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
}
- pthread_mutex_unlock(&trsupport->queryMutex);
} else { // all data has been retrieved to client
tscAllDataRetrievedFromDnode(trsupport, pSql);
}
@@ -1635,18 +1778,16 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
SSubqueryState* pState = trsupport->pState;
- assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 &&
- pParentSql->numOfSubs == pState->numOfTotal);
-
- if (pParentSql->res.code != TSDB_CODE_SUCCESS || pState->code != TSDB_CODE_SUCCESS) {
+ assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
+
+ // todo set error code
+ if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
// stable query is killed, abort further retry
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
code = pParentSql->res.code;
- } else {
- code = pState->code;
}
tscTrace("%p query cancelled or failed, sub:%p, orderOfSub:%d abort, code:%s", pParentSql, pSql,
@@ -1663,7 +1804,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
if (code != TSDB_CODE_SUCCESS) {
if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) {
tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code));
- atomic_val_compare_exchange_32(&pState->code, 0, code);
+ atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code);
} else { // does not reach the maximum retry time, go on
tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry);
@@ -1672,8 +1813,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
if (pNew == NULL) {
tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
trsupport->pParentSqlObj, pSql, pVgroup->vgId, trsupport->subqueryIndex);
-
- pState->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+
+ pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
} else {
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
@@ -1686,13 +1827,13 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
}
}
- if (pState->code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query
- tscTrace("%p sub:%p query failed,ip:%u,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql,
- pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
+ if (pParentSql->res.code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query
+ tscTrace("%p sub:%p query failed,ip:%s,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql,
+ pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pParentSql->res.code);
- tscHandleSubqueryError(param, tres, pState->code);
+ tscHandleSubqueryError(param, tres, pParentSql->res.code);
} else { // success, proceed to retrieve data from dnode
- tscTrace("%p sub:%p query complete, ip:%u, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql,
+ tscTrace("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql,
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex);
if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode
@@ -1704,36 +1845,37 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
}
}
-static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) {
+static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
SInsertSupporter *pSupporter = (SInsertSupporter *)param;
SSqlObj* pParentObj = pSupporter->pSql;
SSqlCmd* pParentCmd = &pParentObj->cmd;
SSubqueryState* pState = pSupporter->pState;
- int32_t total = pState->numOfTotal;
-
- // increase the total inserted rows
+
+ // record the total inserted rows
if (numOfRows > 0) {
pParentObj->res.numOfRows += numOfRows;
- } else {
+ }
+
+ if (taos_errno(tres) != TSDB_CODE_SUCCESS) {
SSqlObj* pSql = (SSqlObj*) tres;
assert(pSql != NULL && pSql->res.code == numOfRows);
pParentObj->res.code = pSql->res.code;
}
-
+
+ // it is not the initial sqlObj, free it
taos_free_result(tres);
- int32_t completed = atomic_add_fetch_32(&pState->numOfCompleted, 1);
- if (completed < total) {
+ tfree(pSupporter);
+
+ if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
return;
}
- tscTrace("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows);
-
- tfree(pState);
- tfree(pSupporter);
-
+ tscTrace("%p Async insertion completed, total inserted:%" PRId64, pParentObj, pParentObj->res.numOfRows);
+
// release data block data
+ tfree(pState);
pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks);
// restore user defined fp
@@ -1747,9 +1889,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
- pRes->qhandle = 1; // hack the qhandle check
SDataBlockList *pDataBlocks = pCmd->pDataBlocks;
-
pSql->pSubs = calloc(pDataBlocks->nSize, POINTER_BYTES);
pSql->numOfSubs = pDataBlocks->nSize;
assert(pDataBlocks->nSize > 0);
@@ -1757,52 +1897,62 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
tscTrace("%p submit data to %d vnode(s)", pSql, pDataBlocks->nSize);
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pSql->numOfSubs;
-
+ pState->numOfRemain = pSql->numOfSubs;
+
pRes->code = TSDB_CODE_SUCCESS;
-
- int32_t i = 0;
- for (; i < pSql->numOfSubs; ++i) {
+ int32_t numOfSub = 0;
+
+ while(numOfSub < pSql->numOfSubs) {
SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
pSupporter->pSql = pSql;
pSupporter->pState = pState;
- SSqlObj *pNew = createSubqueryObj(pSql, 0, multiVnodeInsertMerge, pSupporter, TSDB_SQL_INSERT, NULL);
+ SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT);//createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL);
if (pNew == NULL) {
- tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
- break;
+ tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, numOfSub, strerror(errno));
+ goto _error;
}
/*
* assign the callback function to fetchFp to make sure that the error process function can restore
- * the callback function (multiVnodeInsertMerge) correctly.
+ * the callback function (multiVnodeInsertFinalize) correctly.
*/
pNew->fetchFp = pNew->fp;
- pSql->pSubs[i] = pNew;
- pNew->fetchFp = pNew->fp;
-
- tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, i);
+ pSql->pSubs[numOfSub] = pNew;
+
+ pRes->code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub++]);
+ if (pRes->code == TSDB_CODE_SUCCESS) {
+ tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
+ } else {
+ tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
+ pDataBlocks->nSize, tstrerror(pRes->code));
+ goto _error;
+ }
}
- if (i < pSql->numOfSubs) {
+ if (numOfSub < pSql->numOfSubs) {
tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pRes->code; // free all allocated resource
}
-
- for (int32_t j = 0; j < pSql->numOfSubs; ++j) {
+
+ // use the local variable
+ for (int32_t j = 0; j < numOfSub; ++j) {
SSqlObj *pSub = pSql->pSubs[j];
- int32_t code = tscCopyDataBlockToPayload(pSub, pDataBlocks->pData[j]);
-
- if (code != TSDB_CODE_SUCCESS) {
- tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, j,
- pDataBlocks->nSize, code);
- }
-
tscTrace("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, j);
tscProcessSql(pSub);
}
return TSDB_CODE_SUCCESS;
+
+ _error:
+ for(int32_t j = 0; j < numOfSub; ++j) {
+ tfree(pSql->pSubs[j]->param);
+ taos_free_result(pSql->pSubs[j]);
+ }
+
+ tfree(pState);
+ return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
void tscBuildResFromSubqueries(SSqlObj *pSql) {
@@ -1847,12 +1997,13 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
SColumnIndex* pIndex = &pRes->pColumnIndex[i];
SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res;
pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex];
+ pRes->length[i] = pRes1->length[pIndex->columnIndex];
}
pRes->numOfClauseTotal++;
break;
} else { // continue retrieve data from vnode
- if (!tscHashRemainDataInSubqueryResultSet(pSql)) {
+ if (!tscHasRemainDataInSubqueryResultSet(pSql)) {
tscTrace("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1);
SSubqueryState *pState = NULL;
@@ -1908,7 +2059,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
pRes->length[columnIndex] = length;
} else {
- tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow[columnIndex]);
+ tscError("%p charset:%s to %s. val:%s convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)pRes->tsrow[columnIndex]);
pRes->tsrow[columnIndex] = NULL;
pRes->length[columnIndex] = 0;
}
@@ -2002,7 +2153,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
return pRes->tsrow;
}
-static UNUSED_FUNC bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
+static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
bool hasData = true;
SSqlCmd *pCmd = &pSql->cmd;
@@ -2045,8 +2196,7 @@ static UNUSED_FUNC bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0);
if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) &&
- tscProjectionQueryOnTable(pQueryInfo1)) ||
- (pRes1->numOfRows == 0)) {
+ tscIsProjectionQuery(pQueryInfo1)) || (pRes1->numOfRows == 0)) {
hasData = false;
break;
}
@@ -2055,5 +2205,3 @@ static UNUSED_FUNC bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
return hasData;
}
-
-
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index bcd01a322e84f7d596a3f7715b687cba812c8410..a653b8383342d688d6b9ea42919ac4afe7b69423 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -220,7 +220,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
if (strlen(tsLocale) == 0) { // locale does not set yet
char* defaultLocale = setlocale(LC_CTYPE, "");
- strcpy(tsLocale, defaultLocale);
+ tstrncpy(tsLocale, defaultLocale, sizeof(tsLocale));
}
// set the user specified locale
@@ -304,7 +304,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
- strcpy(tsTimezone, pStr);
+ tstrncpy(tsTimezone, pStr, sizeof(tsTimezone));
tsSetTimeZone();
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscTrace("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index b1f5a99b67d8dcc598d53f12866550aa4e6e9671..b56470e8da56d19becac72674b6ed852fa08dde5 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -22,7 +22,7 @@
#include "tkey.h"
#include "tmd5.h"
#include "tscProfile.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
#include "tscSubquery.h"
#include "tschemautil.h"
#include "tsclient.h"
@@ -32,7 +32,7 @@
static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
-
+
SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
if (pTagCond->pCond == NULL) {
return NULL;
@@ -70,13 +70,6 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
taosArrayPush(pTagCond->pCond, &cond);
}
-bool tscQueryOnSTable(SSqlCmd* pCmd) {
- SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
- return ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) &&
- (pCmd->msgType == TSDB_MSG_TYPE_QUERY);
-}
-
bool tscQueryTags(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
@@ -95,32 +88,8 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) {
return true;
}
-bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) {
- bool hasTags = false;
- int32_t numOfSelectivity = 0;
-
- SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId;
- if (functId == TSDB_FUNC_TAG_DUMMY) {
- hasTags = true;
- continue;
- }
-
- if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
- numOfSelectivity++;
- }
- }
-
- if (numOfSelectivity > 0 && hasTags) {
- return true;
- }
-
- return false;
-}
-
-void tscGetDBInfoFromMeterId(char* tableId, char* db) {
+// todo refactor, extract methods and move the common module
+void tscGetDBInfoFromTableFullName(char* tableId, char* db) {
char* st = strstr(tableId, TS_PATH_DELIMITER);
if (st != NULL) {
char* end = strstr(st + 1, TS_PATH_DELIMITER);
@@ -181,8 +150,14 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
for (int32_t i = 0; i < numOfExprs; ++i) {
int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
- if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
- functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) {
+
+ if (functionId != TSDB_FUNC_PRJ &&
+ functionId != TSDB_FUNC_TAGPRJ &&
+ functionId != TSDB_FUNC_TAG &&
+ functionId != TSDB_FUNC_TS &&
+ functionId != TSDB_FUNC_ARITHM &&
+ functionId != TSDB_FUNC_TS_COMP &&
+ functionId != TSDB_FUNC_TID_TAG) {
return false;
}
}
@@ -209,10 +184,14 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde
return pQueryInfo->order.orderColId >= 0;
}
-bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) {
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
+ size_t size = tscSqlExprNumOfExprs(pQueryInfo);
+
+ for (int32_t i = 0; i < size; ++i) {
int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
- if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TS) {
+
+ if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
+ functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) {
return false;
}
}
@@ -225,9 +204,10 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
- if (pExpr == NULL) {
- return false;
- }
+ assert(pExpr != NULL);
+// if (pExpr == NULL) {
+// return false;
+// }
int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TAG) {
@@ -238,6 +218,7 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
return false;
}
}
+
return true;
}
@@ -383,7 +364,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
tscFreeSqlResult(pSql);
tfree(pSql->pSubs);
- pSql->freed = 0;
pSql->numOfSubs = 0;
tscResetSqlCmdObj(pCmd);
@@ -1600,24 +1580,6 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) {
}
}
-void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) {
- if (index < 0 || index >= pQueryInfo->numOfTables) {
- return;
- }
-
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index);
-
- tscClearTableMetaInfo(pTableMetaInfo, removeFromCache);
- free(pTableMetaInfo);
-
- int32_t after = pQueryInfo->numOfTables - index - 1;
- if (after > 0) {
- memmove(&pQueryInfo->pTableMetaInfo[index], &pQueryInfo->pTableMetaInfo[index + 1], after * POINTER_BYTES);
- }
-
- pQueryInfo->numOfTables -= 1;
-}
-
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
@@ -1690,6 +1652,47 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
pRes->numOfRows = 0;
}
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {
+ SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
+ if (pNew == NULL) {
+ tscError("%p new subquery failed, tableIndex:%d", pSql, 0);
+ return NULL;
+ }
+
+ pNew->pTscObj = pSql->pTscObj;
+ pNew->signature = pNew;
+
+ SSqlCmd* pCmd = &pNew->cmd;
+ pCmd->command = cmd;
+ pCmd->parseFinished = 1;
+
+ if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
+ tscFreeSqlObj(pNew);
+ return NULL;
+ }
+
+ pNew->fp = fp;
+ pNew->param = param;
+ pNew->maxRetry = TSDB_MAX_REPLICA_NUM;
+
+ pNew->sqlstr = strdup(pSql->sqlstr);
+ if (pNew->sqlstr == NULL) {
+ tscError("%p new subquery failed", pSql);
+
+ free(pNew);
+ return NULL;
+ }
+
+ SQueryInfo* pQueryInfo = NULL;
+ tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo);
+
+ assert(pSql->cmd.clauseIndex == 0);
+ STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
+
+ tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
+ return pNew;
+}
+
SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
@@ -1722,6 +1725,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pnCmd->numOfClause = 0;
pnCmd->clauseIndex = 0;
pnCmd->pDataBlocks = NULL;
+ pnCmd->parseFinished = 1;
if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);
@@ -1772,7 +1776,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
SQueryInfo* pPrevQueryInfo = tscGetQueryInfoDetail(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex);
pNewQueryInfo->type = pPrevQueryInfo->type;
} else {
- pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; // it must be the subquery
+ TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery
}
uint64_t uid = pTableMetaInfo->pTableMeta->uid;
@@ -1797,19 +1801,26 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
}
// make sure the the sqlExpr for each fields is correct
-// todo handle the agg arithmetic expression
+ // todo handle the agg arithmetic expression
+ numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
+
for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) {
TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f);
- numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
-
+ bool matched = false;
+
for(int32_t k1 = 0; k1 < numOfExprs; ++k1) {
SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1);
-
- if (strcmp(field->name, pExpr1->aliasName) == 0) { // eatablish link according to the result field name
+
+ if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f);
pInfo->pSqlExpr = pExpr1;
+
+ matched = true;
+ break;
}
}
+
+ assert(matched);
}
tscFieldInfoUpdateOffset(pNewQueryInfo);
@@ -1825,6 +1836,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
if (pPrevSql == NULL) {
STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup
assert(pTableMeta != NULL);
+
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
} else { // transfer the ownership of pTableMeta to the newly create sql object.
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
@@ -1847,12 +1859,12 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
assert(pFinalInfo->vgroupList != NULL);
}
-
+
if (cmd == TSDB_SQL_SELECT) {
size_t size = taosArrayGetSize(pNewQueryInfo->colList);
tscTrace(
- "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d,"
+ "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%zu, colList:%zu,"
"fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64,
pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo),
size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey,
@@ -1898,16 +1910,21 @@ void tscDoQuery(SSqlObj* pSql) {
}
if (QUERY_IS_JOIN_QUERY(type)) {
- if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) {
+ if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) {
tscHandleMasterJoinQuery(pSql);
- return;
- } else {
- // for first stage sub query, iterate all vnodes to get all timestamp
- if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) {
-// doProcessSql(pSql);
- assert(0);
+ } else { // for first stage sub query, iterate all vnodes to get all timestamp
+ if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
+ tscProcessSql(pSql);
+ } else { // secondary stage join query.
+ if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query
+ tscHandleMasterSTableQuery(pSql);
+ } else {
+ tscProcessSql(pSql);
+ }
}
}
+
+ return;
} else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query
tscHandleMasterSTableQuery(pSql);
return;
@@ -1917,11 +1934,13 @@ void tscDoQuery(SSqlObj* pSql) {
}
}
-int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid) {
+int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
if (pTagCond->joinInfo.left.uid == uid) {
- return pTagCond->joinInfo.left.tagCol;
+ return pTagCond->joinInfo.left.tagColId;
+ } else if (pTagCond->joinInfo.right.uid == uid) {
+ return pTagCond->joinInfo.right.tagColId;
} else {
- return pTagCond->joinInfo.right.tagCol;
+ assert(0);
}
}
@@ -1965,6 +1984,11 @@ bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
}
+bool tscResultsetFetchCompleted(TAOS_RES *result) {
+ SSqlRes* pRes = result;
+ return pRes->completed;
+}
+
char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
/**
@@ -1978,11 +2002,10 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
return false;
}
+ assert(pRes->completed);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- assert(pRes->completed);
-
+
// for normal table, no need to try any more if results are all retrieved from one vnode
if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) {
return false;
@@ -2004,12 +2027,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
* if case of: multi-vnode super table projection query
*/
assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes));
-
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
- tscTrace("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%d", pSql,
+ tscTrace("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql,
pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
/*
diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h
index 3ad2df8d3a63f9d7c01c6bf73b843844faf850be..1cd72eafdedd512bb2d675613b178252efe5d5b2 100644
--- a/src/common/inc/tdataformat.h
+++ b/src/common/inc/tdataformat.h
@@ -69,8 +69,8 @@ typedef struct {
int version; // version
int numOfCols; // Number of columns appended
int tlen; // maximum length of a SDataRow without the header part
- uint16_t flen; // First part length in a SDataRow after the header part
- uint16_t vlen; // pure value part length, excluded the overhead
+ uint16_t flen; // First part length in a SDataRow after the header part
+ uint16_t vlen; // pure value part length, excluded the overhead
STColumn columns[];
} STSchema;
@@ -83,8 +83,8 @@ typedef struct {
#define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema);
-void * tdEncodeSchema(void *dst, STSchema *pSchema);
-STSchema *tdDecodeSchema(void **psrc);
+int tdEncodeSchema(void **buf, STSchema *pSchema);
+void * tdDecodeSchema(void *buf, STSchema **pRSchema);
static FORCE_INLINE int comparColId(const void *key1, const void *key2) {
if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
@@ -107,8 +107,8 @@ typedef struct {
int tCols;
int nCols;
int tlen;
- uint16_t flen;
- uint16_t vlen;
+ uint16_t flen;
+ uint16_t vlen;
int version;
STColumn *columns;
} STSchemaBuilder;
@@ -288,7 +288,7 @@ typedef struct {
SKVRow tdKVRowDup(SKVRow row);
int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
-void * tdEncodeKVRow(void *buf, SKVRow row);
+int tdEncodeKVRow(void **buf, SKVRow row);
void * tdDecodeKVRow(void *buf, SKVRow *row);
static FORCE_INLINE int comparTagId(const void *key1, const void *key2) {
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index eaceea468b4d2d5020e11e19b1d7e8b67cbf915c..980635588ee537702287073bcf8c8e361f5e221b 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -92,10 +92,6 @@ extern int32_t tsNumOfMnodes;
extern int32_t tsMaxShellConns;
extern int32_t tsMaxTables;
-extern char tsDefaultDB[];
-extern char tsDefaultUser[];
-extern char tsDefaultPass[];
-
extern char tsMqttBrokerAddress[];
extern char tsMqttBrokerClientId[];
diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c
index 40cdac9833fb37569c5e36e80dd53d5b55b6aac8..cd1f4b1ca447f326d4dfd8432252829fee43d7d7 100644
--- a/src/common/src/tdataformat.c
+++ b/src/common/src/tdataformat.c
@@ -14,6 +14,7 @@
*/
#include "tdataformat.h"
#include "talgo.h"
+#include "tcoding.h"
#include "wchar.h"
/**
@@ -33,50 +34,50 @@ STSchema *tdDupSchema(STSchema *pSchema) {
/**
* Encode a schema to dst, and return the next pointer
*/
-void *tdEncodeSchema(void *dst, STSchema *pSchema) {
+int tdEncodeSchema(void **buf, STSchema *pSchema) {
+ int tlen = 0;
+ tlen += taosEncodeFixedI32(buf, schemaVersion(pSchema));
+ tlen += taosEncodeFixedI32(buf, schemaNCols(pSchema));
- T_APPEND_MEMBER(dst, pSchema, STSchema, version);
- T_APPEND_MEMBER(dst, pSchema, STSchema, numOfCols);
for (int i = 0; i < schemaNCols(pSchema); i++) {
STColumn *pCol = schemaColAt(pSchema, i);
- T_APPEND_MEMBER(dst, pCol, STColumn, type);
- T_APPEND_MEMBER(dst, pCol, STColumn, colId);
- T_APPEND_MEMBER(dst, pCol, STColumn, bytes);
+ tlen += taosEncodeFixedI8(buf, colType(pCol));
+ tlen += taosEncodeFixedI16(buf, colColId(pCol));
+ tlen += taosEncodeFixedI32(buf, colBytes(pCol));
}
- return dst;
+ return tlen;
}
/**
* Decode a schema from a binary.
*/
-STSchema *tdDecodeSchema(void **psrc) {
- int totalCols = 0;
+void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
int version = 0;
- STSchemaBuilder schemaBuilder = {0};
+ int numOfCols = 0;
+ STSchemaBuilder schemaBuilder;
- T_READ_MEMBER(*psrc, int, version);
- T_READ_MEMBER(*psrc, int, totalCols);
+ buf = taosDecodeFixedI32(buf, &version);
+ buf = taosDecodeFixedI32(buf, &numOfCols);
if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL;
- for (int i = 0; i < totalCols; i++) {
+ for (int i = 0; i < numOfCols; i++) {
int8_t type = 0;
int16_t colId = 0;
int32_t bytes = 0;
- T_READ_MEMBER(*psrc, int8_t, type);
- T_READ_MEMBER(*psrc, int16_t, colId);
- T_READ_MEMBER(*psrc, int32_t, bytes);
-
+ buf = taosDecodeFixedI8(buf, &type);
+ buf = taosDecodeFixedI16(buf, &colId);
+ buf = taosDecodeFixedI32(buf, &bytes);
if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
tdDestroyTSchemaBuilder(&schemaBuilder);
return NULL;
}
}
- STSchema *pSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+ *pRSchema = tdGetSchemaFromBuilder(&schemaBuilder);
tdDestroyTSchemaBuilder(&schemaBuilder);
- return pSchema;
+ return buf;
}
int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
@@ -105,7 +106,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
}
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int32_t bytes) {
- if (!isValidDataType(type, 0)) return -1;
+ if (!isValidDataType(type)) return -1;
if (pBuilder->nCols >= pBuilder->tCols) {
pBuilder->tCols *= 2;
@@ -605,14 +606,19 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
return 0;
}
-void *tdEncodeKVRow(void *buf, SKVRow row) {
+int tdEncodeKVRow(void **buf, SKVRow row) {
// May change the encode purpose
- kvRowCpy(buf, row);
- return POINTER_SHIFT(buf, kvRowLen(row));
+ if (buf != NULL) {
+ kvRowCpy(*buf, row);
+ *buf = POINTER_SHIFT(*buf, kvRowLen(row));
+ }
+
+ return kvRowLen(row);
}
void *tdDecodeKVRow(void *buf, SKVRow *row) {
*row = tdKVRowDup(buf);
+ if (*row == NULL) return NULL;
return POINTER_SHIFT(buf, kvRowLen(*row));
}
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index e80597ca954652abd450f20738be0ecca849a04a..2ed6ef3ab7a77ea3f6bc6def2d8456828c6ef1c5 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -109,10 +109,6 @@ int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
int16_t tsAffectedRowsMod = 0;
int32_t tsNumOfMnodes = 3;
int32_t tsMaxShellConns = 5000;
-
-char tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
-char tsDefaultUser[64] = "root";
-char tsDefaultPass[64] = "taosdata";
int32_t tsMaxConnections = 5000;
int32_t tsBalanceInterval = 300; // seconds
@@ -713,37 +709,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
- // login configs
- cfg.option = "defaultDB";
- cfg.ptr = tsDefaultDB;
- cfg.valType = TAOS_CFG_VTYPE_STRING;
- cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
- cfg.minValue = 0;
- cfg.maxValue = 0;
- cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
- cfg.unitType = TAOS_CFG_UTYPE_NONE;
- taosInitConfigOption(cfg);
-
- cfg.option = "defaultUser";
- cfg.ptr = tsDefaultUser;
- cfg.valType = TAOS_CFG_VTYPE_STRING;
- cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
- cfg.minValue = 0;
- cfg.maxValue = 0;
- cfg.ptrLength = TSDB_USER_LEN - 1;
- cfg.unitType = TAOS_CFG_UTYPE_NONE;
- taosInitConfigOption(cfg);
-
- cfg.option = "defaultPass";
- cfg.ptr = tsDefaultPass;
- cfg.valType = TAOS_CFG_VTYPE_STRING;
- cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT;
- cfg.minValue = 0;
- cfg.maxValue = 0;
- cfg.ptrLength = TSDB_PASSWORD_LEN - 1;
- cfg.unitType = TAOS_CFG_UTYPE_NONE;
- taosInitConfigOption(cfg);
-
cfg.option = "mqttBrokerAddress";
cfg.ptr = tsMqttBrokerAddress;
cfg.valType = TAOS_CFG_VTYPE_STRING;
diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c
index 4aee027c0b759aacaf9b1deeb9a3a8e651c33ae1..3c63e2deacc88aed967508a6affc44b1bc7d85ca 100644
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@@ -363,16 +363,8 @@ char tTokenTypeSwitcher[13] = {
TSDB_DATA_TYPE_NCHAR, // TK_NCHAR
};
-bool isValidDataType(int32_t type, int32_t length) {
- if (type < TSDB_DATA_TYPE_NULL || type > TSDB_DATA_TYPE_NCHAR) {
- return false;
- }
-
- if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-// return length >= 0 && length <= TSDB_MAX_BINARY_LEN;
- }
-
- return true;
+bool isValidDataType(int32_t type) {
+ return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
}
bool isNull(const char *val, int32_t type) {
diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath
new file mode 100644
index 0000000000000000000000000000000000000000..39abf1c5e9102a46b2af14da5d6d536eb2cd121a
--- /dev/null
+++ b/src/connector/jdbc/.classpath
@@ -0,0 +1,49 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/connector/jdbc/.project b/src/connector/jdbc/.project
new file mode 100644
index 0000000000000000000000000000000000000000..656ab58d205bf0e01c151728a75e5ab543facee6
--- /dev/null
+++ b/src/connector/jdbc/.project
@@ -0,0 +1,23 @@
+
+
+ taos-jdbcdriver
+
+
+
+
+
+ org.eclipse.jdt.core.javabuilder
+
+
+
+
+ org.eclipse.m2e.core.maven2Builder
+
+
+
+
+
+ org.eclipse.jdt.core.javanature
+ org.eclipse.m2e.core.maven2Nature
+
+
diff --git a/src/connector/jdbc/.settings/org.eclipse.core.resources.prefs b/src/connector/jdbc/.settings/org.eclipse.core.resources.prefs
deleted file mode 100755
index 1935ba2ee400625d903bb90ecbb42d9a185504ed..0000000000000000000000000000000000000000
--- a/src/connector/jdbc/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=GBK
-encoding//src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding/=UTF-8
diff --git a/src/connector/jdbc/.settings/org.eclipse.jdt.core.prefs b/src/connector/jdbc/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100755
index 5ce4518899426199a8ab051fb032691fe545dcee..0000000000000000000000000000000000000000
--- a/src/connector/jdbc/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,13 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/src/connector/jdbc/.settings/org.eclipse.m2e.core.prefs b/src/connector/jdbc/.settings/org.eclipse.m2e.core.prefs
deleted file mode 100755
index f897a7f1cb2389f85fe6381425d29f0a9866fb65..0000000000000000000000000000000000000000
--- a/src/connector/jdbc/.settings/org.eclipse.m2e.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1
diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml
index 36f1a1010c372b319fab7fceaf13c9c44689dc6b..5aa4f6a2e3e41542fd25b655f9959da13fe4efdb 100755
--- a/src/connector/jdbc/pom.xml
+++ b/src/connector/jdbc/pom.xml
@@ -1,105 +1,136 @@
- 4.0.0
-
- com.taosdata.jdbc
- taos-jdbcdriver
- 1.0.3
- jar
-
- JDBCDriver
- https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc
- TDengine JDBC Driver
-
-
-
- GNU AFFERO GENERAL PUBLIC LICENSE Version 3
- https://github.com/taosdata/TDengine/blob/master/LICENSE
- repo
-
-
-
-
- scm:git:git://github.com/taosdata/TDengine.git
- scm:git:git@github.com:taosdata/TDengine.git
- https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc
- HEAD
-
-
-
-
- taosdata
- support@taosdata.com
- https://www.taosdata.com/
- https://www.taosdata.com/
-
-
-
-
- UTF-8
- 1.8
-
- 3.6.0
-
- 1.1.2
- 3.5
-
-
-
-
- commons-logging
- commons-logging
- ${commons-logging.version}
-
-
- *
- *
-
-
-
-
- org.apache.commons
- commons-lang3
- ${commons-lang3.version}
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-assembly-plugin
- 3.0.0
-
-
-
- src/main/assembly/assembly-jar.xml
-
-
-
-
- make-assembly
- package
-
- single
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- ${maven-compiler-plugin.version}
-
- UTF-8
- ${java.version}
- ${java.version}
- true
- true
-
-
-
-
-
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+ com.taosdata.jdbc
+ taos-jdbcdriver
+ 1.0.3
+ jar
+ JDBCDriver
+ https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc
+ TDengine JDBC Driver
+
+
+ GNU AFFERO GENERAL PUBLIC LICENSE Version 3
+ https://github.com/taosdata/TDengine/blob/master/LICENSE
+ repo
+
+
+
+ scm:git:git://github.com/taosdata/TDengine.git
+ scm:git:git@github.com:taosdata/TDengine.git
+ https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc
+ HEAD
+
+
+
+ taosdata
+ support@taosdata.com
+ https://www.taosdata.com/
+ https://www.taosdata.com/
+
+
+
+ UTF-8
+ 1.8
+ 3.6.0
+ 1.1.2
+ 3.5
+
+
+
+ commons-logging
+ commons-logging
+ ${commons-logging.version}
+
+
+ *
+ *
+
+
+
+
+ org.apache.commons
+ commons-lang3
+ ${commons-lang3.version}
+
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.8.3
+
+
+ junit
+ junit
+ 4.13
+ test
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 3.0.0
+
+
+ src/main/assembly/assembly-jar.xml
+
+
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ ${maven-compiler-plugin.version}
+
+ UTF-8
+ ${java.version}
+ ${java.version}
+ true
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 2.12.4
+
+ true
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.8.3
+
+
+ com/**/*
+
+
+
+
+ pre-test
+
+ prepare-agent
+
+
+
+ post-test
+ test
+
+ report
+
+
+
+
+
+
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
index 6cb3263e56749ffb9901fb0a401ab0be244c3646..804e09c6b349ca7c957447b9f1fdf1766f18b3d1 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
@@ -527,8 +527,9 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData {
public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types)
throws SQLException {
- if (conn != null && !conn.isClosed()) {
- Statement stmt = conn.createStatement();
+ Statement stmt = null;
+ if (null != conn && !conn.isClosed()) {
+ stmt = conn.createStatement();
if (catalog == null || catalog.length() < 1) {
catalog = conn.getCatalog();
}
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java
deleted file mode 100644
index bb1b2afd07f4d9478786d462188ca3c43334785f..0000000000000000000000000000000000000000
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/***************************************************************************
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- *****************************************************************************/
-package com.taosdata.jdbc;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- * TDengine error code and error message enumeration.
- *
- */
-public enum TSDBError {
-
- TSDB_CODE_SUCCESS(0, "success"),
- TSDB_CODE_ACTION_IN_PROGRESS(1, "in progress"),
- TSDB_CODE_LAST_SESSION_NOT_FINISHED(5, "last session not finished"),
- TSDB_CODE_INVALID_SESSION_ID(6, "invalid session ID"),
- TSDB_CODE_INVALID_TRAN_ID(7, "invalid tran ID"),
- TSDB_CODE_INVALID_MSG_TYPE(8, "invalid msg type"),
- TSDB_CODE_ALREADY_PROCESSED(9, "alredy processed"),
- TSDB_CODE_AUTH_FAILURE(10, "authentication failure"),
- TSDB_CODE_WRONG_MSG_SIZE(11, "wrong msg size"),
- TSDB_CODE_UNEXPECTED_RESPONSE(12, "unexpected response"),
- TSDB_CODE_INVALID_RESPONSE_TYPE(13, "invalid response type"),
- TSDB_CODE_NO_RESOURCE(14, "no resource"),
- TSDB_CODE_INVALID_TIME_STAMP(15, "invalid time stamp"),
- TSDB_CODE_MISMATCHED_METER_ID(16, "mismatched meter ID"),
- TSDB_CODE_ACTION_TRANS_NOT_FINISHED(17, "transcation not finished"),
- TSDB_CODE_ACTION_NOT_ONLINE(18, "not online"),
- TSDB_CODE_ACTION_SEND_FAILD(19, "send failed"),
- TSDB_CODE_NOT_ACTIVE_SESSION(20, "not active session"),
- TSDB_CODE_INSERT_FAILED(21, "insert failed"),
- TSDB_CODE_APP_ERROR(22, "App error"),
- TSDB_CODE_INVALID_IE(23, "invalid IE"),
- TSDB_CODE_INVALID_VALUE(24, "invalid value"),
- TSDB_CODE_REDIRECT(25, "service not available"),
- TSDB_CODE_ALREADY_THERE(26, "already there"),
- TSDB_CODE_INVALID_TABLE_ID(27, "invalid meter ID"),
- TSDB_CODE_INVALID_SQL(28, "invalid SQL"), // this message often comes with additional info which will vary based on the specific error situation
- TSDB_CODE_NETWORK_UNAVAIL(29, "failed to connect to server"),
- TSDB_CODE_INVALID_MSG_LEN(30, "invalid msg len"),
- TSDB_CODE_INVALID_DB(31, "invalid DB"),
- TSDB_CODE_INVALID_TABLE(32, "invalid table"),
- TSDB_CODE_DB_ALREADY_EXIST(33, "DB already there"),
- TSDB_CODE_TABLE_ALREADY_EXIST(34, "table already there"),
- TSDB_CODE_INVALID_USER(35, "invalid user name"),
- TSDB_CODE_INVALID_ACCT(36, "invalid acct name"),
- TSDB_CODE_INVALID_PASS(37, "invalid password"),
- TSDB_CODE_DB_NOT_SELECTED(38, "DB not selected"),
- TSDB_CODE_MEMORY_CORRUPTED(39, "memory corrupted"),
- TSDB_CODE_USER_ALREADY_EXIST(40, "user name exists"),
- TSDB_CODE_NO_RIGHTS(41, "not authorized"),
- TSDB_CODE_DISCONNECTED(42, "login disconnected), login again"),
- TSDB_CODE_NO_MASTER(43, "mgmt master node not available"),
- TSDB_CODE_NOT_CONFIGURED(44, "not configured"),
- TSDB_CODE_INVALID_OPTION(45, "invalid option"),
- TSDB_CODE_NODE_OFFLINE(46, "node offline"),
- TSDB_CODE_SYNC_REQUIRED(47, "sync required"),
- TSDB_CODE_NO_ENOUGH_DNODES(48, "more dnodes are needed"),
- TSDB_CODE_UNSYNCED(49, "node in unsynced state"),
- TSDB_CODE_TOO_SLOW(50, "too slow"),
- TSDB_CODE_OTHERS(51, "others"),
- TSDB_CODE_NO_REMOVE_MASTER(52, "can't remove dnode which is master"),
- TSDB_CODE_WRONG_SCHEMA(53, "wrong schema"),
- TSDB_CODE_NO_RESULT(54, "no results"),
- TSDB_CODE_TOO_MANY_USERS(55, "num of users execeed maxUsers"),
- TSDB_CODE_TOO_MANY_DATABSES(56, "num of databases execeed maxDbs"),
- TSDB_CODE_TOO_MANY_TABLES(57, "num of tables execeed maxTables"),
- TSDB_CODE_TOO_MANY_DNODES(58, "num of dnodes execeed maxDnodes"),
- TSDB_CODE_TOO_MANY_ACCTS(59, "num of accounts execeed maxAccts"),
- TSDB_CODE_ACCT_ALREADY_EXIST(60, "accout name exists"),
- TSDB_CODE_DNODE_ALREADY_EXIST(61, "dnode ip exists"),
- TSDB_CODE_SDB_ERROR(62, "sdb error"),
- TSDB_CODE_METRICMETA_EXPIRED(63, "metric meta expired"), // local cached metric-meta expired causes error in metric query
- TSDB_CODE_NOT_READY(64, "not ready"), // peer is not ready to process data
- TSDB_CODE_MAX_SESSIONS(65, "too many sessions on server"), // too many sessions
- TSDB_CODE_MAX_CONNECTIONS(66, "too many sessions from app"), // too many connections
- TSDB_CODE_SESSION_ALREADY_EXIST(67, "session to dest is already there"),
- TSDB_CODE_NO_QSUMMARY(68, "query list not there), please show again"),
- TSDB_CODE_SERV_OUT_OF_MEMORY(69, "server out of memory"),
- TSDB_CODE_INVALID_QHANDLE(70, "invalid query handle"),
- TSDB_CODE_RELATED_TABLES_EXIST(71, "tables related to metric exist"),
- TSDB_CODE_MONITOR_DB_FORBEIDDEN(72, "can't drop monitor database or tables"),
- TSDB_CODE_VG_COMMITLOG_INIT_FAILED(73, "commit log init failed"),
- TSDB_CODE_VG_INIT_FAILED(74, "vgroup init failed"),
- TSDB_CODE_DATA_ALREADY_IMPORTED(75, "data is already imported"),
- TSDB_CODE_OPS_NOT_SUPPORT(76, "not supported operation"),
- TSDB_CODE_INVALID_QUERY_ID(77, "invalid query id string"),
- TSDB_CODE_INVALID_STREAM_ID(78, "invalid stream id string"),
- TSDB_CODE_INVALID_CONNECTION(79, "invalid connection string"),
- TSDB_CODE_ACTION_NOT_BALANCED(80, "dnode not balanced"),
- TSDB_CODE_CLI_OUT_OF_MEMORY(81, "client out of memory"),
- TSDB_CODE_DATA_OVERFLOW(82, "data value overflow"),
- TSDB_CODE_QUERY_CANCELLED(83, "query cancelled"),
- TSDB_CODE_GRANT_POINT_LIMITED(84, "grant points limited"),
- TSDB_CODE_GRANT_EXPIRED(85, "grant expired"),
- TSDB_CODE_CLI_NO_DISKSPACE(86, "client no disk space"),
- TSDB_CODE_FILE_CORRUPTED(87, "DB file corrupted"),
- TSDB_CODE_INVALID_CLIENT_VERSION(88, "version of client and server not match");
-
- private long errCode;
- private String errMessage;
- private static Map errorCodeMap = new HashMap<>(86);
- static {
- errorCodeMap.put(0, "success");
- errorCodeMap.put(1, "in progress");
- errorCodeMap.put(5, "last session not finished");
- errorCodeMap.put(6, "invalid session ID");
- errorCodeMap.put(7, "invalid tran ID");
- errorCodeMap.put(8, "invalid msg type");
- errorCodeMap.put(9, "alredy processed");
- errorCodeMap.put(10, "authentication failure");
- errorCodeMap.put(11, "wrong msg size");
- errorCodeMap.put(12, "unexpected response");
- errorCodeMap.put(13, "invalid response type");
- errorCodeMap.put(14, "no resource");
- errorCodeMap.put(15, "invalid time stamp");
- errorCodeMap.put(16, "mismatched meter ID");
- errorCodeMap.put(17, "transcation not finished");
- errorCodeMap.put(18, "not online");
- errorCodeMap.put(19, "send failed");
- errorCodeMap.put(20, "not active session");
- errorCodeMap.put(21, "insert failed");
- errorCodeMap.put(22, "App error");
- errorCodeMap.put(23, "invalid IE");
- errorCodeMap.put(24, "invalid value");
- errorCodeMap.put(25, "service not available");
- errorCodeMap.put(26, "already there");
- errorCodeMap.put(27, "invalid meter ID");
- errorCodeMap.put(28, "invalid SQL"); // this message often comes with additional info which will vary based on the specific error situation
- errorCodeMap.put(29, "failed to connect to server");
- errorCodeMap.put(30, "invalid msg len");
- errorCodeMap.put(31, "invalid DB");
- errorCodeMap.put(32, "invalid table");
- errorCodeMap.put(33, "DB already there");
- errorCodeMap.put(34, "table already there");
- errorCodeMap.put(35, "invalid user name");
- errorCodeMap.put(36, "invalid acct name");
- errorCodeMap.put(37, "invalid password");
- errorCodeMap.put(38, "DB not selected");
- errorCodeMap.put(39, "memory corrupted");
- errorCodeMap.put(40, "user name exists");
- errorCodeMap.put(41, "not authorized");
- errorCodeMap.put(42, "login disconnected); login again");
- errorCodeMap.put(43, "mgmt master node not available");
- errorCodeMap.put(44, "not configured");
- errorCodeMap.put(45, "invalid option");
- errorCodeMap.put(46, "node offline");
- errorCodeMap.put(47, "sync required");
- errorCodeMap.put(48, "more dnodes are needed");
- errorCodeMap.put(49, "node in unsynced state");
- errorCodeMap.put(50, "too slow");
- errorCodeMap.put(51, "others");
- errorCodeMap.put(52, "can't remove dnode which is master");
- errorCodeMap.put(53, "wrong schema");
- errorCodeMap.put(54, "no results");
- errorCodeMap.put(55, "num of users execeed maxUsers");
- errorCodeMap.put(56, "num of databases execeed maxDbs");
- errorCodeMap.put(57, "num of tables execeed maxTables");
- errorCodeMap.put(58, "num of dnodes execeed maxDnodes");
- errorCodeMap.put(59, "num of accounts execeed maxAccts");
- errorCodeMap.put(60, "accout name exists");
- errorCodeMap.put(61, "dnode ip exists");
- errorCodeMap.put(62, "sdb error");
- errorCodeMap.put(63, "metric meta expired"); // local cached metric-meta expired causes error in metric query
- errorCodeMap.put(64, "not ready"); // peer is not ready to process data
- errorCodeMap.put(65, "too many sessions on server"); // too many sessions
- errorCodeMap.put(66, "too many sessions from app"); // too many connections
- errorCodeMap.put(67, "session to dest is already there");
- errorCodeMap.put(68, "query list not there); please show again");
- errorCodeMap.put(69, "server out of memory");
- errorCodeMap.put(70, "invalid query handle");
- errorCodeMap.put(71, "tables related to metric exist");
- errorCodeMap.put(72, "can't drop monitor database or tables");
- errorCodeMap.put(73, "commit log init failed");
- errorCodeMap.put(74, "vgroup init failed");
- errorCodeMap.put(75, "data is already imported");
- errorCodeMap.put(76, "not supported operation");
- errorCodeMap.put(77, "invalid query id string");
- errorCodeMap.put(78, "invalid stream id string");
- errorCodeMap.put(79, "invalid connection string");
- errorCodeMap.put(80, "dnode not balanced");
- errorCodeMap.put(81, "client out of memory");
- errorCodeMap.put(82, "data value overflow");
- errorCodeMap.put(83, "query cancelled");
- errorCodeMap.put(84, "grant points limited");
- errorCodeMap.put(85, "grant expired");
- errorCodeMap.put(86, "client no disk space");
- errorCodeMap.put(87, "DB file corrupted");
- errorCodeMap.put(88, "version of client and server not match");
- }
-
- TSDBError(long code, String message) {
- this.errCode = code;
- this.errMessage = message;
- }
-
- public long getErrCode() {
- return this.errCode;
- }
-
- public String getErrMessage() {
- return this.errMessage;
- }
-
- public static String getErrMessageByCode(long errCode) {
- return errorCodeMap.get(errCode);
- }
-
-}
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
index 8bb7084604851c695961564ed2f8c0142accb7fb..e9acffe0e3ca581f04f9a2677430dc43b0089832 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
@@ -19,7 +19,7 @@ import java.sql.SQLWarning;
import java.util.List;
public class TSDBJNIConnector {
- static volatile Boolean isInitialized = false;
+ private static volatile Boolean isInitialized = false;
static {
System.loadLibrary("taos");
@@ -51,8 +51,6 @@ public class TSDBJNIConnector {
/**
* Returns the status of last result set in current connection
- *
- * @return
*/
public boolean isResultsetClosed() {
return this.isResultsetClosed;
@@ -112,7 +110,7 @@ public class TSDBJNIConnector {
*
* @throws SQLException
*/
- public int executeQuery(String sql) throws SQLException {
+ public long executeQuery(String sql) throws SQLException {
if (!this.isResultsetClosed) {
freeResultSet(taosResultSetPointer);
}
@@ -127,7 +125,6 @@ public class TSDBJNIConnector {
}
int code = this.getErrCode(pSql);
- affectedRows = code;
if (code < 0) {
affectedRows = -1;
if (code == TSDBConstants.JNI_TDENGINE_ERROR) {
@@ -146,7 +143,7 @@ public class TSDBJNIConnector {
if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) {
isResultsetClosed = false;
}
- return code;
+ return pSql;
}
private native long executeQueryImp(byte[] sqlBytes, long connection);
@@ -199,8 +196,6 @@ public class TSDBJNIConnector {
/**
* Close the open result set which is associated to the current connection. If the result set is already
* closed, return 0 for success.
- *
- * @return
*/
public int freeResultSet() {
int resCode = TSDBConstants.JNI_SUCCESS;
@@ -217,7 +212,7 @@ public class TSDBJNIConnector {
/**
* Get affected rows count
*/
- public int getAffectedRows(Long pSql) {
+ public int getAffectedRows(long pSql) {
int affectedRows = this.affectedRows;
if (affectedRows < 0) {
affectedRows = this.getAffectedRowsImp(this.taos, pSql);
@@ -225,7 +220,7 @@ public class TSDBJNIConnector {
return affectedRows;
}
- private native int getAffectedRowsImp(long connection, Long pSql);
+ private native int getAffectedRowsImp(long connection, long pSql);
/**
* Get schema metadata
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
index dbe1e41e6be861f8cf45d9dba2614cb36c527197..1cf024f30e4a49b79f12c0af8cb29e0416bd348b 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
@@ -51,13 +51,16 @@ public class TSDBStatement implements Statement {
if (isClosed) {
throw new SQLException("Invalid method call on a closed statement.");
}
- this.connecter.executeQuery(sql);
+ pSql = this.connecter.executeQuery(sql);
long resultSetPointer = this.connecter.getResultSet();
if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
+ this.connecter.freeResultSet(pSql);
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
} else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) {
+// create/insert/update/del/alter
+ this.connecter.freeResultSet(pSql);
return null;
} else {
return new TSDBResultSet(this.connecter, resultSetPointer);
@@ -68,17 +71,18 @@ public class TSDBStatement implements Statement {
if (isClosed) {
throw new SQLException("Invalid method call on a closed statement.");
}
- long res = this.connecter.executeQuery(sql);
+ pSql = this.connecter.executeQuery(sql);
long resultSetPointer = this.connecter.getResultSet();
if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
- this.connecter.freeResultSet(res);
+ this.connecter.freeResultSet(pSql);
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
} else if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) {
this.connecter.freeResultSet();
throw new SQLException("The executed SQL is not a DML or a DDL");
} else {
- int num = this.connecter.getAffectedRows(res);
+ int num = this.connecter.getAffectedRows(pSql);
+ this.connecter.freeResultSet(pSql);
return num;
}
}
@@ -146,16 +150,19 @@ public class TSDBStatement implements Statement {
throw new SQLException("Invalid method call on a closed statement.");
}
boolean res = true;
- this.connecter.executeQuery(sql);
+ pSql = this.connecter.executeQuery(sql);
long resultSetPointer = this.connecter.getResultSet();
if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
+ this.connecter.freeResultSet(pSql);
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
} else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) {
// no result set is retrieved
res = false;
}
- return res;
+ this.connecter.freeResultSet(pSql);
+
+ return res;
}
public ResultSet getResultSet() throws SQLException {
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java
index 3b479aafc35a50e85e42d9cda1e93c2d68f8e115..e20c6a815c190cfdc8bf01e7b0e1afe676c3790f 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java
@@ -171,8 +171,7 @@ public class TSDBSubscribe {
state = 1;
try {
- TSDBResultSet resultSet = consume(subscription);
- callBack.invoke(resultSet);
+ callBack.invoke(consume(subscription));
} catch (Exception e) {
this.cancel();
throw new RuntimeException(e);
diff --git a/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java
index 5b2b6367ec5fe7cb46f0514b59931d8942f9bc74..55ab2fdc52ab183264088381ca4f2e475c0c4c08 100644
--- a/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java
+++ b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java
@@ -3,10 +3,11 @@ import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.Properties;
public class TestAsyncTSDBSubscribe {
- public static void main(String[] args) {
+ public static void main(String[] args) throws SQLException {
String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName -topic topicName " +
"-tname tableName -h host";
if (args.length < 2) {
@@ -38,15 +39,18 @@ public class TestAsyncTSDBSubscribe {
}
Connection connection = null;
- TSDBSubscribe subscribe = null;
long subscribId = 0;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
- properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties);
String rawSql = "select * from " + tName + ";";
- subscribe = ((TSDBConnection) connection).createSubscribe();
+ TSDBSubscribe subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first"));
long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second"));
int a = 0;
@@ -55,6 +59,9 @@ public class TestAsyncTSDBSubscribe {
System.err.println("cancel subscribe");
} catch (Exception e) {
e.printStackTrace();
+ if (null != connection && !connection.isClosed()) {
+ connection.close();
+ }
}
}
diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java
index 3b84645b5b0f250c453615d0afbd1f018bbe523f..2e387206a41997eb629ead85b55fef9320ea7d14 100644
--- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java
+++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java
@@ -6,24 +6,31 @@ import java.util.Properties;
public class TestPreparedStatement {
- public static void main(String[] args) {
+ public static void main(String[] args) throws SQLException {
+ Connection connection = null;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
- Connection connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties);
+ connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties);
String rawSql = "select * from test.log0601";
// String[] params = new String[]{"ts", "c1"};
PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql);
ResultSet resSet = pstmt.executeQuery();
while(resSet.next()) {
for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
- System.out.printf("%d: %s\n", i, resSet.getString(i));
+ System.out.printf("%d: %s \n", i, resSet.getString(i));
}
}
+ resSet.close();
+ pstmt.close();
+ connection.close();
} catch (Exception e) {
e.printStackTrace();
+ if (null != connection) {
+ connection.close();
+ }
}
}
}
diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java
index 519d7c6ab1f89f08795c1ff010bc5de1e5491f05..f7e0e78441a9c2457a57605fe71e70a5dee609c2 100644
--- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java
+++ b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java
@@ -1,29 +1,33 @@
import com.taosdata.jdbc.TSDBDriver;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
+import java.sql.*;
import java.util.Properties;
public class TestTSDBDatabaseMetaData {
- public static void main(String[] args) {
+ public static void main(String[] args) throws SQLException {
+ Connection connection = null;
+ DatabaseMetaData dbMetaData = null;
+ ResultSet resSet = null;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
- properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.114");
- Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.114:0/?user=root&password=taosdata", properties);
- DatabaseMetaData dbMetaData = connection.getMetaData();
- ResultSet resSet = dbMetaData.getCatalogs();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
+ connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties);
+ dbMetaData = connection.getMetaData();
+ resSet = dbMetaData.getCatalogs();
while(resSet.next()) {
for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
System.out.printf("dbMetaData.getCatalogs(%d) = %s\n", i, resSet.getString(i));
}
}
+ resSet.close();
} catch (Exception e) {
e.printStackTrace();
+ if (null != connection) {
+ connection.close();
+ }
}
}
}
diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java
index f12924c8a61a3dbeeec56f7d2c712472d771e8e7..f628f3cada5ef1542fc756184d6d6ca4955cd422 100644
--- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java
+++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java
@@ -46,16 +46,20 @@ public class TestTSDBSubscribe {
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
- properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata"
, properties);
String rawSql = "select * from " + tName + ";";
subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000);
int a = 0;
+ TSDBResultSet resSet = null;
while (true) {
Thread.sleep(900);
- TSDBResultSet resSet = subscribe.consume(subscribId);
+ resSet = subscribe.consume(subscribId);
while (resSet.next()) {
for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/AsyncSubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/AsyncSubscribeTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..98b0d38b8d31751873f57be5afc99b6cd4a92c43
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/AsyncSubscribeTest.java
@@ -0,0 +1,99 @@
+package com.taosdata.jdbc;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+public class AsyncSubscribeTest {
+ Connection connection = null;
+ Statement statement = null;
+ String dbName = "test";
+ String tName = "t0";
+ String host = "localhost";
+ String topic = "test";
+ long subscribId = 0;
+
+ @Before
+ public void createDatabase() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
+ long ts = System.currentTimeMillis();
+ for (int i = 0; i < 2; i++) {
+ ts += i;
+ statement.executeUpdate("insert into \" + dbName + \".\" + tName + \" values (" + ts + ", " + (100 + i) + ", " + i + ")");
+ }
+ }
+
+ @Test
+ public void subscribe() throws Exception {
+ TSDBSubscribe subscribe = null;
+ try {
+ String rawSql = "select * from " + dbName + "." + tName + ";";
+ System.out.println(rawSql);
+ subscribe = ((TSDBConnection) connection).createSubscribe();
+ subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first"));
+
+ assertTrue(subscribId > 0);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ Thread.sleep(2000);
+ subscribe.unsubscribe(subscribId, true);
+ }
+
+ private static class CallBack implements TSDBSubscribeCallBack {
+ private String name = "";
+
+ public CallBack(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void invoke(TSDBResultSet resultSet) {
+ try {
+ while (null != resultSet && resultSet.next()) {
+ System.out.print("callback_" + name + ": ");
+ for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
+ System.out.printf(i + ": " + resultSet.getString(i) + "\t");
+ }
+ System.out.println();
+ }
+ resultSet.close();
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @After
+ public void close() throws Exception {
+ statement.executeQuery("drop database test");
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+ }
+}
\ No newline at end of file
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..5e5dabfe81722a45ea94858eaf833f2355cc528c
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java
@@ -0,0 +1,62 @@
+package com.taosdata.jdbc;
+
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+public class ConnectionTest {
+ static Connection connection = null;
+ static Statement statement = null;
+ static String dbName = "test";
+ static String stbName = "st";
+ static String host = "localhost";
+
+ @Test
+ public void testConnection() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ assertTrue(null != connection);
+ statement = connection.createStatement();
+ assertTrue(null != statement);
+
+ // try reconnect
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ try {
+ statement.execute("create database if not exists " + dbName);
+ } catch (SQLException e) {
+ assert false : "create database error: " + e.getMessage();
+ }
+
+ try {
+ if (!connection.isClosed()) {
+ if (!statement.isClosed()) {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ }
+ connection.close();
+ Thread.sleep(10);
+ }
+ } catch (Exception e) {
+ assert false : "close connection error: " + e.getMessage();
+ }
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..20ebc093ec82e46c0e09b9a4ebc6289300cf1594
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java
@@ -0,0 +1,245 @@
+package com.taosdata.jdbc;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.Properties;
+
+public class DatabaseMetaDataTest {
+ static Connection connection = null;
+ static PreparedStatement statement = null;
+ static String dbName = "test";
+ static String tName = "t0";
+ static String host = "localhost";
+
+ @BeforeClass
+ public static void createConnection() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata",
+ properties);
+
+ String sql = "drop database if exists " + dbName;
+ statement = (TSDBPreparedStatement) connection.prepareStatement(sql);
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
+ }
+
+ @Test
+ public void testMetaDataTest() throws SQLException {
+
+ DatabaseMetaData databaseMetaData = connection.getMetaData();
+ ResultSet resultSet = databaseMetaData.getTables(dbName, "t*", "t*", new String[]{"t"});
+ while (resultSet.next()) {
+ for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
+ System.out.printf("%d: %s\n", i, resultSet.getString(i));
+ }
+ }
+ resultSet.close();
+ databaseMetaData.isWrapperFor(null);
+ databaseMetaData.allProceduresAreCallable();
+ databaseMetaData.allTablesAreSelectable();
+ databaseMetaData.getURL();
+ databaseMetaData.getUserName();
+ databaseMetaData.isReadOnly();
+ databaseMetaData.nullsAreSortedHigh();
+ databaseMetaData.nullsAreSortedLow();
+ databaseMetaData.nullsAreSortedAtStart();
+ databaseMetaData.nullsAreSortedAtEnd();
+ databaseMetaData.getDatabaseProductName();
+ databaseMetaData.getDatabaseProductVersion();
+ databaseMetaData.getDriverName();
+ databaseMetaData.getDriverVersion();
+ databaseMetaData.getDriverMajorVersion();
+ databaseMetaData.getDriverMinorVersion();
+ databaseMetaData.usesLocalFiles();
+ databaseMetaData.usesLocalFilePerTable();
+ databaseMetaData.supportsMixedCaseIdentifiers();
+ databaseMetaData.storesUpperCaseIdentifiers();
+ databaseMetaData.storesLowerCaseIdentifiers();
+ databaseMetaData.storesMixedCaseIdentifiers();
+ databaseMetaData.supportsMixedCaseQuotedIdentifiers();
+ databaseMetaData.storesUpperCaseQuotedIdentifiers();
+ databaseMetaData.storesLowerCaseQuotedIdentifiers();
+ databaseMetaData.storesMixedCaseQuotedIdentifiers();
+ databaseMetaData.getIdentifierQuoteString();
+ databaseMetaData.getSQLKeywords();
+ databaseMetaData.getNumericFunctions();
+ databaseMetaData.getStringFunctions();
+ databaseMetaData.getSystemFunctions();
+ databaseMetaData.getTimeDateFunctions();
+ databaseMetaData.getSearchStringEscape();
+ databaseMetaData.getExtraNameCharacters();
+ databaseMetaData.supportsAlterTableWithAddColumn();
+ databaseMetaData.supportsAlterTableWithDropColumn();
+ databaseMetaData.supportsColumnAliasing();
+ databaseMetaData.nullPlusNonNullIsNull();
+ databaseMetaData.supportsConvert();
+ databaseMetaData.supportsConvert(0, 0);
+ databaseMetaData.supportsTableCorrelationNames();
+ databaseMetaData.supportsDifferentTableCorrelationNames();
+ databaseMetaData.supportsExpressionsInOrderBy();
+ databaseMetaData.supportsOrderByUnrelated();
+ databaseMetaData.supportsGroupBy();
+ databaseMetaData.supportsGroupByUnrelated();
+ databaseMetaData.supportsGroupByBeyondSelect();
+ databaseMetaData.supportsLikeEscapeClause();
+ databaseMetaData.supportsMultipleResultSets();
+ databaseMetaData.supportsMultipleTransactions();
+ databaseMetaData.supportsNonNullableColumns();
+ databaseMetaData.supportsMinimumSQLGrammar();
+ databaseMetaData.supportsCoreSQLGrammar();
+ databaseMetaData.supportsExtendedSQLGrammar();
+ databaseMetaData.supportsANSI92EntryLevelSQL();
+ databaseMetaData.supportsANSI92IntermediateSQL();
+ databaseMetaData.supportsANSI92FullSQL();
+ databaseMetaData.supportsIntegrityEnhancementFacility();
+ databaseMetaData.supportsOuterJoins();
+ databaseMetaData.supportsFullOuterJoins();
+ databaseMetaData.supportsLimitedOuterJoins();
+ databaseMetaData.getSchemaTerm();
+ databaseMetaData.getProcedureTerm();
+ databaseMetaData.getCatalogTerm();
+ databaseMetaData.isCatalogAtStart();
+ databaseMetaData.getCatalogSeparator();
+ databaseMetaData.supportsSchemasInDataManipulation();
+ databaseMetaData.supportsSchemasInProcedureCalls();
+ databaseMetaData.supportsSchemasInTableDefinitions();
+ databaseMetaData.supportsSchemasInIndexDefinitions();
+ databaseMetaData.supportsSchemasInPrivilegeDefinitions();
+ databaseMetaData.supportsCatalogsInDataManipulation();
+ databaseMetaData.supportsCatalogsInProcedureCalls();
+ databaseMetaData.supportsCatalogsInTableDefinitions();
+ databaseMetaData.supportsCatalogsInIndexDefinitions();
+ databaseMetaData.supportsCatalogsInPrivilegeDefinitions();
+ databaseMetaData.supportsPositionedDelete();
+ databaseMetaData.supportsPositionedUpdate();
+ databaseMetaData.supportsSelectForUpdate();
+ databaseMetaData.supportsStoredProcedures();
+ databaseMetaData.supportsSubqueriesInComparisons();
+ databaseMetaData.supportsSubqueriesInExists();
+ databaseMetaData.supportsSubqueriesInIns();
+ databaseMetaData.supportsSubqueriesInQuantifieds();
+ databaseMetaData.supportsCorrelatedSubqueries();
+ databaseMetaData.supportsUnion();
+ databaseMetaData.supportsUnionAll();
+ databaseMetaData.supportsOpenCursorsAcrossCommit();
+ databaseMetaData.supportsOpenCursorsAcrossRollback();
+ databaseMetaData.supportsOpenStatementsAcrossCommit();
+ databaseMetaData.supportsOpenStatementsAcrossRollback();
+ databaseMetaData.getMaxBinaryLiteralLength();
+ databaseMetaData.getMaxCharLiteralLength();
+ databaseMetaData.getMaxColumnNameLength();
+ databaseMetaData.getMaxColumnsInGroupBy();
+ databaseMetaData.getMaxColumnsInIndex();
+ databaseMetaData.getMaxColumnsInOrderBy();
+ databaseMetaData.getMaxColumnsInSelect();
+ databaseMetaData.getMaxColumnsInTable();
+ databaseMetaData.getMaxConnections();
+ databaseMetaData.getMaxCursorNameLength();
+ databaseMetaData.getMaxIndexLength();
+ databaseMetaData.getMaxSchemaNameLength();
+ databaseMetaData.getMaxProcedureNameLength();
+ databaseMetaData.getMaxCatalogNameLength();
+ databaseMetaData.getMaxRowSize();
+ databaseMetaData.doesMaxRowSizeIncludeBlobs();
+ databaseMetaData.getMaxStatementLength();
+ databaseMetaData.getMaxStatements();
+ databaseMetaData.getMaxTableNameLength();
+ databaseMetaData.getMaxTablesInSelect();
+ databaseMetaData.getMaxUserNameLength();
+ databaseMetaData.getDefaultTransactionIsolation();
+ databaseMetaData.supportsTransactions();
+ databaseMetaData.supportsTransactionIsolationLevel(0);
+ databaseMetaData.supportsDataDefinitionAndDataManipulationTransactions();
+ databaseMetaData.supportsDataManipulationTransactionsOnly();
+ databaseMetaData.dataDefinitionCausesTransactionCommit();
+ databaseMetaData.dataDefinitionIgnoredInTransactions();
+ try {
+ databaseMetaData.getProcedures("", "", "");
+ } catch (Exception e) {
+ }
+ try {
+ databaseMetaData.getProcedureColumns("", "", "", "");
+ } catch (Exception e) {
+ }
+ try {
+ databaseMetaData.getTables("", "", "", new String[]{""});
+ } catch (Exception e) {
+ }
+ databaseMetaData.getSchemas();
+ databaseMetaData.getCatalogs();
+// databaseMetaData.getTableTypes();
+
+ databaseMetaData.getColumns("", "", "", "");
+ databaseMetaData.getColumnPrivileges("", "", "", "");
+ databaseMetaData.getTablePrivileges("", "", "");
+ databaseMetaData.getBestRowIdentifier("", "", "", 0, false);
+ databaseMetaData.getVersionColumns("", "", "");
+ databaseMetaData.getPrimaryKeys("", "", "");
+ databaseMetaData.getImportedKeys("", "", "");
+ databaseMetaData.getExportedKeys("", "", "");
+ databaseMetaData.getCrossReference("", "", "", "", "", "");
+ databaseMetaData.getTypeInfo();
+ databaseMetaData.getIndexInfo("", "", "", false, false);
+ databaseMetaData.supportsResultSetType(0);
+ databaseMetaData.supportsResultSetConcurrency(0, 0);
+ databaseMetaData.ownUpdatesAreVisible(0);
+ databaseMetaData.ownDeletesAreVisible(0);
+ databaseMetaData.ownInsertsAreVisible(0);
+ databaseMetaData.othersUpdatesAreVisible(0);
+ databaseMetaData.othersDeletesAreVisible(0);
+ databaseMetaData.othersInsertsAreVisible(0);
+ databaseMetaData.updatesAreDetected(0);
+ databaseMetaData.deletesAreDetected(0);
+ databaseMetaData.insertsAreDetected(0);
+ databaseMetaData.supportsBatchUpdates();
+ databaseMetaData.getUDTs("", "", "", new int[]{0});
+ databaseMetaData.getConnection();
+ databaseMetaData.supportsSavepoints();
+ databaseMetaData.supportsNamedParameters();
+ databaseMetaData.supportsMultipleOpenResults();
+ databaseMetaData.supportsGetGeneratedKeys();
+ databaseMetaData.getSuperTypes("", "", "");
+ databaseMetaData.getSuperTables("", "", "");
+ databaseMetaData.getAttributes("", "", "", "");
+ databaseMetaData.supportsResultSetHoldability(0);
+ databaseMetaData.getResultSetHoldability();
+ databaseMetaData.getDatabaseMajorVersion();
+ databaseMetaData.getDatabaseMinorVersion();
+ databaseMetaData.getJDBCMajorVersion();
+ databaseMetaData.getJDBCMinorVersion();
+ databaseMetaData.getSQLStateType();
+ databaseMetaData.locatorsUpdateCopy();
+ databaseMetaData.supportsStatementPooling();
+ databaseMetaData.getRowIdLifetime();
+ databaseMetaData.getSchemas("", "");
+ databaseMetaData.supportsStoredFunctionsUsingCallSyntax();
+ databaseMetaData.autoCommitFailureClosesAllResultSets();
+ databaseMetaData.getClientInfoProperties();
+ databaseMetaData.getFunctions("", "", "");
+ databaseMetaData.getFunctionColumns("", "", "", "");
+ databaseMetaData.getPseudoColumns("", "", "", "");
+ databaseMetaData.generatedKeyAlwaysReturned();
+
+ }
+
+ @AfterClass
+ public static void close() throws Exception {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b42d46aefecce035770ca3dbb71ce7342647b6f2
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java
@@ -0,0 +1,93 @@
+package com.taosdata.jdbc;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+public class ImportTest {
+ Connection connection = null;
+ Statement statement = null;
+ String dbName = "test";
+ String tName = "t0";
+ String host = "localhost";
+
+ @Before
+ public void createDatabase() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("drop database if exists " + dbName);
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
+
+ }
+
+ @Test
+ public void insertData() throws Exception {
+ long ts = 1496732686000l;
+
+ for (int i = 0; i < 50; i++) {
+ ts++;
+ int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")");
+ System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row);
+ assertEquals(1, row);
+ }
+ }
+
+ @Test
+ public void selectData() throws Exception {
+ insertData();
+ String sql = "select * from test.t0";
+ ResultSet resSet = statement.executeQuery(sql);
+
+ while (resSet.next()) {
+ for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
+ System.out.printf(i + ": " + resSet.getString(i) + "\t");
+ }
+ }
+ resSet.close();
+ }
+
+ @Test
+ public void importData() throws Exception {
+ // 避免时间重复
+ long ts = 1496732686000l;
+
+ StringBuilder sqlBuilder = new StringBuilder("insert into ").append(dbName).append(".").append(tName).append(" values ");
+
+ for (int i = 0; i < 50; i++) {
+ int a = i / 5;
+ long t = ts + a;
+ sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") ");
+ }
+ System.out.println(sqlBuilder.toString());
+ int rows = statement.executeUpdate(sqlBuilder.toString());
+ System.out.println(rows);
+ assertEquals(10, rows);
+ }
+
+ @After
+ public void close() throws Exception {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4c49727a7f0f0dee26c1e0a9b5c31a7a3df0724f
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java
@@ -0,0 +1,201 @@
+package com.taosdata.jdbc;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import java.sql.*;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@FixMethodOrder(MethodSorters.DEFAULT)
+public class PreparedStatementTest {
+ static Connection connection = null;
+ static PreparedStatement statement = null;
+ static String dbName = "test";
+ static String tName = "t0";
+ static String host = "localhost";
+ static ResultSet resSet = null;
+
+ @BeforeClass
+ public static void createConnection() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata",
+ properties);
+
+ String sql = "drop database if exists " + dbName;
+ statement = (TSDBPreparedStatement) connection.prepareStatement(sql);
+
+ }
+
+ @Test
+ public void createTableAndQuery() throws SQLException {
+ long ts = System.currentTimeMillis();
+
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)");
+ statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)");
+
+ PreparedStatement selectStatement = connection.prepareStatement("select * from " + dbName + "." + tName);
+
+ ResultSet resultSet = selectStatement.executeQuery();
+ assertTrue(null != resultSet);
+
+ boolean isClosed = statement.isClosed();
+ assertEquals(false, isClosed);
+ }
+
+ @Test
+ public void testPreparedStatement() throws SQLException {
+ long ts = System.currentTimeMillis() + 20000;
+ PreparedStatement saveStatement = connection
+ .prepareStatement("insert into " + dbName + "." + tName + " values (" + ts + ", 1)");
+
+ int affectedRows = saveStatement.executeUpdate();
+ assertTrue(1 == affectedRows);
+ }
+
+ @Test
+ public void testSavedPreparedStatement() throws SQLException {
+ long ts = System.currentTimeMillis();
+
+ TSDBPreparedStatement saveStatement = (TSDBPreparedStatement) connection
+ .prepareStatement("insert into " + dbName + "." + tName + " values (?, ?)");
+
+ saveStatement.setObject(1, ts + 10000);
+ saveStatement.setObject(2, 3);
+ int rows = saveStatement.executeUpdate();
+ assertEquals(1, rows);
+ }
+
+ @Test
+ public void testUnsupport() {
+ // if(null == resSet) {
+ // return;
+ // }
+ TSDBPreparedStatement tsdbStatement = (TSDBPreparedStatement) statement;
+ try {
+ tsdbStatement.unwrap(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isWrapperFor(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMaxFieldSize();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setMaxFieldSize(0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setEscapeProcessing(true);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.cancel();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getWarnings();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.clearWarnings();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setCursorName(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMoreResults();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setFetchDirection(0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getFetchDirection();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetConcurrency();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetType();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getConnection();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMoreResults();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getGeneratedKeys();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, 0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, new int[]{0});
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"});
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetHoldability();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setPoolable(true);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isPoolable();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.closeOnCompletion();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isCloseOnCompletion();
+ } catch (SQLException e) {
+ }
+ }
+
+ @AfterClass
+ public static void close() throws Exception {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2b74a98db2304b517f01f75cb5e0ca13d8858cce
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java
@@ -0,0 +1,829 @@
+package com.taosdata.jdbc;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.sql.rowset.serial.SerialBlob;
+import javax.sql.rowset.serial.SerialClob;
+import java.sql.*;
+import java.util.HashMap;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ResultSetTest {
+ static Connection connection = null;
+ static Statement statement = null;
+ static String dbName = "test";
+ static String tName = "t0";
+ static String host = "localhost";
+ static ResultSet resSet = null;
+
+ @BeforeClass
+ public static void createDatabaseAndTable() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("drop database if exists " + dbName);
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName +
+ " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))");
+
+ statement.executeQuery("use " + dbName);
+ }
+
+ @Test
+ public void testResultSet() {
+ String sql = null;
+ long ts = 1496732686000l;
+ int v1 = 2147483600;
+ long v2 = ts + 1000;
+ float v3 = 3.1415926f;
+ double v4 = 3.1415926535897;
+ String v5 = "涛思数据，强~！";
+ short v6 = 12;
+ boolean v7 = false;
+ String v8 = "TDengine is powerful";
+
+ sql = "insert into " + dbName + "." + tName + " values (" + ts + "," + v1 + "," + v2 + "," + v3 + "," + v4
+ + ",\"" + v5 + "\"," + v6 + "," + v7 + ",\"" + v8 + "\")";
+
+ try {
+ statement.executeUpdate(sql);
+ assertEquals(1, statement.getUpdateCount());
+ } catch (SQLException e) {
+ assert false : "insert error " + e.getMessage();
+ }
+
+ try {
+ statement.executeQuery("select * from " + dbName + "." + tName);
+ resSet = statement.getResultSet();
+ System.out.println(((TSDBResultSet) resSet).getRowData());
+ while (resSet.next()) {
+ assertEquals(ts, resSet.getLong(1));
+ assertEquals(ts, resSet.getLong("ts"));
+
+ System.out.println(resSet.getTimestamp(1));
+
+ assertEquals(v1, resSet.getInt(2));
+ assertEquals(v1, resSet.getInt("k1"));
+
+ assertEquals(v2, resSet.getLong(3));
+ assertEquals(v2, resSet.getLong("k2"));
+
+ assertEquals(v3, resSet.getFloat(4), 7);
+ assertEquals(v3, resSet.getFloat("k3"), 7);
+
+ assertEquals(v4, resSet.getDouble(5), 13);
+ assertEquals(v4, resSet.getDouble("k4"), 13);
+
+ assertEquals(v5, resSet.getString(6));
+ assertEquals(v5, resSet.getString("k5"));
+
+ assertEquals(v6, resSet.getShort(7));
+ assertEquals(v6, resSet.getShort("k6"));
+
+ assertEquals(v7, resSet.getBoolean(8));
+ assertEquals(v7, resSet.getBoolean("k7"));
+
+ assertEquals(v8, resSet.getString(9));
+ assertEquals(v8, resSet.getString("k8"));
+
+ resSet.getBytes(9);
+ resSet.getObject(6);
+ resSet.getObject("k8");
+ }
+ if (!resSet.isClosed()) {
+ resSet.close();
+ }
+ } catch (SQLException e) {
+ assert false : "insert error " + e.getMessage();
+ }
+ }
+
+ @Test
+ public void testUnsupport() throws SQLException {
+ statement.executeQuery("show databases");
+ resSet = statement.getResultSet();
+ try {
+ resSet.unwrap(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.isWrapperFor(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getAsciiStream(0);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getUnicodeStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getBinaryStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getAsciiStream("");
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getUnicodeStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getBinaryStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getWarnings();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.clearWarnings();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getCursorName();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getCharacterStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getCharacterStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.isBeforeFirst();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.isAfterLast();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.isFirst();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.isLast();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.beforeFirst();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.afterLast();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.first();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.last();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.absolute(1);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.relative(1);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.previous();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.setFetchDirection(0);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getFetchDirection();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.setFetchSize(0);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getFetchSize();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getConcurrency();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.rowUpdated();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.rowInserted();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.rowDeleted();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNull(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBoolean(0, true);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateByte(0, (byte) 2);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateShort(0, (short) 1);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateInt(0, 0);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateLong(0, 0l);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateFloat(0, 3.14f);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateDouble(0, 3.1415);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBigDecimal(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateString(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBytes(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateDate(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateTime(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateTimestamp(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateObject(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateObject(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNull(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBoolean("", false);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateByte("", (byte) 1);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateShort("", (short) 1);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateInt("", 0);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateLong("", 0l);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateFloat("", 3.14f);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateDouble("", 3.1415);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBigDecimal(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateString(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBytes(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateDate(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateTime(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateTimestamp(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateObject(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateObject(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.insertRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.deleteRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.refreshRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.cancelRowUpdates();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.moveToInsertRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.moveToCurrentRow();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getStatement();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getObject(0, new HashMap<>());
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getRef(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getBlob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getClob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getArray(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getObject("", new HashMap<>());
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getRef(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getBlob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getClob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getArray(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getDate(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getDate(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getTime(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getTime(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getTimestamp(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getTimestamp(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getURL(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getURL(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateRef(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateRef(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBlob(0, new SerialBlob("".getBytes("UTF8")));
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBlob("", new SerialBlob("".getBytes("UTF8")));
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateClob("", new SerialClob("".toCharArray()));
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateClob(0, new SerialClob("".toCharArray()));
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateArray(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateArray(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getRowId(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getRowId(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateRowId(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateRowId(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getHoldability();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNString(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNString(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+
+ try {
+ resSet.getNClob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getNClob(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getSQLXML(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getSQLXML(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateSQLXML(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateSQLXML(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getNCharacterStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.getNCharacterStream(null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+
+ try {
+ resSet.updateNCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateNCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateAsciiStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateBinaryStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ try {
+ resSet.updateCharacterStream(null, null);
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("this operation is NOT supported currently!"));
+ }
+ }
+
+ @Test
+ public void testBatch() throws SQLException {
+ String[] sqls = new String[]{"insert into test.t0 values (1496732686001,2147483600,1496732687000,3.1415925,3.1415926\n" +
+ "535897,\"涛思数据，强~！\",12,12,\"TDengine is powerful\")", "insert into test.t0 values (1496732686002,2147483600,1496732687000,3.1415925,3.1415926\n" +
+ "535897,\"涛思数据，强~！\",12,12,\"TDengine is powerful\")"};
+ for (String sql : sqls) {
+ statement.addBatch(sql);
+ }
+ int[] res = statement.executeBatch();
+ assertEquals(res.length, 2);
+ statement.clearBatch();
+ }
+
+ @AfterClass
+ public static void close() throws Exception {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..282ca6c1a7ca404a2c77d9ed1165f0cda606ebfb
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java
@@ -0,0 +1,72 @@
+package com.taosdata.jdbc;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+public class SelectTest {
+ Connection connection = null;
+ Statement statement = null;
+ String dbName = "test";
+ String tName = "t0";
+ String host = "localhost";
+
+ @Before
+ public void createDatabaseAndTable() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("drop database if exists " + dbName);
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
+
+ }
+
+ @Test
+ public void selectData() throws SQLException {
+ long ts = 1496732686000l;
+
+ for (int i = 0; i < 50; i++) {
+ ts++;
+ int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")");
+ System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row);
+ assertEquals(1, row);
+ }
+
+ String sql = "select * from " + dbName + "." + tName;
+ ResultSet resSet = statement.executeQuery(sql);
+
+ int num = 0;
+ while (resSet.next()) {
+ num++;
+ }
+ resSet.close();
+
+ assertEquals(num, 50);
+ }
+
+ @After
+ public void close() throws Exception {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b275112e186bffa0fe90b468f71e04354f4eaed3
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java
@@ -0,0 +1,123 @@
+package com.taosdata.jdbc;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import java.sql.*;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class StableTest {
+ static Connection connection = null;
+ static Statement statement = null;
+ static String dbName = "test";
+ static String stbName = "st";
+ static String host = "localhost";
+
+ @BeforeClass
+ public static void createDatabase() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeQuery("use " + dbName);
+ }
+
+// @Test
+ public void createStable() {
+ String sql = "create table " + stbName + " (ts timestamp, v1 int, v2 int) tags (tg nchar(20)) ";
+
+ try {
+ statement.executeUpdate(sql);
+ } catch (SQLException e) {
+ assert false : "error create stable" + e.getMessage();
+ }
+ }
+
+// @Test
+ public void createTable() {
+ String sql = "create table t1 using " + stbName + " tags (\"beijing\")";
+
+ try {
+ statement.executeUpdate(sql);
+ } catch (SQLException e) {
+ assert false : "error create table" + e.getMessage();
+ }
+ }
+
+ @Test
+ public void describeSTable() {
+ createStable();
+ String sql = "describe " + stbName;
+ int num = 0;
+ System.out.println("describe stable");
+ try {
+ ResultSet res = statement.executeQuery(sql);
+ while (res.next()) {
+ for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) {
+ System.out.printf("%d: %s\n", i, res.getString(i));
+ }
+ num++;
+ }
+ res.close();
+ assertEquals(4, num);
+ } catch (SQLException e) {
+ assert false : "error describe stable" + e.getMessage();
+ }
+ }
+
+ @Test
+ public void describeTable() {
+ createTable();
+ String sql = "describe t1";
+ int num = 0;
+ System.out.println("describe table");
+ try {
+ ResultSet res = statement.executeQuery(sql);
+ while (res.next()) {
+ for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) {
+ System.out.printf("%d: %s\n", i, res.getString(i));
+ }
+ num++;
+ }
+ res.close();
+ assertEquals(4, num);
+ } catch (SQLException e) {
+ assert false : "error describe stable" + e.getMessage();
+ }
+ }
+
+ // @Test
+ public void validCreateSql() {
+ String sql = "create table t2 using " + stbName + " tags (\"beijing\")";
+ boolean valid = ((TSDBConnection) connection).getConnection().validateCreateTableSql(sql);
+ assertEquals(true, valid);
+ }
+
+ @AfterClass
+ public static void close() throws Exception {
+ if (!statement.isClosed()) {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4be68519c5ba10601e23debf15db254c42638830
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java
@@ -0,0 +1,174 @@
+package com.taosdata.jdbc;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class StatementTest {
+ static Connection connection = null;
+ static Statement statement = null;
+ static String dbName = "test";
+ static String tName = "t0";
+ static String host = "localhost";
+ static ResultSet resSet = null;
+
+ @BeforeClass
+ public static void createConnection() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("drop database if exists " + dbName);
+
+ }
+
+ @Test
+ public void createTableAndQuery() throws SQLException {
+ long ts = System.currentTimeMillis();
+
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)");
+ statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)");
+ statement.executeQuery("select * from " + dbName + "." + tName);
+ ResultSet resultSet = statement.getResultSet();
+ assertTrue(null != resultSet);
+
+ boolean isClosed = statement.isClosed();
+ assertEquals(false, isClosed);
+ }
+
+ @Test
+ public void testUnsupport() {
+// if(null == resSet) {
+// return;
+// }
+ TSDBStatement tsdbStatement = (TSDBStatement) statement;
+ try {
+ tsdbStatement.unwrap(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isWrapperFor(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMaxFieldSize();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setMaxFieldSize(0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setEscapeProcessing(true);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.cancel();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getWarnings();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.clearWarnings();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setCursorName(null);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMoreResults();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setFetchDirection(0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getFetchDirection();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetConcurrency();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetType();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getConnection();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getMoreResults();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getGeneratedKeys();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, 0);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, new int[]{0});
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"});
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.getResultSetHoldability();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.setPoolable(true);
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isPoolable();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.closeOnCompletion();
+ } catch (SQLException e) {
+ }
+ try {
+ tsdbStatement.isCloseOnCompletion();
+ } catch (SQLException e) {
+ }
+ }
+
+ @AfterClass
+ public static void close() throws Exception {
+ if (!statement.isClosed()) {
+ statement.executeUpdate("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+
+ }
+ }
+}
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..25f7f7a285773240c222e4c6a735053dd83f217b
--- /dev/null
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java
@@ -0,0 +1,94 @@
+package com.taosdata.jdbc;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+public class SubscribeTest {
+ Connection connection = null;
+ Statement statement = null;
+ String dbName = "test";
+ String tName = "t0";
+ String host = "localhost";
+ String topic = "test";
+
+ @Before
+ public void createDatabase() throws SQLException {
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ return;
+ }
+ Properties properties = new Properties();
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
+ , properties);
+
+ statement = connection.createStatement();
+ statement.executeUpdate("create database if not exists " + dbName);
+ statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
+ long ts = System.currentTimeMillis();
+ for (int i = 0; i < 2; i++) {
+ ts += i;
+ statement.executeUpdate("insert into \" + dbName + \".\" + tName + \" values (" + ts + ", " + (100 + i) + ", " + i + ")");
+ }
+ }
+
+ @Test
+ public void subscribe() throws Exception {
+ TSDBSubscribe subscribe = null;
+ long subscribId = 0;
+ try {
+
+ String rawSql = "select * from " + dbName + "." + tName + ";";
+ System.out.println(rawSql);
+ subscribe = ((TSDBConnection) connection).createSubscribe();
+ subscribId = subscribe.subscribe(topic, rawSql, false, 1000);
+
+ assertTrue(subscribId > 0);
+
+ int a = 0;
+ while (true) {
+ Thread.sleep(900);
+ TSDBResultSet resSet = subscribe.consume(subscribId);
+
+ while (resSet.next()) {
+ for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
+ System.out.printf(i + ": " + resSet.getString(i) + "\t");
+ }
+ System.out.println("\n======" + a + "==========");
+ }
+ resSet.close();
+ a++;
+ if (a >= 2) {
+ break;
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (null != subscribe && 0 != subscribId) {
+ subscribe.unsubscribe(subscribId, true);
+ }
+ }
+ }
+
+ @After
+ public void close() throws Exception {
+ statement.executeQuery("drop database " + dbName);
+ statement.close();
+ connection.close();
+ Thread.sleep(10);
+ }
+}
\ No newline at end of file
diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c
index 13f3c0a2b0a0a75e07329a24f13ab0b9a274d3ac..769dfea774b86c9cd420edcd1b15d1020da3a67f 100644
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@@ -256,11 +256,30 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
SDataRow trow = (SDataRow)pBlk->data;
tdInitDataRow(trow, pSchema);
+ union {
+ char buf[sizeof(int64_t)];
+ tstr str;
+ } nullVal;
+
for (int32_t i = 0; i < pSchema->numOfCols; i++) {
STColumn *c = pSchema->columns + i;
char* val = (char*)row[i];
if (IS_VAR_DATA_TYPE(c->type)) {
- val -= sizeof(VarDataLenT);
+ if (val == NULL) {
+ val = nullVal.buf;
+ if (c->type == TSDB_DATA_TYPE_BINARY) {
+ setNull(nullVal.str.data, TSDB_DATA_TYPE_BINARY, 1);
+ nullVal.str.len = 1;
+ } else {
+ setNull(nullVal.str.data, TSDB_DATA_TYPE_NCHAR, 4);
+ nullVal.str.len = 4;
+ }
+ } else {
+ val -= sizeof(VarDataLenT);
+ }
+ } else if (val == NULL) {
+ val = nullVal.buf;
+ setNull(val, c->type, c->bytes);
}
tdAppendColVal(trow, val, c->type, c->bytes, c->offset);
}
diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c
index 1416a591bec5e4ec61e7a4581db955e91feec2e8..d06f14d53fcc6e434401636ee91f6c9bd5a44e5e 100644
--- a/src/cq/test/cqtest.c
+++ b/src/cq/test/cqtest.c
@@ -48,8 +48,8 @@ int main(int argc, char *argv[]) {
taosInitLog("cq.log", 100000, 10);
SCqCfg cqCfg;
- strcpy(cqCfg.user, "root");
- strcpy(cqCfg.pass, "taosdata");
+ strcpy(cqCfg.user, TSDB_DEFAULT_USER);
+ strcpy(cqCfg.pass, TSDB_DEFAULT_PASS);
cqCfg.vgId = 2;
cqCfg.cqWrite = writeToQueue;
diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c
index eb9d8653e56fd8984c96615015cb593fec3e6fa3..e3a228919fcffc39b135c1685c7b3385e3253e39 100644
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@@ -118,6 +118,8 @@ void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
mnodeCreateMsg(pWrite, pMsg);
+
+ dTrace("app:%p:%p, msg:%s is put into mwrite queue", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType]);
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
}
@@ -128,6 +130,7 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) {
SMnodeMsg *pWrite = pRaw;
+ if (pWrite == NULL) return;
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
dnodeReprocessMnodeWriteMsg(pWrite);
@@ -146,19 +149,21 @@ void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) {
}
static void *dnodeProcessMnodeWriteQueue(void *param) {
- SMnodeMsg *pWriteMsg;
+ SMnodeMsg *pWrite;
int32_t type;
void * unUsed;
while (1) {
- if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWriteMsg, &unUsed) == 0) {
+ if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) {
dTrace("dnodeProcessMnodeWriteQueue: got no message from qset, exiting...");
break;
}
- dTrace("%p, msg:%s will be processed in mwrite queue", pWriteMsg->rpcMsg.ahandle, taosMsg[pWriteMsg->rpcMsg.msgType]);
- int32_t code = mnodeProcessWrite(pWriteMsg);
- dnodeSendRpcMnodeWriteRsp(pWriteMsg, code);
+ dTrace("app:%p:%p, msg:%s will be processed in mwrite queue", pWrite->rpcMsg.ahandle, pWrite,
+ taosMsg[pWrite->rpcMsg.msgType]);
+
+ int32_t code = mnodeProcessWrite(pWrite);
+ dnodeSendRpcMnodeWriteRsp(pWrite, code);
}
return NULL;
@@ -168,9 +173,15 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
SMnodeMsg *pWrite = pMsg;
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
+ dTrace("app:%p:%p, msg:%s is redirected for mnode not running, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+ taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
+
dnodeSendRedirectMsg(pMsg, true);
dnodeFreeMnodeWriteMsg(pWrite);
- } else {
+ } else {
+ dTrace("app:%p:%p, msg:%s is reput into mwrite queue, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+ taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
+
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
}
}
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 30fdd21b99b4ffaf413a27d957b391bfabecc0dd..d971e3ad6dc2a20c241c0ba1ea5aac12dfa878f8 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -407,11 +407,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId);
}
- SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
- for (int32_t i = 0; i < pCfg->numOfVnodes; ++i) {
- pVgAcccess[i].vgId = htonl(pVgAcccess[i].vgId);
- }
-
+ vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
dnodeProcessModuleStatus(pCfg->moduleStatus);
dnodeUpdateDnodeCfg(pCfg);
diff --git a/src/inc/query.h b/src/inc/query.h
index 49ee5248f5e267db5e67a04e47c18d16dd692613..5fd2ede034ebfaaf86eecce7a429c33996606027 100644
--- a/src/inc/query.h
+++ b/src/inc/query.h
@@ -32,11 +32,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs
/**
* Destroy QInfo object
- *
- * @param qinfo
- * @return
+ * @param qinfo qhandle
+ * @param fp destroy callback function, while the qhandle is destoried, invoke the fp
+ * @param param free callback params
*/
-void qDestroyQueryInfo(qinfo_t qinfo);
+void qDestroyQueryInfo(qinfo_t qinfo, void (*fp)(void*), void* param);
/**
* the main query execution function, including query on both table and multitables,
@@ -45,7 +45,7 @@ void qDestroyQueryInfo(qinfo_t qinfo);
* @param qinfo
* @return
*/
-void qTableQuery(qinfo_t qinfo);
+void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param);
/**
* Retrieve the produced results information, if current query is not paused or completed,
@@ -80,9 +80,12 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
/**
* kill current ongoing query and free query handle automatically
- * @param qinfo
+ * @param qinfo qhandle
+ * @param fp destroy callback function, while the qhandle is destoried, invoke the fp
+ * @param param free callback params
+ * @return
*/
-int32_t qKillQuery(qinfo_t qinfo);
+int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param);
#ifdef __cplusplus
}
diff --git a/src/inc/taos.h b/src/inc/taos.h
index bf6942be8b7ddff9ffff4af222fb5e938e63d096..d524d95685581560fa3e565bce4cf189145af128 100644
--- a/src/inc/taos.h
+++ b/src/inc/taos.h
@@ -55,7 +55,7 @@ typedef enum {
typedef struct taosField {
char name[65];
uint8_t type;
- short bytes;
+ int16_t bytes;
} TAOS_FIELD;
#ifdef _TD_GO_DLL_
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 7490de90d014927c27b2c9b19fbc33f5fd570d05..d8e21c159b08130dc8a4d3e35c0a438a72997391 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -83,6 +83,9 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_NULL_STR "NULL"
#define TSDB_DATA_NULL_STR_L "null"
+#define TSDB_DEFAULT_USER "root"
+#define TSDB_DEFAULT_PASS "taosdata"
+
#define TSDB_TRUE 1
#define TSDB_FALSE 0
#define TSDB_OK 0
@@ -156,7 +159,7 @@ typedef struct tDataTypeDescriptor {
extern tDataTypeDescriptor tDataTypeDesc[11];
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
-bool isValidDataType(int32_t type, int32_t length);
+bool isValidDataType(int32_t type);
bool isNull(const char *val, int32_t type);
void setVardataNull(char* val, int32_t type);
@@ -209,8 +212,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_SQL_SHOW_LEN 256
#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 8mb
-#define TSDB_MAX_BYTES_PER_ROW 65535
-#define TSDB_MAX_TAGS_LEN 65535
+#define TSDB_MAX_BYTES_PER_ROW 16384
+#define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128
#define TSDB_AUTH_LEN 16
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index df9992d53a5111a4000a19eb5998892eb2155372..33e0aa24b6eab113242337b04133766f2ea9b94f 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -113,8 +113,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "mnode inva
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "mnode invalid stream id")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "mnode invalid connection")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "mnode object already there")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "mnode sdb error")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "sdb object already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "sdb app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, 0, 0x0322, "sdb invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, 0, 0x0323, "sdb object not there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, 0, 0x0324, "sdb invalid meta row")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, 0, 0x0325, "sdb invalid key type")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "mnode dnode already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "mnode dnode not exist")
@@ -175,11 +179,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, 0, 0x0506, "vnode no d
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "vnode no such file or directory")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "vnode out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "vnode app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0214, "vnode no write auth")
// tsdb
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "tsdb invalid table id")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "tsdb invalid table schema version")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_SCHEMA_VERSION, 0, 0x0602, "tsdb invalid table schema version")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "tsdb invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, 0, 0x0602, "tsdb invalid table schema version")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, 0, 0x0603, "tsdb table already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, 0, 0x0604, "tsdb invalid configuration")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, 0, 0x0605, "tsdb init failed")
@@ -189,6 +194,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, 0, 0x0608, "tsdb file
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, 0, 0x0609, "tsdb out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, 0, 0x060A, "tsdb tag version is out of date")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "tsdb timestamp is out of range")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "tsdb submit message is messed up")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "tsdb invalid action")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb invalid create table message")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "tsdb file already exists")
// query
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle")
@@ -196,6 +206,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, 0, 0x0701, "query inva
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, 0, 0x0702, "query no diskspace")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "query out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key")
// grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired")
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index 43e34b6ce2364dd5bf1950dea57d47e9ff72dfee..78255a45d5432fffe3ea0d2daf87d94dff041e42 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -455,7 +455,7 @@ typedef struct {
int16_t orderType; // used in group by xx order by xxx
int64_t limit;
int64_t offset;
- uint16_t queryType; // denote another query process
+ uint32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t tagNameRelType; // relation of tag criteria and tbname criteria
int16_t fillType; // interpolate type
@@ -543,6 +543,7 @@ typedef struct {
int32_t dnodeId;
uint32_t moduleStatus;
uint32_t numOfVnodes;
+ uint32_t reserved;
} SDMDnodeCfg;
typedef struct {
diff --git a/src/inc/trpc.h b/src/inc/trpc.h
index 6c5d7fa889b855b91fb47e99e1bf5a80bc88f99e..30e0f9eee1889e8b832812fcc3f17f81c157724e 100644
--- a/src/inc/trpc.h
+++ b/src/inc/trpc.h
@@ -84,7 +84,7 @@ void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
-void rpcCanelRequest(void *pContext);
+void rpcCancelRequest(void *pContext);
#ifdef __cplusplus
}
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index add85d9438440b9992649bdbd349b378de691405..ee18396b61e1e56985a829827f961c95931e7e42 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -19,11 +19,11 @@
#include
#include
-#include "tdataformat.h"
-#include "tname.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tarray.h"
+#include "tdataformat.h"
+#include "tname.h"
#ifdef __cplusplus
extern "C" {
@@ -35,7 +35,7 @@ extern "C" {
#define TSDB_INVALID_SUPER_TABLE_ID -1
#define TSDB_STATUS_COMMIT_START 1
-#define TSDB_STATUS_COMMIT_OVER 2
+#define TSDB_STATUS_COMMIT_OVER 2
// --------- TSDB APPLICATION HANDLE DEFINITION
typedef struct {
@@ -53,9 +53,9 @@ typedef struct {
int32_t tsdbId;
int32_t cacheBlockSize;
int32_t totalBlocks;
- int32_t maxTables; // maximum number of tables this repository can have
- int32_t daysPerFile; // day per file sharding policy
- int32_t keep; // day of data to keep
+ int32_t maxTables; // maximum number of tables this repository can have
+ int32_t daysPerFile; // day per file sharding policy
+ int32_t keep; // day of data to keep
int32_t keep1;
int32_t keep2;
int32_t minRowsPerFileBlock; // minimum rows per file block
@@ -72,19 +72,16 @@ typedef struct {
int64_t pointsWritten; // total data points written
} STsdbStat;
-typedef void TsdbRepoT; // use void to hide implementation details from outside
+typedef void TSDB_REPO_T; // use void to hide implementation details from outside
-void tsdbSetDefaultCfg(STsdbCfg *pCfg);
-STsdbCfg *tsdbCreateDefaultCfg();
-void tsdbFreeCfg(STsdbCfg *pCfg);
-STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo);
+STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
// --------- TSDB REPOSITORY DEFINITION
-int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
-int32_t tsdbDropRepo(TsdbRepoT *repo);
-TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
-int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit);
-int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg);
+int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
+int32_t tsdbDropRepo(char *rootDir);
+TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
+void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
+int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
// --------- TSDB TABLE DEFINITION
typedef struct {
@@ -106,28 +103,19 @@ typedef struct {
char * sql;
} STableCfg;
-int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid);
-int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid);
-int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
-int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
-int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup);
-int tsdbTableSetName(STableCfg *config, char *name, bool dup);
-int tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
-int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup);
void tsdbClearTableCfg(STableCfg *config);
-void* tsdbGetTableTagVal(TsdbRepoT* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes);
-char* tsdbGetTableName(TsdbRepoT *repo, const STableId *id);
+void * tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes);
+char * tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id);
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
-int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
-int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
-int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
-int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg);
-TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid);
-void tsdbStartStream(TsdbRepoT *repo);
+int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
+int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
+int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
+TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
+void tsdbStartStream(TSDB_REPO_T *repo);
-uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size);
+uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size);
// the TSDB repository info
typedef struct STsdbRepoInfo {
@@ -137,7 +125,7 @@ typedef struct STsdbRepoInfo {
int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository
// TODO: Other informations to add
} STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo);
+STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
// the meter information report structure
typedef struct {
@@ -146,7 +134,7 @@ typedef struct {
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;
-STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
+STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid);
// -- FOR INSERT DATA
/**
@@ -156,7 +144,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
*
* @return the number of points inserted, -1 for failure and the error number is set
*/
-int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ;
+int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
// -- FOR QUERY TIME SERIES DATA
@@ -164,10 +152,10 @@ typedef void *TsdbQueryHandleT; // Use void to hide implementation details
// query condition to build vnode iterator
typedef struct STsdbQueryCond {
- STimeWindow twindow;
- int32_t order; // desc|asc order to iterate the data block
- int32_t numOfCols;
- SColumnInfo *colList;
+ STimeWindow twindow;
+ int32_t order; // desc|asc order to iterate the data block
+ int32_t numOfCols;
+ SColumnInfo *colList;
} STsdbQueryCond;
typedef struct SDataBlockInfo {
@@ -199,7 +187,7 @@ typedef void *TsdbPosT;
* @param qinfo query info handle from query processor
* @return
*/
-TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -207,15 +195,17 @@ TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STable
* all tables in this group.
*
* @param tsdb tsdb handle
- * @param pCond query condition, including time window, result set order, and basic required columns for each block
+ * @param pCond query condition, including time window, result set order, and basic required columns for each
+ * block
* @param groupInfo tableId list.
* @return
*/
-TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
-SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle);
+SArray *tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle);
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+ void *qinfo);
/**
* move to next block if exists
@@ -293,7 +283,7 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
* @param stableid. super table sid
* @param pTagCond. tag query condition
*/
-int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pTagCond, size_t len,
int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
SColIndex *pColIndex, int32_t numOfCols);
@@ -305,7 +295,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTag
* @param pGroupInfo the generated result
* @return
*/
-int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
/**
* clean up the query handle
diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index d388bc9dbeea2802ccebfca85e6126c5cdc3220d..b313d6ebb1a86bca699f1517b6b16cdc16ed3150 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -221,6 +221,16 @@
#define TK_INTO 203
#define TK_VALUES 204
+
+#define TK_SPACE 300
+#define TK_COMMENT 301
+#define TK_ILLEGAL 302
+#define TK_HEX 303 // hex number 0x123
+#define TK_OCT 304 // oct number
+#define TK_BIN 305 // bin format data 0b111
+#define TK_FILE 306
+#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
+
#endif
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index 0da1f51e27fc8ba882a6c7d60ccfdaf0245e83dc..9f0c8cc24171184607b93ddaab30142ff29c4e7d 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -58,6 +58,7 @@ void* vnodeGetWal(void *pVnode);
int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
void vnodeBuildStatusMsg(void * param);
+void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 7705db4b27e252a01933950fd309bd7ca179a008..e28c691a8986fce0bcb7a04b8bb1e5f7f5c6605c 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -56,11 +56,11 @@ TAOS *shellInit(SShellArguments *args) {
if (args->is_use_passwd) {
if (args->password == NULL) args->password = getpass("Enter password: ");
} else {
- args->password = tsDefaultPass;
+ args->password = TSDB_DEFAULT_PASS;
}
if (args->user == NULL) {
- args->user = tsDefaultUser;
+ args->user = TSDB_DEFAULT_USER;
}
taos_init();
@@ -276,6 +276,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
st = taosGetTimestampUs();
TAOS_RES* pSql = taos_query(con, command);
+ result = pSql; // set it into the global variable
+
if (taos_errno(pSql)) {
taos_error(pSql);
return;
@@ -284,7 +286,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) {
fprintf(stdout, "Database changed.\n\n");
fflush(stdout);
-
+
+ result = NULL;
taos_free_result(pSql);
return;
}
@@ -294,6 +297,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
int error_no = 0;
int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode);
if (numOfRows < 0) {
+ result = NULL;
taos_free_result(pSql);
return;
}
@@ -315,7 +319,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
if (fname != NULL) {
wordfree(&full_path);
}
-
+
+ result = NULL;
taos_free_result(pSql);
}
@@ -419,8 +424,8 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_
}
}
-static int dumpResultToFile(const char* fname, TAOS_RES* result) {
- TAOS_ROW row = taos_fetch_row(result);
+static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
+ TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
}
@@ -441,9 +446,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
wordfree(&full_path);
- int num_fields = taos_num_fields(result);
- TAOS_FIELD *fields = taos_fetch_fields(result);
- int precision = taos_result_precision(result);
+ int num_fields = taos_num_fields(tres);
+ TAOS_FIELD *fields = taos_fetch_fields(tres);
+ int precision = taos_result_precision(tres);
for (int col = 0; col < num_fields; col++) {
if (col > 0) {
@@ -455,7 +460,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
int numOfRows = 0;
do {
- int32_t* length = taos_fetch_lengths(result);
+ int32_t* length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
if (i > 0) {
fputc(',', fp);
@@ -465,10 +470,13 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
fputc('\n', fp);
numOfRows++;
- row = taos_fetch_row(result);
+ row = taos_fetch_row(tres);
} while( row != NULL);
+ result = NULL;
+ taos_free_result(tres);
fclose(fp);
+
return numOfRows;
}
@@ -769,8 +777,7 @@ void write_history() {
void taos_error(TAOS_RES *tres) {
fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
-
- /* free local resouce: allocated memory/metric-meta refcnt */
+ result = NULL;
taos_free_result(tres);
}
@@ -845,9 +852,9 @@ void shellGetGrantInfo(void *con) {
char sql[] = "show grants";
- result = taos_query(con, sql);
+ TAOS_RES* tres = taos_query(con, sql);
- int code = taos_errno(result);
+ int code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_COM_OPS_NOT_SUPPORT) {
fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con));
@@ -857,18 +864,18 @@ void shellGetGrantInfo(void *con) {
return;
}
- int num_fields = taos_field_count(result);
+ int num_fields = taos_field_count(tres);
if (num_fields == 0) {
fprintf(stderr, "\nInvalid grant information.\n");
exit(0);
} else {
- if (result == NULL) {
+ if (tres == NULL) {
fprintf(stderr, "\nGrant information is null.\n");
exit(0);
}
- TAOS_FIELD *fields = taos_fetch_fields(result);
- TAOS_ROW row = taos_fetch_row(result);
+ TAOS_FIELD *fields = taos_fetch_fields(tres);
+ TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
fprintf(stderr, "\nFailed to get grant information from server. Abort.\n");
exit(0);
@@ -888,8 +895,8 @@ void shellGetGrantInfo(void *con) {
fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will expire at %s.\n", serverVersion, taos_get_server_info(con), expiretime);
}
- taos_free_result(result);
result = NULL;
+ taos_free_result(tres);
}
fprintf(stdout, "\n");
diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c
index 92474bdd03f56e6b5b0e0bde17a94a10606ac8d4..8481f498dd395cd449192d625d307ae812acaa30 100644
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@@ -15,7 +15,6 @@
#include "os.h"
#include "shell.h"
-#include "tsclient.h"
pthread_t pid;
@@ -23,14 +22,6 @@ pthread_t pid;
void interruptHandler(int signum) {
#ifdef LINUX
taos_stop_query(result);
- if (result != NULL) {
- /*
- * we need to free result in async model, in order to avoid free
- * results while the master thread is waiting for server response.
- */
- tscQueueAsyncFreeResult(result);
- }
-
result = NULL;
#else
printf("\nReceive ctrl+c or other signal, quit shell.\n");
diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index ab5781e74fc49da5c70d73ec5bc57f9808f4fb27..4a455271fff51e1bd9438ce01d044eee3760f6c2 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -306,7 +306,7 @@ double getCurrentTime();
void callBack(void *param, TAOS_RES *res, int code);
int main(int argc, char *argv[]) {
- SDemoArguments arguments = {NULL, // host
+ SDemoArguments arguments = { NULL, // host
0, // port
"root", // user
"taosdata", // password
diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c
index ee792c51166b090dd8ea104120ae5fd105791948..ccef783709e6cee69ff1178d939716b33a410e39 100644
--- a/src/kit/taosdump/taosdump.c
+++ b/src/kit/taosdump/taosdump.c
@@ -34,6 +34,8 @@
#include "taosdef.h"
#include "tutil.h"
+#include "tglobal.h"
+
#define COMMAND_SIZE 65536
#define DEFAULT_DUMP_FILE "taosdump.sql"
@@ -293,7 +295,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
static struct argp argp = {options, parse_opt, args_doc, doc};
TAOS *taos = NULL;
-TAOS_RES *result = NULL;
char *command = NULL;
char *lcommand = NULL;
char *buffer = NULL;
@@ -324,7 +325,7 @@ void taosFreeDbInfos();
int main(int argc, char *argv[]) {
SDumpArguments arguments = {
// connection option
- NULL, "root", "taosdata", 0,
+ NULL, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS, 0,
// output file
DEFAULT_DUMP_FILE, DEFAULT_DUMP_FILE, NULL,
// dump unit option
@@ -424,6 +425,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) {
int taosDumpOut(SDumpArguments *arguments) {
TAOS_ROW row;
+ TAOS_RES* result = NULL;
char *temp = NULL;
FILE *fp = NULL;
int count = 0;
@@ -466,7 +468,7 @@ int taosDumpOut(SDumpArguments *arguments) {
result = taos_query(taos, command);
int32_t code = taos_errno(result);
if (code != 0) {
- fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos));
+ fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result));
taos_free_result(result);
goto _exit_failure;
}
@@ -502,7 +504,7 @@ int taosDumpOut(SDumpArguments *arguments) {
}
strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
- if (strcmp(arguments->user, "root") == 0) {
+ if (strcmp(arguments->user, TSDB_DEFAULT_USER) == 0) {
dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]));
dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]));
dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]);
@@ -570,7 +572,7 @@ int taosDumpOut(SDumpArguments *arguments) {
fclose(fp);
taos_close(taos);
taos_free_result(result);
- free(temp);
+ tfree(temp);
taosFreeDbInfos();
return 0;
@@ -578,7 +580,7 @@ _exit_failure:
fclose(fp);
taos_close(taos);
taos_free_result(result);
- free(temp);
+ tfree(temp);
taosFreeDbInfos();
return -1;
}
@@ -613,7 +615,7 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
fprintf(fp, "USE %s\n\n", dbInfo->name);
sprintf(command, "show tables");
- result = taos_query(taos,command);
+ TAOS_RES* result = taos_query(taos,command);
int32_t code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -717,7 +719,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name);
- result = taos_query(taos, command);
+ TAOS_RES* result = taos_query(taos, command);
int32_t code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -795,7 +797,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) {
sprintf(command, "describe %s", table);
- result = taos_query(taos, command);
+ TAOS_RES* result = taos_query(taos, command);
int32_t code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -875,7 +877,7 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
sprintf(command, "select tbname from %s", metric);
- result = taos_query(taos, command);
+ TAOS_RES* result = taos_query(taos, command);
int32_t code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -928,7 +930,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
sprintf(command, "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", tbname, arguments->start_time,
arguments->end_time);
- result = taos_query(taos, command);
+ TAOS_RES* result = taos_query(taos, command);
int32_t code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "failed to run command %s, reason: %s\n", command, taos_errstr(result));
@@ -1177,9 +1179,13 @@ int taosDumpIn(SDumpArguments *arguments) {
tcommand = command;
}
taosReplaceCtrlChar(tcommand);
- if (taos_query(taos, tcommand) == NULL)
+
+ TAOS_RES* result = taos_query(taos, tcommand);
+ if (taos_errno(result) != 0){
fprintf(stderr, "linenu: %" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command,
- taos_errstr(taos));
+ taos_errstr(result));
+ taos_free_result(result);
+ }
pstr = command;
pstr[0] = '\0';
@@ -1225,12 +1231,12 @@ int taosDumpIn(SDumpArguments *arguments) {
tcommand = command;
}
taosReplaceCtrlChar(tcommand);
- result = taos_query(taos, tcommand);
+ TAOS_RES* result = taos_query(taos, tcommand);
int32_t code = taos_errno(result);
if (code != 0)
{
fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason: %s \ncontinue...\n", linenu, command,
- taos_errstr(taos));
+ taos_errstr(result));
}
taos_free_result(result);
}
diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c
index 65b866a99b302a83472eff54367682dcd42b749c..b2c7f729b59b8a697e1f078f25dbad180c849442 100644
--- a/src/kit/taosnetwork/client.c
+++ b/src/kit/taosnetwork/client.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#include
#include
#include
#include
@@ -34,6 +35,36 @@ typedef struct {
char *host[15];
} info;
+typedef struct Arguments {
+ char * host;
+ uint16_t port;
+ uint16_t max_port;
+} SArguments;
+
+static struct argp_option options[] = {
+ {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0},
+ {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6020.", 1},
+ {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6050.", 2}};
+
+static error_t parse_opt(int key, char *arg, struct argp_state *state) {
+
+ SArguments *arguments = state->input;
+ switch (key) {
+ case 'h':
+ arguments->host = arg;
+ break;
+ case 'p':
+ arguments->port = atoi(arg);
+ break;
+ case 'm':
+ arguments->max_port = atoi(arg);
+ break;
+ }
+ return 0;
+}
+
+static struct argp argp = {options, parse_opt, 0, 0};
+
void *checkPort(void *sarg) {
info *pinfo = (info *)sarg;
int port = pinfo->port;
@@ -97,7 +128,7 @@ void *checkUPort(void *sarg) {
sprintf(sendbuf, "send msg port_%d by udp", port);
- socklen_t sin_size = sizeof(*(struct sockaddr*)&serverAddr);
+ socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr);
sendto(clientSocket, sendbuf, strlen(sendbuf), 0, (struct sockaddr *)&serverAddr, (int)sin_size);
@@ -113,14 +144,19 @@ void *checkUPort(void *sarg) {
return NULL;
}
-int main() {
- int port = 6020;
- char *host = "127.0.0.1";
+int main(int argc, char *argv[]) {
+ SArguments arguments = {"127.0.0.1", 6020, 6050};
+
+ argp_parse(&argp, argc, argv, 0, 0, &arguments);
+
+ printf("host: %s\tport: %d\tmax_port: %d\n", arguments.host, arguments.port, arguments.max_port);
+
+ int port = arguments.port;
+ char *host = arguments.host;
info *tinfo = malloc(sizeof(info));
info *uinfo = malloc(sizeof(info));
- for (size_t i = 0; i < 30; i++) {
- port++;
+ for (; port < arguments.max_port; port++) {
printf("For test: %s:%d\n", host, port);
*tinfo->host = host;
diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c
index 7dcc9cbeda70d3c51fd678138f1273071d7a7e7f..c967828f0b28ea8bd55c91c3473a7762ac815356 100644
--- a/src/kit/taosnetwork/server.c
+++ b/src/kit/taosnetwork/server.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#include
#include
#include
#include
@@ -34,6 +35,36 @@ typedef struct {
int type; // 0: tcp, 1: udo, default: 0
} info;
+typedef struct Arguments {
+ char * host;
+ uint16_t port;
+ uint16_t max_port;
+} SArguments;
+
+static struct argp_option options[] = {
+ {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0},
+ {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6020.", 1},
+ {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6050.", 2}};
+
+static error_t parse_opt(int key, char *arg, struct argp_state *state) {
+
+ SArguments *arguments = state->input;
+ switch (key) {
+ case 'h':
+ arguments->host = arg;
+ break;
+ case 'p':
+ arguments->port = atoi(arg);
+ break;
+ case 'm':
+ arguments->max_port = atoi(arg);
+ break;
+ }
+ return 0;
+}
+
+static struct argp argp = {options, parse_opt, 0, 0};
+
static void *bindPort(void *sarg) {
info *pinfo = (info *)sarg;
int port = pinfo->port;
@@ -170,15 +201,21 @@ static void *bindUPort(void *sarg) {
}
-int main() {
- int port = 6020;
- pthread_t *pids = malloc(60 * sizeof(pthread_t));
- info * infos = malloc(30 * sizeof(info));
- info * uinfos = malloc(30 * sizeof(info));
+int main(int argc, char *argv[]) {
+ SArguments arguments = {"127.0.0.1", 6020, 6050};
+ argp_parse(&argp, argc, argv, 0, 0, &arguments);
+ int port = arguments.port;
- for (size_t i = 0; i < 30; i++) {
- port++;
+ int num = arguments.max_port - arguments.port;
+
+ if (num < 0) {
+ num = 1;
+ }
+ pthread_t *pids = malloc(2 * num * sizeof(pthread_t));
+ info * infos = malloc(num * sizeof(info));
+ info * uinfos = malloc(num * sizeof(info));
+ for (size_t i = 0; i < num; i++) {
info *pinfo = infos++;
pinfo->port = port;
@@ -191,14 +228,15 @@ int main() {
info *uinfo = uinfos++;
uinfo->port = port;
uinfo->type = 1;
- if (pthread_create(pids + 30 + i, NULL, bindUPort, uinfo) != 0) //创建线程
+ port++;
+ if (pthread_create(pids + num + i, NULL, bindUPort, uinfo) != 0) //创建线程
{ //创建线程失败
printf("创建线程失败: %d.\n", port);
exit(0);
}
}
- for (int i = 0; i < 30; i++) {
+ for (int i = 0; i < num; i++) {
pthread_join(pids[i], NULL);
- pthread_join(pids[(10 + i)], NULL);
+ pthread_join(pids[(num + i)], NULL);
}
}
diff --git a/src/kit/taosnetwork/taosnetwork_client.c b/src/kit/taosnetwork/taosnetwork_client.c
deleted file mode 100644
index 072610a0357211e2412b6a07f9621cac32e80c50..0000000000000000000000000000000000000000
--- a/src/kit/taosnetwork/taosnetwork_client.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#define SERVER_PORT 8000
-#define SIZE 200
-
-int main() {
- struct sockaddr_in servaddr, cliaddr;
- socklen_t cliaddr_len;
- int client_sockfd;
- char buf[SIZE];
- char recvbuf[SIZE];
-
- int i, n, flag = 0;
-
- int len, iDataNum;
-
- client_sockfd = socket(AF_INET, SOCK_STREAM, 0);
- bzero(&servaddr, sizeof(servaddr));
- servaddr.sin_family = AF_INET;
- servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
- servaddr.sin_port = htons(SERVER_PORT);
-
- if (connect(client_sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) < 0) {
- printf("Connected error..\n");
- return 0;
- }
- printf("Connected to server..\n");
-
- /*循环的发送接收信息并打印接收信息（可以按需发送）--recv返回接收到的字节数，send返回发送的字节数*/
- while (1) {
- printf("Enter string to send:");
- scanf("%s", buf);
- if (!strcmp(buf, "quit")) {
- break;
- }
- len = (sizeof buf);
-
- recvbuf[0] = '\0';
-
- iDataNum = recv(client_sockfd, recvbuf, SIZE, 0);
-
- recvbuf[iDataNum] = '\0';
-
- printf("%s\n", recvbuf);
- }
- return 0;
-}
diff --git a/src/kit/taosnetwork/taosnetwork_server.c b/src/kit/taosnetwork/taosnetwork_server.c
deleted file mode 100644
index 1ec20716fa49bd01eeca7116bb5bd33c76f5886c..0000000000000000000000000000000000000000
--- a/src/kit/taosnetwork/taosnetwork_server.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#define SERVER_PORT 8000
-#define SIZE 200
-
-int main() {
- struct sockaddr_in servaddr, cliaddr;
- socklen_t cliaddr_len;
- int listenfd, connfd;
- char buf[BUFSIZ];
- int i, n, flag = 0;
-
- listenfd = socket(AF_INET, SOCK_STREAM, 0);
- bzero(&servaddr, sizeof(servaddr));
- servaddr.sin_family = AF_INET;
- servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
- servaddr.sin_port = htons(SERVER_PORT);
- bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr));
- listen(listenfd, 20);
-
- printf("Accepting connections..\n");
- while (1) {
- cliaddr_len = sizeof(cliaddr);
- connfd = accept(listenfd, (struct sockaddr *)&cliaddr,
- &cliaddr_len); //如果得不到客户端发来的消息，将会被阻塞，一直等到消息到来
- n = read(connfd, buf, SIZE); //如果n<=0，表示客户端已断开
- while (1) {
- if (n != 0) {
- for (i = 0; i < n; i++) printf("%c", buf[i]); //输出客户端发来的信息
- } else {
- printf("Client say close the connection..\n");
- break;
- }
- n = read(connfd, buf, SIZE);
- }
- close(connfd);
- }
-}
diff --git a/src/kit/taosnetwork/test_client.c b/src/kit/taosnetwork/test_client.c
deleted file mode 100644
index 0c863c6a999cc7c3e8e7e12ec088fd3c2a1f2b2c..0000000000000000000000000000000000000000
--- a/src/kit/taosnetwork/test_client.c
+++ /dev/null
@@ -1,50 +0,0 @@
-#include
-#include
-#include
-#include
-#include
-
-#define SERVER_PORT 8888
-#define BUFF_LEN 512
-#define SERVER_IP "172.0.5.182"
-
-void udp_msg_sender(int fd, struct sockaddr* dst) {}
-
-/*
- client:
- socket-->sendto-->revcfrom-->close
-*/
-
-int main(int argc, char* argv[]) {
- int client_fd;
- struct sockaddr_in ser_addr;
-
- client_fd = socket(AF_INET, SOCK_DGRAM, 0);
- if (client_fd < 0) {
- printf("create socket fail!\n");
- return -1;
- }
-
- memset(&ser_addr, 0, sizeof(ser_addr));
- ser_addr.sin_family = AF_INET;
- // ser_addr.sin_addr.s_addr = inet_addr(SERVER_IP);
- ser_addr.sin_addr.s_addr = htonl(INADDR_ANY); //注意网络序转换
- ser_addr.sin_port = htons(SERVER_PORT); //注意网络序转换
-
- socklen_t len;
- struct sockaddr_in src;
- while (1) {
- char buf[BUFF_LEN] = "TEST UDP MSG!\n";
- len = sizeof(*(struct sockaddr*)&ser_addr);
- printf("client:%s\n", buf); //打印自己发送的信息
- sendto(client_fd, buf, BUFF_LEN, 0, (struct sockaddr*)&ser_addr, len);
- memset(buf, 0, BUFF_LEN);
- recvfrom(client_fd, buf, BUFF_LEN, 0, (struct sockaddr*)&src, &len); //接收来自server的信息
- printf("server:%s\n", buf);
- sleep(1); //一秒发送一次消息
- }
-
- close(client_fd);
-
- return 0;
-}
\ No newline at end of file
diff --git a/src/kit/taosnetwork/test_server.c b/src/kit/taosnetwork/test_server.c
deleted file mode 100644
index 3bfbaa4f7c746854e2d243712a0f663402bddaaf..0000000000000000000000000000000000000000
--- a/src/kit/taosnetwork/test_server.c
+++ /dev/null
@@ -1,63 +0,0 @@
-#include
-#include
-#include
-#include
-#include
-
-#define SERVER_PORT 8888
-#define BUFF_LEN 1024
-
-void handle_udp_msg(int fd) {
- char buf[BUFF_LEN]; //接收缓冲区，1024字节
- socklen_t len;
- int count;
- struct sockaddr_in clent_addr; // clent_addr用于记录发送方的地址信息
- while (1) {
- memset(buf, 0, BUFF_LEN);
- len = sizeof(clent_addr);
- count =
- recvfrom(fd, buf, BUFF_LEN, 0, (struct sockaddr*)&clent_addr, &len); // recvfrom是拥塞函数，没有数据就一直拥塞
- if (count == -1) {
- printf("recieve data fail!\n");
- return;
- }
- printf("client:%s\n", buf); //打印client发过来的信息
- memset(buf, 0, BUFF_LEN);
- sprintf(buf, "I have recieved %d bytes data!\n", count); //回复client
- printf("server:%s\n", buf); //打印自己发送的信息给
- sendto(fd, buf, BUFF_LEN, 0, (struct sockaddr*)&clent_addr,
- len); //发送信息给client，注意使用了clent_addr结构体指针
- }
-}
-
-/*
- server:
- socket-->bind-->recvfrom-->sendto-->close
-*/
-
-int main(int argc, char* argv[]) {
- int server_fd, ret;
- struct sockaddr_in ser_addr;
-
- server_fd = socket(AF_INET, SOCK_DGRAM, 0); // AF_INET:IPV4;SOCK_DGRAM:UDP
- if (server_fd < 0) {
- printf("create socket fail!\n");
- return -1;
- }
-
- memset(&ser_addr, 0, sizeof(ser_addr));
- ser_addr.sin_family = AF_INET;
- ser_addr.sin_addr.s_addr = htonl(INADDR_ANY); // IP地址，需要进行网络序转换，INADDR_ANY：本地地址
- ser_addr.sin_port = htons(SERVER_PORT); //端口号，需要网络序转换
-
- ret = bind(server_fd, (struct sockaddr*)&ser_addr, sizeof(ser_addr));
- if (ret < 0) {
- printf("socket bind fail!\n");
- return -1;
- }
-
- handle_udp_msg(server_fd); //处理接收到的数据
-
- close(server_fd);
- return 0;
-}
\ No newline at end of file
diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index 2baf28f88f9b51cae7e1159a2bd9446217adbdcd..ae1ba98a0f4a15d37824de71e5db81c73effa89c 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -29,21 +29,28 @@ struct SAcctObj;
struct SUserObj;
struct SMnodeObj;
+/*
+struct define notes:
+1. The first field must be the xxxxId field or name field , e.g. 'int32_t dnodeId', 'int32_t mnodeId', 'char name[]', 'char user[]', ...
+2. From the dnodeId field to the updataEnd field, these information will be falled disc;
+3. The fields behind the updataEnd field can be changed;
+*/
+
typedef struct SDnodeObj {
int32_t dnodeId;
- uint16_t dnodePort;
- char dnodeFqdn[TSDB_FQDN_LEN];
- char dnodeEp[TSDB_EP_LEN];
- int64_t createdTime;
- uint32_t lastAccess;
int32_t openVnodes;
+ int64_t createdTime;
int32_t totalVnodes; // from dnode status msg, config information
int32_t customScore; // config by user
+ uint32_t lastAccess;
uint16_t numOfCores; // from dnode status msg
+ uint16_t dnodePort;
+ char dnodeFqdn[TSDB_FQDN_LEN];
+ char dnodeEp[TSDB_EP_LEN];
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function
int8_t isMgmt;
- int8_t reserved[15];
+ int8_t reserved0[14];
int8_t updateEnd[1];
int32_t refCount;
uint32_t moduleStatus;
@@ -54,15 +61,18 @@ typedef struct SDnodeObj {
int16_t cpuAvgUsage; // calc from sys.cpu
int16_t memoryAvgUsage; // calc from sys.mem
int16_t bandwidthUsage; // calc from sys.band
+ int8_t reserved1[2];
} SDnodeObj;
typedef struct SMnodeObj {
int32_t mnodeId;
+ int8_t reserved0[4];
int64_t createdTime;
- int8_t reserved[14];
+ int8_t reserved1[7];
int8_t updateEnd[1];
int32_t refCount;
int8_t role;
+ int8_t reserved2[3];
} SMnodeObj;
typedef struct STableObj {
@@ -71,35 +81,38 @@ typedef struct STableObj {
} STableObj;
typedef struct SSuperTableObj {
- STableObj info;
+ STableObj info;
+ int8_t reserved0[1]; // for fill struct STableObj to 4byte align
+ int16_t nextColId;
+ int32_t sversion;
uint64_t uid;
int64_t createdTime;
- int32_t sversion;
int32_t tversion;
int32_t numOfColumns;
int32_t numOfTags;
- int8_t reserved[15];
+ int8_t reserved1[3];
int8_t updateEnd[1];
int32_t refCount;
int32_t numOfTables;
- int16_t nextColId;
SSchema * schema;
void * vgHash;
+ int8_t reserved2[6];
} SSuperTableObj;
typedef struct {
- STableObj info;
+ STableObj info;
+ int8_t reserved0[1]; // for fill struct STableObj to 4byte align
+ int16_t nextColId; //used by normal table
+ int32_t sversion; //used by normal table
uint64_t uid;
+ uint64_t suid;
int64_t createdTime;
- int32_t sversion; //used by normal table
int32_t numOfColumns; //used by normal table
int32_t sid;
int32_t vgId;
- uint64_t suid;
int32_t sqlLen;
- int8_t reserved[1];
int8_t updateEnd[1];
- int16_t nextColId; //used by normal table
+ int8_t reserved1[1];
int32_t refCount;
char* sql; //used by normal table
SSchema* schema; //used by normal table
@@ -115,22 +128,24 @@ typedef struct {
typedef struct SVgObj {
uint32_t vgId;
- char dbName[TSDB_DB_NAME_LEN];
- int64_t createdTime;
- SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
int32_t numOfVnodes;
+ int64_t createdTime;
int32_t lbDnodeId;
int32_t lbTime;
+ char dbName[TSDB_DB_NAME_LEN];
int8_t inUse;
- int8_t reserved[13];
+ int8_t accessState;
+ int8_t reserved0[5];
+ SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
+ int8_t reserved1[7];
int8_t updateEnd[1];
int32_t refCount;
- struct SVgObj *prev, *next;
- struct SDbObj *pDb;
int32_t numOfTables;
int64_t totalStorage;
int64_t compStorage;
int64_t pointsWritten;
+ struct SVgObj *prev, *next;
+ struct SDbObj *pDb;
void * idPool;
SChildTableObj **tableList;
} SVgObj;
@@ -150,17 +165,18 @@ typedef struct {
int8_t compression;
int8_t walLevel;
int8_t replications;
- int8_t reserved[16];
+ int8_t reserved[12];
} SDbCfg;
typedef struct SDbObj {
char name[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+ int8_t reserved0[4];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int32_t cfgVersion;
SDbCfg cfg;
int8_t status;
- int8_t reserved[14];
+ int8_t reserved1[14];
int8_t updateEnd[1];
int32_t refCount;
int32_t numOfVgroups;
@@ -185,13 +201,6 @@ typedef struct SUserObj {
} SUserObj;
typedef struct {
- int32_t numOfUsers;
- int32_t numOfDbs;
- int32_t numOfTimeSeries;
- int32_t numOfPointsPerSecond;
- int32_t numOfConns;
- int32_t numOfQueries;
- int32_t numOfStreams;
int64_t totalStorage; // Total storage wrtten from this account
int64_t compStorage; // Compressed storage on disk
int64_t queryTime;
@@ -199,34 +208,44 @@ typedef struct {
int64_t inblound;
int64_t outbound;
int64_t sKey;
+ int32_t numOfUsers;
+ int32_t numOfDbs;
+ int32_t numOfTimeSeries;
+ int32_t numOfPointsPerSecond;
+ int32_t numOfConns;
+ int32_t numOfQueries;
+ int32_t numOfStreams;
int8_t accessState; // Checked by mgmt heartbeat message
+ int8_t reserved[3];
} SAcctInfo;
typedef struct SAcctObj {
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
SAcctCfg cfg;
- int32_t acctId;
int64_t createdTime;
+ int32_t acctId;
int8_t status;
- int8_t reserved[14];
+ int8_t reserved0[10];
int8_t updateEnd[1];
- int32_t refCount;
SAcctInfo acctInfo;
+ int32_t refCount;
+ int8_t reserved1[4];
pthread_mutex_t mutex;
} SAcctObj;
typedef struct {
- int8_t type;
- int32_t index;
char db[TSDB_DB_NAME_LEN];
- void * pIter;
+ int8_t type;
int16_t numOfColumns;
+ int32_t index;
int32_t rowSize;
int32_t numOfRows;
- int32_t numOfReads;
+ void * pIter;
int16_t offset[TSDB_MAX_COLUMNS];
int16_t bytes[TSDB_MAX_COLUMNS];
+ int32_t numOfReads;
+ int8_t reserved0[2];
uint16_t payloadLen;
char payload[];
} SShowObj;
diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h
index f95a163d838fc10ea9bb9acadef7bd9df7a83769..9e21f8f56ae5942382a162a9a2c9a4eac12f5908 100644
--- a/src/mnode/inc/mnodeDnode.h
+++ b/src/mnode/inc/mnodeDnode.h
@@ -47,7 +47,7 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode);
void * mnodeGetDnode(int32_t dnodeId);
void * mnodeGetDnodeByEp(char *ep);
void mnodeUpdateDnode(SDnodeObj *pDnode);
-int32_t mnodeDropDnode(SDnodeObj *pDnode);
+int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg);
extern int32_t tsAccessSquence;
diff --git a/src/mnode/inc/mnodeInt.h b/src/mnode/inc/mnodeInt.h
index 7405cef6f3ce9786864db8ff69a9e248c7dfc53b..708f11d1f78e8fb9006c6afb039a6fe366835d14 100644
--- a/src/mnode/inc/mnodeInt.h
+++ b/src/mnode/inc/mnodeInt.h
@@ -36,10 +36,10 @@ extern int32_t sdbDebugFlag;
#define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) }
#define mLPrint(...) { monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) }
-#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); }}
-#define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); }}
-#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__);}}
-#define sdbPrint(...) { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); }
+#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR SDB ", 255, __VA_ARGS__); }}
+#define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("WARN SDB ", sdbDebugFlag, __VA_ARGS__); }}
+#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__);}}
+#define sdbPrint(...) { taosPrintLog("SDB ", 255, __VA_ARGS__); }
#define sdbLError(...) { monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__) }
#define sdbLWarn(...) { monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__) }
diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h
index 975206d52e50dbdcb17ad272ef792143e88f6c98..ca2fffe24c76b48dc647a08d08ffdce9e8873240 100644
--- a/src/mnode/inc/mnodeSdb.h
+++ b/src/mnode/inc/mnodeSdb.h
@@ -20,6 +20,8 @@
extern "C" {
#endif
+struct SMnodeMsg;
+
typedef enum {
SDB_TABLE_DNODE = 0,
SDB_TABLE_MNODE = 1,
@@ -48,8 +50,11 @@ typedef struct {
ESdbOper type;
void * table;
void * pObj;
- int32_t rowSize;
void * rowData;
+ int32_t rowSize;
+ int32_t retCode; // for callback in sdb queue
+ int32_t (*cb)(struct SMnodeMsg *pMsg, int32_t code);
+ struct SMnodeMsg *pMsg;
} SSdbOper;
typedef struct {
diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h
index 338b43b47de070d2ebe76fe23499b55f3d42d318..073460f9d3621c7ec1f1eb57fd7da15791753b9e 100644
--- a/src/mnode/inc/mnodeUser.h
+++ b/src/mnode/inc/mnodeUser.h
@@ -28,7 +28,8 @@ void * mnodeGetNextUser(void *pIter, SUserObj **pUser);
void mnodeIncUserRef(SUserObj *pUser);
void mnodeDecUserRef(SUserObj *pUser);
SUserObj *mnodeGetUserFromConn(void *pConn);
-int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass);
+char * mnodeGetUserFromMsg(void *pMnodeMsg);
+int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg);
void mnodeDropAllUsers(SAcctObj *pAcct);
#ifdef __cplusplus
diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h
index d61145d9b82ffa643a52c0e4feb6c75443079153..29a0fe1799a0f31e4697ecc2bde57e8dcd230a13 100644
--- a/src/mnode/inc/mnodeVgroup.h
+++ b/src/mnode/inc/mnodeVgroup.h
@@ -34,7 +34,8 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
void mnodeUpdateVgroup(SVgObj *pVgroup);
-void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
+void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
+void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);
int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg, SDbObj *pDb);
void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c
index 7ea5188b96a47981f5c42e9c28389f8cd0410409..ad52a4c3acc2d89463d14df00d120711a767ffc2 100644
--- a/src/mnode/src/mnodeAcct.c
+++ b/src/mnode/src/mnodeAcct.c
@@ -25,6 +25,8 @@
#include "mnodeSdb.h"
#include "mnodeUser.h"
+#include "tglobal.h"
+
void * tsAcctSdb = NULL;
static int32_t tsAcctUpdateSize;
static int32_t mnodeCreateRootAcct();
@@ -39,6 +41,7 @@ static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
static int32_t mnodeAcctActionInsert(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
+ pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS;
pthread_mutex_init(&pAcct->mutex, NULL);
return TSDB_CODE_SUCCESS;
}
@@ -78,7 +81,9 @@ static int32_t mnodeAcctActionDecode(SSdbOper *pOper) {
}
static int32_t mnodeAcctActionRestored() {
- if (dnodeIsFirstDeploy()) {
+ int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb);
+ if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
+ mPrint("dnode first deploy, create root acct");
int32_t code = mnodeCreateRootAcct();
if (code != TSDB_CODE_SUCCESS) {
mError("failed to create root account, reason:%s", tstrerror(code));
@@ -171,8 +176,8 @@ static int32_t mnodeCreateRootAcct() {
SAcctObj *pAcct = malloc(sizeof(SAcctObj));
memset(pAcct, 0, sizeof(SAcctObj));
- strcpy(pAcct->user, "root");
- taosEncryptPass((uint8_t*)"taosdata", strlen("taosdata"), pAcct->pass);
+ strcpy(pAcct->user, TSDB_DEFAULT_USER);
+ taosEncryptPass((uint8_t *)TSDB_DEFAULT_PASS, strlen(TSDB_DEFAULT_PASS), pAcct->pass);
pAcct->cfg = (SAcctCfg){
.maxUsers = 10,
.maxDbs = 64,
@@ -204,4 +209,4 @@ int32_t acctInit() { return TSDB_CODE_SUCCESS; }
void acctCleanUp() {}
int32_t acctCheck(void *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index aeb6f4beb9e7a51b92f4c878454f41e82fe18597..9709d7786aaa3f6e0f563bd85ed453679ce0c4f6 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -41,7 +41,7 @@
static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg);
static int32_t mnodeDropDb(SMnodeMsg *newMsg);
static int32_t mnodeSetDbDropping(SDbObj *pDb);
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -311,7 +311,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->replications < 0) pCfg->replications = tsReplications;
}
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) {
int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
if (code != 0) return code;
@@ -364,12 +364,15 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
.table = tsDbSdb,
.pObj = pDb,
.rowSize = sizeof(SDbObj),
+ .pMsg = pMsg
};
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pDb);
- code = TSDB_CODE_MND_SDB_ERROR;
+ } else {
+ mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
@@ -475,7 +478,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
cols++;
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
@@ -487,7 +490,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
#endif
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
@@ -511,7 +514,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
cols++;
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
@@ -625,7 +628,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
cols++;
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->numOfVgroups;
@@ -635,7 +638,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
#endif
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.replications;
@@ -656,7 +659,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
cols++;
#ifndef __CLOUD_VERSION__
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1
@@ -771,12 +774,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
} else if (!pMsg->pUser->writeAuth) {
code = TSDB_CODE_MND_NO_RIGHTS;
} else {
- code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate);
- if (code == TSDB_CODE_SUCCESS) {
- mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user);
- } else {
- mError("db:%s, failed to create, reason:%s", pCreate->db, tstrerror(code));
- }
+ code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate, pMsg);
}
return code;
@@ -893,7 +891,31 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
return newCfg;
}
-static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
+static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
+ if (code != TSDB_CODE_SUCCESS) return code;
+ SDbObj *pDb = pMsg->pDb;
+
+ void *pIter = NULL;
+ while (1) {
+ SVgObj *pVgroup = NULL;
+ pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+ if (pVgroup == NULL) break;
+ if (pVgroup->pDb == pDb) {
+ mnodeSendCreateVgroupMsg(pVgroup, NULL);
+ }
+ mnodeDecVgroupRef(pVgroup);
+ }
+ sdbFreeIter(pIter);
+
+ mTrace("db:%s, all vgroups is altered", pDb->name);
+ mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+
+ balanceNotify();
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
if (terrno != TSDB_CODE_SUCCESS) {
return terrno;
@@ -904,38 +926,24 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
return code;
}
- int32_t oldReplica = pDb->cfg.replications;
-
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
pDb->cfgVersion++;
SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
+ .type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
- .pObj = pDb
+ .pObj = pDb,
+ .pMsg = pMsg,
+ .cb = mnodeAlterDbCb
};
- int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ code = sdbUpdateRow(&oper);
+ if (code == TSDB_CODE_SUCCESS) {
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
- void *pIter = NULL;
- while (1) {
- SVgObj *pVgroup = NULL;
- pIter = mnodeGetNextVgroup(pIter, &pVgroup);
- if (pVgroup == NULL) break;
- mnodeSendCreateVgroupMsg(pVgroup, NULL);
- mnodeDecVgroupRef(pVgroup);
- }
- sdbFreeIter(pIter);
-
- if (oldReplica != pDb->cfg.replications) {
- balanceNotify();
- }
-
- return TSDB_CODE_SUCCESS;
+ return code;
}
static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
@@ -948,28 +956,26 @@ static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_INVALID_DB;
}
- int32_t code = mnodeAlterDb(pMsg->pDb, pAlter);
- if (code != TSDB_CODE_SUCCESS) {
- mError("db:%s, failed to alter, invalid db option", pAlter->db);
- return code;
- }
-
- mTrace("db:%s, all vgroups is altered", pMsg->pDb->name);
- return TSDB_CODE_SUCCESS;
+ return mnodeAlterDb(pMsg->pDb, pAlter, pMsg);
}
static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
+ if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
SDbObj *pDb = pMsg->pDb;
mPrint("db:%s, drop db from sdb", pDb->name);
SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
+ .type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
- .pObj = pDb
+ .pObj = pDb,
+ .pMsg = pMsg
};
+
int32_t code = sdbDeleteRow(&oper);
- if (code != 0) {
- code = TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index fda3f48da5fdd7af0f1782d29fb07812539ca82c..e16195499b17335d44e5d51ae756e01d89e727ec 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -44,7 +44,7 @@ static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
extern void * tsVgroupSdb;
-static int32_t mnodeCreateDnode(char *ep);
+static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg);
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg);
@@ -90,11 +90,12 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId);
- if (pDnode != pSaved && pDnode != NULL && pSaved != NULL) {
+ if (pSaved != NULL && pDnode != pSaved) {
memcpy(pSaved, pDnode, pOper->rowSize);
free(pDnode);
+ mnodeDecDnodeRef(pSaved);
}
- mnodeDecDnodeRef(pSaved);
+
return TSDB_CODE_SUCCESS;
}
@@ -117,10 +118,13 @@ static int32_t mnodeDnodeActionDecode(SSdbOper *pOper) {
static int32_t mnodeDnodeActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
- mnodeCreateDnode(tsLocalEp);
+ mPrint("dnode first deploy, create dnode:%s", tsLocalEp);
+ mnodeCreateDnode(tsLocalEp, NULL);
SDnodeObj *pDnode = mnodeGetDnodeByEp(tsLocalEp);
- mnodeAddMnode(pDnode->dnodeId);
- mnodeDecDnodeRef(pDnode);
+ if (pDnode != NULL) {
+ mnodeAddMnode(pDnode->dnodeId);
+ mnodeDecDnodeRef(pDnode);
+ }
}
return TSDB_CODE_SUCCESS;
@@ -250,7 +254,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
// TODO temporary disabled for compiling: strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep);
}
- if (strcmp(pMsg->pUser->user, "root") != 0) {
+ if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
}
@@ -334,6 +338,18 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
}
int32_t openVnodes = htons(pStatus->openVnodes);
+ int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
+ SDMStatusRsp *pRsp = rpcMallocCont(contLen);
+ if (pRsp == NULL) {
+ mnodeDecDnodeRef(pDnode);
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
+ }
+
+ pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
+ pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
+ pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
+ SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
+
for (int32_t j = 0; j < openVnodes; ++j) {
SVnodeLoad *pVload = &pStatus->load[j];
pVload->vgId = htonl(pVload->vgId);
@@ -346,8 +362,12 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
} else {
mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload);
+ pAccess->vgId = htonl(pVload->vgId);
+ pAccess->accessState = pVgroup->accessState;
+ pAccess++;
mnodeDecVgroupRef(pVgroup);
}
+
}
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@@ -355,6 +375,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
bool ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg));
if (false == ret) {
mnodeDecDnodeRef(pDnode);
+ rpcFreeCont(pRsp);
mError("dnode %s cluster cfg parameters inconsistent", pStatus->dnodeEp);
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
}
@@ -365,33 +386,24 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
balanceNotify();
}
- mnodeDecDnodeRef(pDnode);
-
- int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SDMVgroupAccess);
- SDMStatusRsp *pRsp = rpcMallocCont(contLen);
- if (pRsp == NULL) {
- return TSDB_CODE_MND_OUT_OF_MEMORY;
+ if (openVnodes != pDnode->openVnodes) {
+ mnodeCheckUnCreatedVgroup(pDnode, pStatus->load, openVnodes);
}
pDnode->lastAccess = tsAccessSquence;
+ //this func should be called after sdb replica changed
mnodeGetMnodeInfos(&pRsp->mnodes);
-
- pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
- pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
- pRsp->dnodeCfg.numOfVnodes = 0;
- contLen = sizeof(SDMStatusRsp);
+ mnodeDecDnodeRef(pDnode);
- //TODO: set vnode access
-
pMsg->rpcRsp.len = contLen;
pMsg->rpcRsp.rsp = pRsp;
return TSDB_CODE_SUCCESS;
}
-static int32_t mnodeCreateDnode(char *ep) {
+static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
int32_t grantCode = grantCheck(TSDB_GRANT_DNODE);
if (grantCode != TSDB_CODE_SUCCESS) {
return grantCode;
@@ -415,7 +427,8 @@ static int32_t mnodeCreateDnode(char *ep) {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
- .rowSize = sizeof(SDnodeObj)
+ .rowSize = sizeof(SDnodeObj),
+ .pMsg = pMsg
};
int32_t code = sdbInsertRow(&oper);
@@ -423,30 +436,32 @@ static int32_t mnodeCreateDnode(char *ep) {
int dnodeId = pDnode->dnodeId;
tfree(pDnode);
mError("failed to create dnode:%d, result:%s", dnodeId, tstrerror(code));
- return TSDB_CODE_MND_SDB_ERROR;
+ } else {
+ mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
- mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
return code;
}
-int32_t mnodeDropDnode(SDnodeObj *pDnode) {
+int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
- .pObj = pDnode
+ .pObj = pDnode,
+ .pMsg = pMsg
};
- int32_t code = sdbDeleteRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- code = TSDB_CODE_MND_SDB_ERROR;
+ int32_t code = sdbDeleteRow(&oper);
+ if (code == TSDB_CODE_SUCCESS) {
+ mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
- mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
return code;
}
-static int32_t mnodeDropDnodeByEp(char *ep) {
+static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
SDnodeObj *pDnode = mnodeGetDnodeByEp(ep);
if (pDnode == NULL) {
mError("dnode:%s, is not exist", ep);
@@ -461,7 +476,7 @@ static int32_t mnodeDropDnodeByEp(char *ep) {
mPrint("dnode:%d, start to drop it", pDnode->dnodeId);
#ifndef _SYNC
- return mnodeDropDnode(pDnode);
+ return mnodeDropDnode(pDnode, pMsg);
#else
return balanceDropDnode(pDnode);
#endif
@@ -470,38 +485,20 @@ static int32_t mnodeDropDnodeByEp(char *ep) {
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
- if (strcmp(pMsg->pUser->user, "root") != 0) {
+ if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
} else {
- int32_t code = mnodeCreateDnode(pCreate->ep);
-
- if (code == TSDB_CODE_SUCCESS) {
- SDnodeObj *pDnode = mnodeGetDnodeByEp(pCreate->ep);
- mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
- mnodeDecDnodeRef(pDnode);
- } else {
- mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(code));
- }
-
- return code;
+ return mnodeCreateDnode(pCreate->ep, pMsg);
}
}
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) {
SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
- if (strcmp(pMsg->pUser->user, "root") != 0) {
+ if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
} else {
- int32_t code = mnodeDropDnodeByEp(pDrop->ep);
-
- if (code == TSDB_CODE_SUCCESS) {
- mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user);
- } else {
- mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(code));
- }
-
- return code;
+ return mnodeDropDnodeByEp(pDrop->ep, pMsg);
}
}
@@ -509,7 +506,7 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->pAcct->user, "root") != 0) {
+ if (strcmp(pUser->pAcct->user, TSDB_DEFAULT_USER) != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
@@ -638,7 +635,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
@@ -748,7 +745,7 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
@@ -835,7 +832,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index 41bd6974dbe9eb422b3b72ec650bf06b995276df..219fb0b0a3b1e33292a55fc866e0982078e18469 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -31,6 +31,8 @@
#include "mnodeShow.h"
#include "mnodeUser.h"
+#include "tglobal.h"
+
static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static SRpcIpSet tsMnodeIpSetForShell;
@@ -333,7 +335,7 @@ static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->pAcct->user, "root") != 0) {
+ if (strcmp(pUser->pAcct->user, TSDB_DEFAULT_USER) != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index 31ca47929c74e8df1a659bd33e07a76afe1f6dbd..2a895c9496f1c5243770099285eea1fce27b1ed2 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -39,6 +39,7 @@
#define CONN_KEEP_TIME (tsShellActivityTimer * 3)
#define CONN_CHECK_TIME (tsShellActivityTimer * 2)
#define QUERY_ID_SIZE 20
+#define QUERY_STREAM_SAVE_SIZE 20
extern void *tsMnodeTmr;
static SCacheObj *tsMnodeConnCache = NULL;
@@ -168,7 +169,7 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
@@ -264,16 +265,27 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
// not thread safe, need optimized
int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
pConn->numOfQueries = htonl(pHBMsg->numOfQueries);
- if (pConn->numOfQueries > 0 && pConn->numOfQueries < 20) {
- pConn->pQueries = calloc(sizeof(SQueryDesc), pConn->numOfQueries);
- memcpy(pConn->pQueries, pHBMsg->pData, pConn->numOfQueries * sizeof(SQueryDesc));
+ if (pConn->numOfQueries > 0) {
+ if (pConn->pQueries == NULL) {
+ pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE);
+ }
+
+ int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfQueries) * sizeof(SQueryDesc);
+ if (saveSize > 0 && pConn->pQueries != NULL) {
+ memcpy(pConn->pQueries, pHBMsg->pData, saveSize);
+ }
}
pConn->numOfStreams = htonl(pHBMsg->numOfStreams);
- if (pConn->numOfStreams > 0 && pConn->numOfStreams < 20) {
- pConn->pStreams = calloc(sizeof(SStreamDesc), pConn->numOfStreams);
- memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc),
- pConn->numOfStreams * sizeof(SStreamDesc));
+ if (pConn->numOfStreams > 0) {
+ if (pConn->pStreams == NULL) {
+ pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE);
+ }
+
+ int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfStreams) * sizeof(SStreamDesc);
+ if (saveSize > 0 && pConn->pStreams != NULL) {
+ memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), saveSize);
+ }
}
return TSDB_CODE_SUCCESS;
@@ -282,7 +294,7 @@ int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
@@ -391,7 +403,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
@@ -519,7 +531,7 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mPrint("kill query msg is received, queryId:%s", pKill->queryId);
@@ -549,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mPrint("kill stream msg is received, streamId:%s", pKill->queryId);
@@ -579,7 +591,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
- if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont;
SConnObj * pConn = taosCacheAcquireByName(tsMnodeConnCache, pKill->queryId);
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 659ac159a8fee323e2a1677c9032598ecadd2331..193ec1924bb80aa5ed9af2ceee92987714ed8aa0 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -24,6 +24,7 @@
#include "tsync.h"
#include "tglobal.h"
#include "dnode.h"
+#include "mnode.h"
#include "mnodeDef.h"
#include "mnodeInt.h"
#include "mnodeMnode.h"
@@ -31,6 +32,7 @@
#include "mnodeSdb.h"
#define SDB_TABLE_LEN 12
+#define SDB_SYNC_HACK 16
typedef enum {
SDB_ACTION_INSERT,
@@ -83,8 +85,29 @@ typedef struct {
void * row;
} SSdbRow;
+typedef struct {
+ pthread_t thread;
+ int32_t workerId;
+} SSdbWriteWorker;
+
+typedef struct {
+ int32_t num;
+ SSdbWriteWorker *writeWorker;
+} SSdbWriteWorkerPool;
+
static SSdbObject tsSdbObj = {0};
-static int sdbWrite(void *param, void *data, int type);
+static taos_qset tsSdbWriteQset;
+static taos_qall tsSdbWriteQall;
+static taos_queue tsSdbWriteQueue;
+static SSdbWriteWorkerPool tsSdbPool;
+
+static int sdbWrite(void *param, void *data, int type);
+static int sdbWriteToQueue(void *param, void *data, int type);
+static void * sdbWorkerFp(void *param);
+static int32_t sdbInitWriteWorker();
+static void sdbCleanupWriteWorker();
+static int32_t sdbAllocWriteQueue();
+static void sdbFreeWritequeue();
int32_t sdbGetId(void *handle) {
return ((SSdbTable *)handle)->autoIndex;
@@ -302,7 +325,7 @@ void sdbUpdateSync() {
syncInfo.ahandle = NULL;
syncInfo.getWalInfo = sdbGetWalInfo;
syncInfo.getFileInfo = sdbGetFileInfo;
- syncInfo.writeToCache = sdbWrite;
+ syncInfo.writeToCache = sdbWriteToQueue;
syncInfo.confirmForward = sdbConfirmForward;
syncInfo.notifyRole = sdbNotifyRole;
tsSdbObj.cfg = syncCfg;
@@ -319,10 +342,14 @@ int32_t sdbInit() {
pthread_mutex_init(&tsSdbObj.mutex, NULL);
sem_init(&tsSdbObj.sem, 0, 0);
+ if (sdbInitWriteWorker() != 0) {
+ return -1;
+ }
+
if (sdbInitWal() != 0) {
return -1;
}
-
+
sdbRestoreTables();
if (mnodeGetMnodesNum() == 1) {
@@ -340,6 +367,8 @@ void sdbCleanUp() {
tsSdbObj.status = SDB_STATUS_CLOSING;
+ sdbCleanupWriteWorker();
+
if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync);
tsSdbObj.sync = NULL;
@@ -475,7 +504,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
- sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 "version:%" PRIu64, pTable->tableName,
+ sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
@@ -494,9 +523,10 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
}
static int sdbWrite(void *param, void *data, int type) {
+ SSdbOper *pOper = param;
SWalHead *pHead = data;
- int32_t tableId = pHead->msgType / 10;
- int32_t action = pHead->msgType % 10;
+ int32_t tableId = pHead->msgType / 10;
+ int32_t action = pHead->msgType % 10;
SSdbTable *pTable = sdbGetTableFromId(tableId);
assert(pTable != NULL);
@@ -531,21 +561,22 @@ static int sdbWrite(void *param, void *data, int type) {
pthread_mutex_unlock(&tsSdbObj.mutex);
return code;
}
- walFsync(tsSdbObj.wal);
-
+
code = sdbForwardToPeer(pHead);
pthread_mutex_unlock(&tsSdbObj.mutex);
// from app, oper is created
- if (param != NULL) {
- //sdbTrace("request from app is disposed, version:%" PRIu64 " code:%s", pHead->version, tstrerror(code));
+ if (pOper != NULL) {
+ sdbTrace("record from app is disposed, version:%" PRIu64 " result:%s", pHead->version, tstrerror(code));
return code;
}
// from wal or forward msg, oper not created, should add into hash
if (tsSdbObj.sync != NULL) {
- sdbTrace("forward request is received, version:%" PRIu64 " result:%s, confirm it", pHead->version, tstrerror(code));
+ sdbTrace("record from wal forward is disposed, version:%" PRIu64 " confirm it", pHead->version);
syncConfirmForward(tsSdbObj.sync, pHead->version, code);
+ } else {
+ sdbTrace("record from wal restore is disposed, version:%" PRIu64 , pHead->version);
}
if (action == SDB_ACTION_INSERT) {
@@ -568,7 +599,7 @@ static int sdbWrite(void *param, void *data, int type) {
int32_t sdbInsertRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
- if (pTable == NULL) return -1;
+ if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
if (sdbGetRowFromObj(pTable, pOper->pObj)) {
sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj));
@@ -587,98 +618,133 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
pthread_mutex_unlock(&pTable->mutex);
}
- if (pOper->type == SDB_OPER_GLOBAL) {
- int32_t size = sizeof(SWalHead) + pTable->maxRowSize;
- SWalHead *pHead = taosAllocateQitem(size);
- pHead->version = 0;
- pHead->len = pOper->rowSize;
- pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
+ int32_t code = sdbInsertHash(pTable, pOper);
+ if (code != TSDB_CODE_SUCCESS) {
+ sdbError("table:%s, failed to insert into hash", pTable->tableName);
+ return code;
+ }
+
+ // just insert data into memory
+ if (pOper->type != SDB_OPER_GLOBAL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
+ SSdbOper *pNewOper = taosAllocateQitem(size);
+
+ SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+ pHead->version = 0;
+ pHead->len = pOper->rowSize;
+ pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
- pOper->rowData = pHead->cont;
- (*pTable->encodeFp)(pOper);
- pHead->len = pOper->rowSize;
+ pOper->rowData = pHead->cont;
+ (*pTable->encodeFp)(pOper);
+ pHead->len = pOper->rowSize;
- int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
- taosFreeQitem(pHead);
- if (code < 0) return code;
+ memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+ if (pNewOper->pMsg != NULL) {
+ sdbTrace("app:%p:%p, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
}
- return sdbInsertHash(pTable, pOper);
+ taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+ return TSDB_CODE_SUCCESS;
}
int32_t sdbDeleteRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
- if (pTable == NULL) return -1;
+ if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
- return -1;
+ return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
}
- void * pMetaRow = pMeta->row;
- assert(pMetaRow != NULL);
-
- if (pOper->type == SDB_OPER_GLOBAL) {
- void * key = sdbGetObjKey(pTable, pOper->pObj);
- int32_t keySize = 0;
- switch (pTable->keyType) {
- case SDB_KEY_STRING:
- case SDB_KEY_VAR_STRING:
- keySize = strlen((char *)key) + 1;
- break;
- case SDB_KEY_INT:
- case SDB_KEY_AUTO:
- keySize = sizeof(uint32_t);
- break;
- default:
- return -1;
- }
+ void *pMetaRow = pMeta->row;
+ if (pMetaRow == NULL) {
+ sdbError("table:%s, record meta is null", pTable->tableName);
+ return TSDB_CODE_MND_SDB_INVAID_META_ROW;
+ }
- int32_t size = sizeof(SWalHead) + keySize;
- SWalHead *pHead = taosAllocateQitem(size);
- pHead->version = 0;
- pHead->len = keySize;
- pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
- memcpy(pHead->cont, key, keySize);
+ int32_t code = sdbDeleteHash(pTable, pOper);
+ if (code != TSDB_CODE_SUCCESS) {
+ sdbError("table:%s, failed to delete from hash", pTable->tableName);
+ return code;
+ }
+
+ // just delete data from memory
+ if (pOper->type != SDB_OPER_GLOBAL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
+ SSdbOper *pNewOper = taosAllocateQitem(size);
+
+ SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+ pHead->version = 0;
+ pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
+
+ pOper->rowData = pHead->cont;
+ (*pTable->encodeFp)(pOper);
+ pHead->len = pOper->rowSize;
- int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
- taosFreeQitem(pHead);
- if (code < 0) return code;
+ memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+ if (pNewOper->pMsg != NULL) {
+ sdbTrace("app:%p:%p, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
}
- return sdbDeleteHash(pTable, pOper);
+ taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+ return TSDB_CODE_SUCCESS;
}
int32_t sdbUpdateRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
- if (pTable == NULL) return -1;
+ if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
if (pMeta == NULL) {
- sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
- return -1;
+ sdbTrace("table:%s, record is not there, update failed", pTable->tableName);
+ return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
+ }
+
+ void *pMetaRow = pMeta->row;
+ if (pMetaRow == NULL) {
+ sdbError("table:%s, record meta is null", pTable->tableName);
+ return TSDB_CODE_MND_SDB_INVAID_META_ROW;
}
- void * pMetaRow = pMeta->row;
- assert(pMetaRow != NULL);
+ int32_t code = sdbUpdateHash(pTable, pOper);
+ if (code != TSDB_CODE_SUCCESS) {
+ sdbError("table:%s, failed to update hash", pTable->tableName);
+ return code;
+ }
+
+ // just update data in memory
+ if (pOper->type != SDB_OPER_GLOBAL) {
+ return TSDB_CODE_SUCCESS;
+ }
- if (pOper->type == SDB_OPER_GLOBAL) {
- int32_t size = sizeof(SWalHead) + pTable->maxRowSize;
- SWalHead *pHead = taosAllocateQitem(size);
- pHead->version = 0;
- pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
+ int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
+ SSdbOper *pNewOper = taosAllocateQitem(size);
- pOper->rowData = pHead->cont;
- (*pTable->encodeFp)(pOper);
- pHead->len = pOper->rowSize;
+ SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+ pHead->version = 0;
+ pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
- int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
- taosFreeQitem(pHead);
- if (code < 0) return code;
- }
-
- return sdbUpdateHash(pTable, pOper);
+ pOper->rowData = pHead->cont;
+ (*pTable->encodeFp)(pOper);
+ pHead->len = pOper->rowSize;
+
+ memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+ if (pNewOper->pMsg != NULL) {
+ sdbTrace("app:%p:%p, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
+ }
+
+ taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+ return TSDB_CODE_SUCCESS;
}
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
@@ -775,3 +841,158 @@ void sdbCloseTable(void *handle) {
free(pTable);
}
+int32_t sdbInitWriteWorker() {
+ tsSdbPool.num = 1;
+ tsSdbPool.writeWorker = (SSdbWriteWorker *)calloc(sizeof(SSdbWriteWorker), tsSdbPool.num);
+
+ if (tsSdbPool.writeWorker == NULL) return -1;
+ for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+ SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+ pWorker->workerId = i;
+ }
+
+ sdbAllocWriteQueue();
+
+ mPrint("sdb write is opened");
+ return 0;
+}
+
+void sdbCleanupWriteWorker() {
+ for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+ SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+ if (pWorker->thread) {
+ taosQsetThreadResume(tsSdbWriteQset);
+ }
+ }
+
+ for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+ SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+ if (pWorker->thread) {
+ pthread_join(pWorker->thread, NULL);
+ }
+ }
+
+ sdbFreeWritequeue();
+
+ mPrint("sdb write is closed");
+}
+
+int32_t sdbAllocWriteQueue() {
+ tsSdbWriteQueue = taosOpenQueue();
+ if (tsSdbWriteQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
+
+ tsSdbWriteQset = taosOpenQset();
+ if (tsSdbWriteQset == NULL) {
+ taosCloseQueue(tsSdbWriteQueue);
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
+ }
+ taosAddIntoQset(tsSdbWriteQset, tsSdbWriteQueue, NULL);
+
+ tsSdbWriteQall = taosAllocateQall();
+ if (tsSdbWriteQall == NULL) {
+ taosCloseQset(tsSdbWriteQset);
+ taosCloseQueue(tsSdbWriteQueue);
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
+ }
+
+ for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+ SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+ pWorker->workerId = i;
+
+ pthread_attr_t thAttr;
+ pthread_attr_init(&thAttr);
+ pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+ if (pthread_create(&pWorker->thread, &thAttr, sdbWorkerFp, pWorker) != 0) {
+ mError("failed to create thread to process sdb write queue, reason:%s", strerror(errno));
+ taosFreeQall(tsSdbWriteQall);
+ taosCloseQset(tsSdbWriteQset);
+ taosCloseQueue(tsSdbWriteQueue);
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
+ }
+
+ pthread_attr_destroy(&thAttr);
+ mTrace("sdb write worker:%d is launched, total:%d", pWorker->workerId, tsSdbPool.num);
+ }
+
+ mTrace("sdb write queue:%p is allocated", tsSdbWriteQueue);
+ return TSDB_CODE_SUCCESS;
+}
+
+void sdbFreeWritequeue() {
+ taosCloseQset(tsSdbWriteQueue);
+ taosFreeQall(tsSdbWriteQall);
+ taosCloseQset(tsSdbWriteQset);
+ tsSdbWriteQall = NULL;
+ tsSdbWriteQset = NULL;
+ tsSdbWriteQueue = NULL;
+}
+
+int sdbWriteToQueue(void *param, void *data, int type) {
+ SWalHead *pHead = data;
+ int size = sizeof(SWalHead) + pHead->len;
+ SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
+ memcpy(pWal, pHead, size);
+
+ taosWriteQitem(tsSdbWriteQueue, type, pWal);
+ return 0;
+}
+
+static void *sdbWorkerFp(void *param) {
+ SWalHead *pHead;
+ SSdbOper *pOper;
+ int32_t type;
+ int32_t numOfMsgs;
+ void * item;
+ void * unUsed;
+
+ while (1) {
+ numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed);
+ if (numOfMsgs == 0) {
+ sdbTrace("sdbWorkerFp: got no message from qset, exiting...");
+ break;
+ }
+
+ for (int32_t i = 0; i < numOfMsgs; ++i) {
+ taosGetQitem(tsSdbWriteQall, &type, &item);
+ if (type == TAOS_QTYPE_RPC) {
+ pOper = (SSdbOper *)item;
+ pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+ } else {
+ pHead = (SWalHead *)item;
+ pOper = NULL;
+ }
+
+ if (pOper != NULL && pOper->pMsg != NULL) {
+ sdbTrace("app:%p:%p, will be processed in sdb queue", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg);
+ }
+
+ int32_t code = sdbWrite(pOper, pHead, type);
+ if (pOper) pOper->retCode = code;
+ }
+
+ walFsync(tsSdbObj.wal);
+
+ // browse all items, and process them one by one
+ taosResetQitems(tsSdbWriteQall);
+ for (int32_t i = 0; i < numOfMsgs; ++i) {
+ taosGetQitem(tsSdbWriteQall, &type, &item);
+ if (type == TAOS_QTYPE_RPC) {
+ pOper = (SSdbOper *)item;
+ if (pOper != NULL && pOper->cb != NULL) {
+ pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode);
+ }
+
+ if (pOper != NULL && pOper->pMsg != NULL) {
+ sdbTrace("app:%p:%p, msg is processed, result:%s", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg,
+ tstrerror(pOper->retCode));
+ }
+
+ dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode);
+ }
+ taosFreeQitem(item);
+ }
+ }
+
+ return NULL;
+}
diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c
index 0ff4da42c557d999842783b084e0c8d5642f1713..996e9ced0bbdbd02a18ffc6bb7a1c86188aca39a 100644
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@@ -65,7 +65,7 @@ int32_t mnodeInitShow() {
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
- tsMnodeShowCache = taosCacheInitWithCb(10, mnodeFreeShowObj);
+ tsMnodeShowCache = taosCacheInitWithCb(5, mnodeFreeShowObj);
return 0;
}
@@ -122,7 +122,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
pShow->type = pShowMsg->type;
pShow->payloadLen = htons(pShowMsg->payloadLen);
- strcpy(pShow->db, pShowMsg->db);
+ tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN);
memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
pShow = mnodePutShowObj(pShow, showObjSize);
@@ -139,7 +139,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
pShowRsp->qhandle = htobe64((uint64_t) pShow);
int32_t code = (*tsMnodeShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->rpcMsg.handle);
- mTrace("%p, show type:%s index:%d, get meta finished, rows:%d cols:%d result:%s", pShow,
+ mTrace("%p, show type:%s index:%d, get meta finished, numOfRows:%d cols:%d result:%s", pShow,
mnodeGetShowType(pShowMsg->type), pShow->index, pShow->numOfRows, pShow->numOfColumns, tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
@@ -179,7 +179,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
mTrace("%p, show is already read finished, numOfReads:%d numOfRows:%d", pShow, pShow->numOfReads, pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows;
}
-
+
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsToRead = pShow->numOfRows - pShow->numOfReads;
}
@@ -219,8 +219,10 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1;
+ mTrace("%p, retrieve completed", pShow);
mnodeReleaseShowObj(pShow, true);
} else {
+ mTrace("%p, retrieve not completed yet", pShow);
mnodeReleaseShowObj(pShow, false);
}
@@ -379,10 +381,10 @@ static void *mnodePutShowObj(SShowObj *pShow, int32_t size) {
pShow->index = atomic_add_fetch_32(&tsShowObjIndex, 1);
sprintf(key, "%d", pShow->index);
- SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, pShow, size, 60);
+ SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, pShow, size, 6);
free(pShow);
- mTrace("%p, show is put into cache", newQhandle);
+ mTrace("%p, show is put into cache, index:%s", newQhandle, key);
return newQhandle;
}
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index a2deeaa489faee88f1ad4906466fa6c9b651d383..565ce9910ebcf74b6b97677e71a339050d022e65 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -182,12 +182,19 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
SChildTableObj *pNew = pOper->pObj;
SChildTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
if (pTable != pNew) {
- void *oldTableId = pTable->info.tableId;
+ void *oldTableId = pTable->info.tableId;
void *oldSql = pTable->sql;
void *oldSchema = pTable->schema;
- memcpy(pTable, pNew, pOper->rowSize);
+ void *oldSTable = pTable->superTable;
+ int32_t oldRefCount = pTable->refCount;
+
+ memcpy(pTable, pNew, sizeof(SChildTableObj));
+
+ pTable->refCount = oldRefCount;
pTable->sql = pNew->sql;
pTable->schema = pNew->schema;
+ pTable->superTable = oldSTable;
+
free(pNew);
free(oldSql);
free(oldSchema);
@@ -375,7 +382,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt
pStable->numOfTables++;
if (pStable->vgHash == NULL) {
- pStable->vgHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+ pStable->vgHash = taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
}
if (pStable->vgHash != NULL) {
@@ -436,17 +443,24 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
SSuperTableObj *pNew = pOper->pObj;
SSuperTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
- if (pTable != pNew) {
+ if (pTable != NULL && pTable != pNew) {
void *oldTableId = pTable->info.tableId;
void *oldSchema = pTable->schema;
- memcpy(pTable, pNew, pOper->rowSize);
+ void *oldVgHash = pTable->vgHash;
+ int32_t oldRefCount = pTable->refCount;
+
+ memcpy(pTable, pNew, sizeof(SSuperTableObj));
+
+ pTable->vgHash = oldVgHash;
+ pTable->refCount = oldRefCount;
pTable->schema = pNew->schema;
- free(pNew->vgHash);
free(pNew);
free(oldTableId);
free(oldSchema);
+
+ mnodeDecTableRef(pTable);
}
- mnodeDecTableRef(pTable);
+
return TSDB_CODE_SUCCESS;
}
@@ -669,29 +683,32 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
- mError("table:%s, failed to create, db not selected", pCreate->tableId);
+ mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);
if (pMsg->pTable != NULL && pMsg->retry == 0) {
if (pCreate->getMeta) {
- mTrace("table:%s, continue to get meta", pCreate->tableId);
+ mTrace("app:%p:%p, table:%s, continue to get meta", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return mnodeGetChildTableMeta(pMsg);
} else if (pCreate->igExists) {
- mTrace("table:%s, is already exist", pCreate->tableId);
+ mTrace("app:%p:%p, table:%s, is already exist", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_SUCCESS;
} else {
- mError("table:%s, failed to create, table already exist", pCreate->tableId);
+ mError("app:%p:%p, table:%s, failed to create, table already exist", pMsg->rpcMsg.ahandle, pMsg,
+ pCreate->tableId);
return TSDB_CODE_MND_TABLE_ALREADY_EXIST;
}
}
if (pCreate->numOfTags != 0) {
- mTrace("table:%s, create stable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
+ mTrace("app:%p:%p, table:%s, create stable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+ pCreate->tableId, pMsg->rpcMsg.handle);
return mnodeProcessCreateSuperTableMsg(pMsg);
} else {
- mTrace("table:%s, create ctable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
+ mTrace("app:%p:%p, table:%s, create ctable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+ pCreate->tableId, pMsg->rpcMsg.handle);
return mnodeProcessCreateChildTableMsg(pMsg);
}
}
@@ -700,31 +717,32 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
- mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
+ mError("app:%p:%p, table:%s, failed to drop table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED;
}
if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
- mError("table:%s, failed to drop table, in monitor database", pDrop->tableId);
+ mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg,
+ pDrop->tableId);
return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId);
if (pMsg->pTable == NULL) {
if (pDrop->igNotExists) {
- mTrace("table:%s, table is not exist, think drop success", pDrop->tableId);
+ mTrace("app:%p:%p, table:%s, table is not exist, think drop success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return TSDB_CODE_SUCCESS;
} else {
- mError("table:%s, failed to drop table, table not exist", pDrop->tableId);
+ mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
}
}
if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
- mPrint("table:%s, start to drop stable", pDrop->tableId);
+ mPrint("app:%p:%p, table:%s, start to drop stable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return mnodeProcessDropSuperTableMsg(pMsg);
} else {
- mPrint("table:%s, start to drop ctable", pDrop->tableId);
+ mPrint("app:%p:%p, table:%s, start to drop ctable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return mnodeProcessDropChildTableMsg(pMsg);
}
}
@@ -732,21 +750,25 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->createFlag = htons(pInfo->createFlag);
- mTrace("table:%s, table meta msg is received from thandle:%p, createFlag:%d", pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
+ mTrace("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg,
+ pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
- mError("table:%s, failed to get table meta, db not selected", pInfo->tableId);
+ mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg,
+ pInfo->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId);
if (pMsg->pTable == NULL) {
if (!pInfo->createFlag) {
- mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
+ mError("app:%p:%p, table:%s, failed to get table meta, table not exist", pMsg->rpcMsg.ahandle, pMsg,
+ pInfo->tableId);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
} else {
- mTrace("table:%s, failed to get table meta, start auto create table ", pInfo->tableId);
+ mTrace("app:%p:%p, table:%s, failed to get table meta, start auto create table ", pMsg->rpcMsg.ahandle, pMsg,
+ pInfo->tableId);
return mnodeAutoCreateChildTable(pMsg);
}
} else {
@@ -759,10 +781,12 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
+ if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
- SSuperTableObj *pStable = calloc(1, sizeof(SSuperTableObj));
+ SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj));
if (pStable == NULL) {
- mError("table:%s, failed to create, no enough memory", pCreate->tableId);
+ mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
@@ -780,7 +804,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
pStable->schema = (SSchema *)calloc(1, schemaSize);
if (pStable->schema == NULL) {
free(pStable);
- mError("table:%s, failed to create, no schema input", pCreate->tableId);
+ mError("app:%p:%p, table:%s, failed to create, no schema input", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
}
memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
@@ -799,21 +823,26 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
- .rowSize = sizeof(SSuperTableObj) + schemaSize
+ .rowSize = sizeof(SSuperTableObj) + schemaSize,
+ .pMsg = pMsg
};
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
mnodeDestroySuperTable(pStable);
- mError("table:%s, failed to create, sdb error", pCreate->tableId);
- return TSDB_CODE_MND_SDB_ERROR;
+ mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
} else {
- mLPrint("table:%s, is created, tags:%d fields:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns);
- return TSDB_CODE_SUCCESS;
+ mLPrint("app:%p:%p, table:%s, is created, tags:%d fields:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ pStable->numOfTags, pStable->numOfColumns);
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
+
+ return code;
}
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
+ if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
if (pStable->numOfTables != 0) {
SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
@@ -827,10 +856,11 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
pDrop->vgId = htonl(pVgroup->vgId);
pDrop->uid = htobe64(pStable->uid);
mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
-
- mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, pVgroup->vgId);
+
+ mPrint("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ pVgroup->vgId);
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup);
- SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
+ SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
dnodeSendMsgToDnode(&ipSet, &rpcMsg);
mnodeDecVgroupRef(pVgroup);
}
@@ -842,11 +872,16 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg
};
int32_t code = sdbDeleteRow(&oper);
- mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
+ if (code == TSDB_CODE_SUCCESS) {
+ mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+ }
+
return code;
}
@@ -861,20 +896,31 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *
return -1;
}
-static int32_t mnodeAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) {
+static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ mLPrint("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tstrerror(code));
+
+ return code;
+}
+
+static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
- mError("stable:%s, add tag, too many tags", pStable->info.tableId);
+ mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
return TSDB_CODE_MND_TOO_MANY_TAGS;
}
for (int32_t i = 0; i < ntags; i++) {
if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
- mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name);
+ mError("app:%p:%p, stable:%s, add tag, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+ pStable->info.tableId, schema[i].name);
return TSDB_CODE_MND_TAG_ALREAY_EXIST;
}
if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
- mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name);
+ mError("app:%p:%p, stable:%s, add tag, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ schema[i].name);
return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
}
}
@@ -892,25 +938,38 @@ static int32_t mnodeAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[],
pStable->numOfTags += ntags;
pStable->tversion++;
+ mPrint("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ schema[0].name);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg,
+ .cb = mnodeAddSuperTableTagCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
- mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name);
- return TSDB_CODE_SUCCESS;
+ return code;
}
-static int32_t mnodeDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
+static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ mLPrint("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tstrerror(code));
+ return code;
+}
+
+static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName);
if (col < 0) {
- mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName);
+ mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tagName);
return TSDB_CODE_MND_TAG_NOT_EXIST;
}
@@ -919,25 +978,37 @@ static int32_t mnodeDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
pStable->numOfTags--;
pStable->tversion++;
+ mPrint("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg,
+ .cb = mnodeDropSuperTableTagCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName);
- return TSDB_CODE_SUCCESS;
+
+ return code;
+}
+
+static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ mLPrint("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tstrerror(code));
+ return code;
}
-static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) {
+static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName);
if (col < 0) {
- mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName);
+ mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
+ pStable->info.tableId, oldTagName, newTagName);
return TSDB_CODE_MND_TAG_NOT_EXIST;
}
@@ -950,24 +1021,28 @@ static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTa
if (mnodeFindSuperTableTagIndex(pStable, newTagName) >= 0) {
return TSDB_CODE_MND_TAG_ALREAY_EXIST;
}
-
+
// update
SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
tstrncpy(schema->name, newTagName, sizeof(schema->name));
+ mPrint("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ oldTagName, newTagName);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg,
+ .cb = mnodeModifySuperTableTagNameCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) {
@@ -981,20 +1056,31 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col
return -1;
}
-static int32_t mnodeAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) {
+static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ mLPrint("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tstrerror(code));
+ return code;
+}
+
+static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
+ SDbObj *pDb = pMsg->pDb;
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
if (ncols <= 0) {
- mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId);
+ mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols);
return TSDB_CODE_MND_APP_ERROR;
}
for (int32_t i = 0; i < ncols; i++) {
if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
- mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name);
+ mError("app:%p:%p, stable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+ pStable->info.tableId, schema[i].name);
return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
}
if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
- mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name);
+ mError("app:%p:%p, stable:%s, add column, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+ pStable->info.tableId, schema[i].name);
return TSDB_CODE_MND_TAG_ALREAY_EXIST;
}
}
@@ -1020,25 +1106,38 @@ static int32_t mnodeAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SS
mnodeDecAcctRef(pAcct);
}
+ mPrint("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg,
+ .cb = mnodeAddSuperTableColumnCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("stable %s, succeed to add column", pStable->info.tableId);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
-static int32_t mnodeDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) {
+static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ mLPrint("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ tstrerror(code));
+ return code;
+}
+
+static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
+ SDbObj *pDb = pMsg->pDb;
+ SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName);
if (col <= 0) {
- mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName);
+ mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+ colName);
return TSDB_CODE_MND_FIELD_NOT_EXIST;
}
@@ -1057,19 +1156,22 @@ static int32_t mnodeDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, c
mnodeDecAcctRef(pAcct);
}
+ mPrint("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
- .pObj = pStable
+ .pObj = pStable,
+ .pMsg = pMsg,
+ .cb = mnodeDropSuperTableColumnCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("stable %s, succeed to delete column", pStable->info.tableId);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
// show super tables
@@ -1258,8 +1360,9 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
pMeta->contLen = htons(pMeta->contLen);
pMsg->rpcRsp.rsp = pMeta;
-
- mTrace("stable:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
+
+ mTrace("app:%p:%p, stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId, pTable->uid);
return TSDB_CODE_SUCCESS;
}
@@ -1290,12 +1393,13 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN)*i;
SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
if (pTable == NULL) {
- mError("stable:%s, not exist while get stable vgroup info", stableName);
+ mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName);
mnodeDecTableRef(pTable);
continue;
}
if (pTable->vgHash == NULL) {
- mError("stable:%s, not vgroup exist while get stable vgroup info", stableName);
+ mError("app:%p:%p, stable:%s, not vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
+ stableName);
mnodeDecTableRef(pTable);
// even this super table has no corresponding table, still return
@@ -1320,7 +1424,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
if (pDnode == NULL) break;
- strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
+ tstrncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort);
pVgroupInfo->vgroups[vgSize].numOfIps++;
@@ -1364,10 +1468,10 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
int32_t contLen = 0;
if (pTable->info.type == TSDB_CHILD_TABLE) {
totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags;
- contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen;
+ contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen;
if (pMsg != NULL) {
pTagData = (STagData *)pMsg->schema;
- tagDataLen = ntohl(pTagData->dataLen);
+ tagDataLen = htonl(pTagData->dataLen);
contLen += tagDataLen;
}
} else {
@@ -1430,12 +1534,35 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
return pCreate;
}
-static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) {
+static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
+ if (code != TSDB_CODE_SUCCESS) return code;
+
+ SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
+ SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable);
+ if (pMDCreate == NULL) {
+ return terrno;
+ }
+
+ SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup);
+ SRpcMsg rpcMsg = {
+ .handle = pMsg,
+ .pCont = pMDCreate,
+ .contLen = htonl(pMDCreate->contLen),
+ .code = 0,
+ .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
+ };
+
+ dnodeSendMsgToDnode(&ipSet, &rpcMsg);
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
+ SVgObj *pVgroup = pMsg->pVgroup;
+ SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
if (pTable == NULL) {
- mError("table:%s, failed to alloc memory", pCreate->tableId);
- terrno = TSDB_CODE_MND_OUT_OF_MEMORY;
- return NULL;
+ mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
}
if (pCreate->numOfColumns == 0) {
@@ -1453,10 +1580,10 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb
STagData *pTagData = (STagData *) pCreate->schema; // it is a tag key
SSuperTableObj *pSuperTable = mnodeGetSuperTable(pTagData->name);
if (pSuperTable == NULL) {
- mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
+ mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
+ pCreate->tableId, pTagData->name);
mnodeDestroyChildTable(pTable);
- terrno = TSDB_CODE_MND_INVALID_TABLE_NAME;
- return NULL;
+ return TSDB_CODE_MND_INVALID_TABLE_NAME;
}
mnodeDecTableRef(pSuperTable);
@@ -1475,8 +1602,7 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb
pTable->schema = (SSchema *) calloc(1, schemaSize);
if (pTable->schema == NULL) {
free(pTable);
- terrno = TSDB_CODE_MND_OUT_OF_MEMORY;
- return NULL;
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
}
memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
@@ -1492,42 +1618,51 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb
pTable->sql = calloc(1, pTable->sqlLen);
if (pTable->sql == NULL) {
free(pTable);
- terrno = TSDB_CODE_MND_OUT_OF_MEMORY;
- return NULL;
+ return TSDB_CODE_MND_OUT_OF_MEMORY;
}
memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen);
pTable->sql[pTable->sqlLen - 1] = 0;
- mTrace("table:%s, stream sql len:%d sql:%s", pTable->info.tableId, pTable->sqlLen, pTable->sql);
+ mTrace("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+ pTable->sqlLen, pTable->sql);
}
}
-
+
+ pMsg->pTable = (STableObj *)pTable;
+ mnodeIncTableRef(pMsg->pTable);
+
SSdbOper desc = {0};
desc.type = SDB_OPER_GLOBAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
+ desc.pMsg = pMsg;
+ desc.cb = mnodeDoCreateChildTableCb;
- if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) {
+ int32_t code = sdbInsertRow(&desc);
+ if (code != TSDB_CODE_SUCCESS) {
free(pTable);
- mError("table:%s, update sdb error", pCreate->tableId);
- terrno = TSDB_CODE_MND_SDB_ERROR;
- return NULL;
+ mError("app:%p:%p, table:%s, update sdb error, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
+ tstrerror(code));
+ pMsg->pTable = NULL;
+ return code;
+ } else {
+ mTrace("app:%p:%p, table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid);
+ return TSDB_CODE_SUCCESS;
}
-
- mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64 , pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid);
- return pTable;
}
static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
int32_t code = grantCheck(TSDB_GRANT_TIMESERIES);
if (code != TSDB_CODE_SUCCESS) {
- mError("table:%s, failed to create, grant timeseries failed", pCreate->tableId);
+ mError("app:%p:%p, table:%s, failed to create, grant timeseries failed", pMsg->rpcMsg.ahandle, pMsg,
+ pCreate->tableId);
return code;
}
SVgObj *pVgroup = mnodeGetAvailableVgroup(pMsg->pDb);
if (pVgroup == NULL) {
- mTrace("table:%s, start to create a new vgroup", pCreate->tableId);
+ mTrace("app:%p:%p, table:%s, start to create a new vgroup", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return mnodeCreateVgroup(pMsg, pMsg->pDb);
}
@@ -1535,59 +1670,57 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
if (pMsg->pTable == NULL) {
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid <= 0) {
- mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId);
+ mTrace("app:%p:%p, table:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
+ pVgroup->vgId);
return mnodeCreateVgroup(pMsg, pMsg->pDb);
}
- pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pCreate, pVgroup, sid);
- if (pMsg->pTable == NULL) {
- return terrno;
+ if (pMsg->pVgroup == NULL) {
+ pMsg->pVgroup = pVgroup;
+ mnodeIncVgroupRef(pVgroup);
}
- mnodeIncTableRef(pMsg->pTable);
+ mTrace("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
+ pVgroup->vgId, sid);
+
+ code = mnodeDoCreateChildTable(pMsg, sid);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ } else {
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+ }
}
} else {
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);
}
if (pMsg->pTable == NULL) {
+ mError("app:%p:%p, table:%s, object not found, retry:%d reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, pMsg->retry,
+ tstrerror(terrno));
return terrno;
+ } else {
+ mTrace("app:%p:%p, table:%s, send create msg to vnode again", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+ return mnodeDoCreateChildTableCb(pMsg, TSDB_CODE_SUCCESS);
}
-
- SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable);
- if (pMDCreate == NULL) {
- return terrno;
- }
-
- SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup);
- SRpcMsg rpcMsg = {
- .handle = pMsg,
- .pCont = pMDCreate,
- .contLen = htonl(pMDCreate->contLen),
- .code = 0,
- .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
- };
-
- dnodeSendMsgToDnode(&ipSet, &rpcMsg);
-
- return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
- mError("table:%s, failed to drop ctable, vgroup not exist", pTable->info.tableId);
+ mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId);
return TSDB_CODE_MND_APP_ERROR;
}
SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg));
if (pDrop == NULL) {
- mError("table:%s, failed to drop ctable, no enough memory", pTable->info.tableId);
+ mError("app:%p:%p, table:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
- strcpy(pDrop->tableId, pTable->info.tableId);
+ tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN);
pDrop->vgId = htonl(pTable->vgId);
pDrop->contLen = htonl(sizeof(SMDDropTableMsg));
pDrop->sid = htonl(pTable->sid);
@@ -1595,7 +1728,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup);
- mPrint("table:%s, send drop ctable msg", pDrop->tableId);
+ mPrint("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
SRpcMsg rpcMsg = {
.handle = pMsg,
.pCont = pDrop,
@@ -1609,10 +1742,6 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-static int32_t mnodeModifyChildTableTagValue(SChildTableObj *pTable, char *tagName, char *nContent) {
- return TSDB_CODE_COM_OPS_NOT_SUPPORT;
-}
-
static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) {
SSchema *schema = (SSchema *) pTable->schema;
for (int32_t col = 0; col < pTable->numOfColumns; col++) {
@@ -1624,15 +1753,57 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col
return -1;
}
-static int32_t mnodeAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) {
+static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
+ SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ if (code != TSDB_CODE_SUCCESS) {
+ mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+ tstrerror(code));
+ return code;
+ }
+
+ SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(NULL, pTable);
+ if (pMDCreate == NULL) {
+ return terrno;
+ }
+
+ if (pMsg->pVgroup == NULL) {
+ pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
+ if (pMsg->pVgroup == NULL) {
+ rpcFreeCont(pMDCreate);
+ mError("app:%p:%p, ctable %s, vgId:%d not exist in mnode", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+ pTable->vgId);
+ return TSDB_CODE_MND_VGROUP_NOT_EXIST;
+ }
+ }
+
+ SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup);
+ SRpcMsg rpcMsg = {
+ .handle = pMsg,
+ .pCont = pMDCreate,
+ .contLen = htonl(pMDCreate->contLen),
+ .code = 0,
+ .msgType = TSDB_MSG_TYPE_MD_ALTER_TABLE
+ };
+
+ mTrace("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+ pMsg->pVgroup->vgId);
+
+ dnodeSendMsgToDnode(&ipSet, &rpcMsg);
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
+ SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SDbObj *pDb = pMsg->pDb;
if (ncols <= 0) {
- mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId);
+ mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols);
return TSDB_CODE_MND_APP_ERROR;
}
for (int32_t i = 0; i < ncols; i++) {
if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) {
- mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name);
+ mError("app:%p:%p, ctable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId, schema[i].name);
return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
}
}
@@ -1655,26 +1826,32 @@ static int32_t mnodeAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SS
pAcct->acctInfo.numOfTimeSeries += ncols;
mnodeDecAcctRef(pAcct);
}
-
+
+ mPrint("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
- .pObj = pTable
+ .pObj = pTable,
+ .pMsg = pMsg,
+ .cb = mnodeAlterNormalTableColumnCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("table %s, succeed to add column", pTable->info.tableId);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
-static int32_t mnodeDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) {
+static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
+ SDbObj *pDb = pMsg->pDb;
+ SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName);
if (col <= 0) {
- mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName);
+ mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+ colName);
return TSDB_CODE_MND_FIELD_NOT_EXIST;
}
@@ -1687,20 +1864,23 @@ static int32_t mnodeDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, c
pAcct->acctInfo.numOfTimeSeries--;
mnodeDecAcctRef(pAcct);
}
-
+
+ mPrint("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName);
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
- .pObj = pTable
+ .pObj = pTable,
+ .pMsg = pMsg,
+ .cb = mnodeAlterNormalTableColumnCb
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
-
- mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) {
@@ -1724,7 +1904,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
pMeta->sid = htonl(pTable->sid);
pMeta->precision = pDb->cfg.precision;
pMeta->tableType = pTable->info.type;
- strncpy(pMeta->tableId, pTable->info.tableId, strlen(pTable->info.tableId));
+ tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN);
if (pTable->info.type == TSDB_CHILD_TABLE) {
pMeta->sversion = htons(pTable->superTable->sversion);
@@ -1739,10 +1919,11 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable);
}
-
+
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
- mError("table:%s, failed to get table meta, vgroup not exist", pTable->info.tableId);
+ mError("app:%p:%p, table:%s, failed to get table meta, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId);
return TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
@@ -1756,7 +1937,8 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
}
pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId);
- mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
+ mTrace("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg,
+ pTable->info.tableId, pTable->uid);
return TSDB_CODE_SUCCESS;
}
@@ -1765,10 +1947,11 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
STagData *pTag = (STagData *)pInfo->tags;
- int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen);
+ int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + htonl(pTag->dataLen);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
- mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
+ mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle,
+ pMsg, pInfo->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
@@ -1780,7 +1963,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
pCreateMsg->contLen = htonl(contLen);
memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg));
- mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, ((STagData *)(pCreateMsg->schema))->name);
+ mTrace("app:%p:%p, table:%s, start to create on demand, stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId,
+ ((STagData *)(pCreateMsg->schema))->name);
rpcFreeCont(pMsg->rpcMsg.pCont);
pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
@@ -1791,9 +1975,11 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
}
static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) {
- STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
+ STableMetaMsg *pMeta =
+ rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
if (pMeta == NULL) {
- mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId);
+ mError("app:%p:%p, table:%s, failed to get table meta, no enough memory", pMsg->rpcMsg.ahandle, pMsg,
+ pMsg->pTable->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
@@ -1868,7 +2054,7 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
int32_t numOfTables = 0;
SChildTableObj *pTable = NULL;
- mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables);
+ mPrint("stable:%s, all child tables(%d) will dropped from sdb", pStable->info.tableId, numOfTables);
while (1) {
pIter = mnodeGetNextChildTable(pIter, &pTable);
@@ -1908,11 +2094,13 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
pCfg->sid = htonl(pCfg->sid);
- mTrace("dnode:%d, vgId:%d sid:%d, receive table config msg", pCfg->dnodeId, pCfg->vgId, pCfg->sid);
+ mTrace("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
+ pCfg->vgId, pCfg->sid);
SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
if (pTable == NULL) {
- mError("dnode:%d, vgId:%d sid:%d, table not found", pCfg->dnodeId, pCfg->vgId, pCfg->sid);
+ mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
+ pCfg->vgId, pCfg->sid);
return TSDB_CODE_MND_INVALID_TABLE_ID;
}
@@ -1936,44 +2124,49 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
assert(pTable);
- mPrint("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+ mPrint("app:%p:%p, table:%s, drop table rsp received, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
if (rpcMsg->code != TSDB_CODE_SUCCESS) {
- mError("table:%s, failed to drop in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code));
+ mError("app:%p:%p, table:%s, failed to drop in dnode, reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, tstrerror(rpcMsg->code));
dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
return;
}
if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (mnodeMsg->pVgroup == NULL) {
- mError("table:%s, failed to get vgroup", pTable->info.tableId);
+ mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId);
dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
return;
}
-
+
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
};
-
+
int32_t code = sdbDeleteRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
- mError("table:%s, update ctables sdb error", pTable->info.tableId);
+ mError("app:%p:%p, table:%s, update ctables sdb error", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId);
dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_SDB_ERROR);
return;
}
if (mnodeMsg->pVgroup->numOfTables <= 0) {
- mPrint("vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->pVgroup->vgId);
+ mPrint("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ mnodeMsg->pVgroup->vgId);
mnodeDropVgroup(mnodeMsg->pVgroup, NULL);
}
dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
}
-// handle create table response from dnode
-// if failed, drop the table cached
+/*
+ * handle create table response from dnode
+ * if failed, drop the table cached
+ */
static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
if (rpcMsg->handle == NULL) return;
@@ -1982,44 +2175,68 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
assert(pTable);
- mTrace("table:%s, create table rsp received, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle,
- tstrerror(rpcMsg->code));
- if (rpcMsg->code != TSDB_CODE_SUCCESS) {
- if (mnodeMsg->retry++ < 10) {
- mTrace("table:%s, create table rsp received, retry:%d thandle:%p result:%s", pTable->info.tableId,
- mnodeMsg->retry, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
- dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
- } else {
- mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId,
- mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
-
- SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
- .table = tsChildTableSdb,
- .pObj = pTable
- };
- sdbDeleteRow(&oper);
-
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
- }
- } else {
- mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle,
- tstrerror(rpcMsg->code));
+ if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont;
if (pCreate->getMeta) {
- mTrace("table:%s, continue to get meta", pTable->info.tableId);
+ mTrace("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p result:%s",
+ mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle,
+ tstrerror(rpcMsg->code));
+
mnodeMsg->retry = 0;
dnodeReprocessMnodeWriteMsg(mnodeMsg);
} else {
+ mTrace("app:%p:%p, table:%s, created in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+
+ dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ }
+ } else {
+ if (mnodeMsg->retry++ < 10) {
+ mTrace("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p",
+ mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
+ mnodeMsg->rpcMsg.handle);
+
+ dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ } else {
+ mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+
+ SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
+ sdbDeleteRow(&oper);
+
dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
}
}
}
-// not implemented yet
static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
- mTrace("alter table rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code));
+ if (rpcMsg->handle == NULL) return;
+
+ SMnodeMsg *mnodeMsg = rpcMsg->handle;
+ mnodeMsg->received++;
+
+ SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ assert(pTable);
+
+ if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
+ mTrace("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+
+ dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ } else {
+ if (mnodeMsg->retry++ < 3) {
+ mTrace("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
+ mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
+ mnodeMsg->rpcMsg.handle);
+
+ dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ } else {
+ mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+ pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+ dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ }
+ }
}
static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
@@ -2202,22 +2419,23 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
- mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->rpcMsg.handle);
+ mTrace("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+ pAlter->tableId, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
- mError("table:%s, failed to alter table, db not selected", pAlter->tableId);
+ mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED;
}
if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
- mError("table:%s, failed to alter table, its log db", pAlter->tableId);
+ mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId);
if (pMsg->pTable == NULL) {
- mError("table:%s, failed to alter table, table not exist", pMsg->pTable->tableId);
+ mError("app:%p:%p, table:%s, failed to alter table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
}
@@ -2226,7 +2444,8 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
pAlter->tagValLen = htonl(pAlter->tagValLen);
if (pAlter->numOfCols > 2) {
- mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
+ mError("app:%p:%p, table:%s, error numOfCols:%d in alter table", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId,
+ pAlter->numOfCols);
return TSDB_CODE_MND_APP_ERROR;
}
@@ -2236,35 +2455,32 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT;
if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
- mTrace("table:%s, start to alter stable", pAlter->tableId);
+ mTrace("app:%p:%p, table:%s, start to alter stable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
- code = mnodeAddSuperTableTag(pTable, pAlter->schema, 1);
+ code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
- code = mnodeDropSuperTableTag(pTable, pAlter->schema[0].name);
+ code = mnodeDropSuperTableTag(pMsg, pAlter->schema[0].name);
} else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
- code = mnodeModifySuperTableTagName(pTable, pAlter->schema[0].name, pAlter->schema[1].name);
+ code = mnodeModifySuperTableTagName(pMsg, pAlter->schema[0].name, pAlter->schema[1].name);
} else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
- code = mnodeAddSuperTableColumn(pMsg->pDb, pTable, pAlter->schema, 1);
+ code = mnodeAddSuperTableColumn(pMsg, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
- code = mnodeDropSuperTableColumn(pMsg->pDb, pTable, pAlter->schema[0].name);
+ code = mnodeDropSuperTableColumn(pMsg, pAlter->schema[0].name);
} else {
}
} else {
- mTrace("table:%s, start to alter ctable", pAlter->tableId);
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ mTrace("app:%p:%p, table:%s, start to alter ctable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
- char *tagVal = (char*)(pAlter->schema + pAlter->numOfCols);
- code = mnodeModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal);
+ return TSDB_CODE_COM_OPS_NOT_SUPPORT;
} else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
- code = mnodeAddNormalTableColumn(pMsg->pDb, pTable, pAlter->schema, 1);
+ code = mnodeAddNormalTableColumn(pMsg, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
- code = mnodeDropNormalTableColumn(pMsg->pDb, pTable, pAlter->schema[0].name);
+ code = mnodeDropNormalTableColumn(pMsg, pAlter->schema[0].name);
} else {
}
}
- return code;
+ return code;
}
static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c
index 053ea3ef75471a6ceca058e511b219275b8974b4..b559116e585001e70e3081a9d669f81ae34962ea 100644
--- a/src/mnode/src/mnodeUser.c
+++ b/src/mnode/src/mnodeUser.c
@@ -102,11 +102,13 @@ static int32_t mnodeUserActionDecode(SSdbOper *pOper) {
}
static int32_t mnodeUserActionRestored() {
- if (dnodeIsFirstDeploy()) {
- SAcctObj *pAcct = mnodeGetAcct("root");
- mnodeCreateUser(pAcct, "root", "taosdata");
- mnodeCreateUser(pAcct, "monitor", tsInternalPass);
- mnodeCreateUser(pAcct, "_root", tsInternalPass);
+ int32_t numOfRows = sdbGetNumOfRows(tsUserSdb);
+ if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
+ mPrint("dnode first deploy, create root user");
+ SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER);
+ mnodeCreateUser(pAcct, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS, NULL);
+ mnodeCreateUser(pAcct, "monitor", tsInternalPass, NULL);
+ mnodeCreateUser(pAcct, "_"TSDB_DEFAULT_USER, tsInternalPass, NULL);
mnodeDecAcctRef(pAcct);
}
@@ -170,22 +172,24 @@ void mnodeDecUserRef(SUserObj *pUser) {
return sdbDecRef(tsUserSdb, pUser);
}
-static int32_t mnodeUpdateUser(SUserObj *pUser) {
+static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) {
SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
+ .type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
- .pObj = pUser
+ .pObj = pUser,
+ .pMsg = pMsg
};
int32_t code = sdbUpdateRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- code = TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ mLPrint("user:%s, is altered by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
}
-int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) {
+int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
@@ -212,42 +216,47 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) {
}
pUser = calloc(1, sizeof(SUserObj));
- strcpy(pUser->user, name);
+ tstrncpy(pUser->user, name, TSDB_USER_LEN);
taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass);
strcpy(pUser->acct, pAcct->user);
pUser->createdTime = taosGetTimestampMs();
pUser->superAuth = 0;
pUser->writeAuth = 1;
- if (strcmp(pUser->user, "root") == 0 || strcmp(pUser->user, pUser->acct) == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0 || strcmp(pUser->user, pUser->acct) == 0) {
pUser->superAuth = 1;
}
SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
- .table = tsUserSdb,
- .pObj = pUser,
- .rowSize = sizeof(SUserObj)
+ .type = SDB_OPER_GLOBAL,
+ .table = tsUserSdb,
+ .pObj = pUser,
+ .rowSize = sizeof(SUserObj),
+ .pMsg = pMsg
};
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pUser);
- code = TSDB_CODE_MND_SDB_ERROR;
+ } else {
+ mLPrint("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
}
-static int32_t mnodeDropUser(SUserObj *pUser) {
+static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) {
SSdbOper oper = {
- .type = SDB_OPER_GLOBAL,
+ .type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
- .pObj = pUser
+ .pObj = pUser,
+ .pMsg = pMsg
};
int32_t code = sdbDeleteRow(&oper);
- if (code != TSDB_CODE_SUCCESS) {
- code = TSDB_CODE_MND_SDB_ERROR;
+ if (code == TSDB_CODE_SUCCESS) {
+ mLPrint("user:%s, is dropped by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+ if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
@@ -357,22 +366,25 @@ SUserObj *mnodeGetUserFromConn(void *pConn) {
}
}
+char *mnodeGetUserFromMsg(void *pMsg) {
+ SMnodeMsg *pMnodeMsg = pMsg;
+ if (pMnodeMsg != NULL && pMnodeMsg->pUser != NULL) {
+ return pMnodeMsg->pUser->user;
+ } else {
+ return "system";
+ }
+}
+
static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) {
- int32_t code;
SUserObj *pOperUser = pMsg->pUser;
if (pOperUser->superAuth) {
SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
- code = mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass);
- if (code == TSDB_CODE_SUCCESS) {
- mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user);
- }
+ return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg);
} else {
mError("user:%s, no rights to create user", pOperUser->user);
- code = TSDB_CODE_MND_NO_RIGHTS;
+ return TSDB_CODE_MND_NO_RIGHTS;
}
-
- return code;
}
static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
@@ -392,12 +404,12 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
if ((pAlter->flag & TSDB_ALTER_USER_PASSWD) != 0) {
bool hasRight = false;
- if (strcmp(pOperUser->user, "root") == 0) {
+ if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = true;
} else if (strcmp(pUser->user, pOperUser->user) == 0) {
hasRight = true;
} else if (pOperUser->superAuth) {
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = false;
} else if (strcmp(pOperUser->acct, pUser->acct) != 0) {
hasRight = false;
@@ -409,8 +421,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
if (hasRight) {
memset(pUser->pass, 0, sizeof(pUser->pass));
taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass);
- code = mnodeUpdateUser(pUser);
- mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
+ code = mnodeUpdateUser(pUser, pMsg);
} else {
mError("user:%s, no rights to alter user", pOperUser->user);
code = TSDB_CODE_MND_NO_RIGHTS;
@@ -418,16 +429,16 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
} else if ((pAlter->flag & TSDB_ALTER_USER_PRIVILEGES) != 0) {
bool hasRight = false;
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = false;
} else if (strcmp(pUser->user, pUser->acct) == 0) {
hasRight = false;
- } else if (strcmp(pOperUser->user, "root") == 0) {
+ } else if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = true;
} else if (strcmp(pUser->user, pOperUser->user) == 0) {
hasRight = false;
} else if (pOperUser->superAuth) {
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = false;
} else if (strcmp(pOperUser->acct, pUser->acct) != 0) {
hasRight = false;
@@ -450,8 +461,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
pUser->writeAuth = 1;
}
- code = mnodeUpdateUser(pUser);
- mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
+ code = mnodeUpdateUser(pUser, pMsg);
} else {
mError("user:%s, no rights to alter user", pOperUser->user);
code = TSDB_CODE_MND_NO_RIGHTS;
@@ -482,9 +492,9 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
}
bool hasRight = false;
- if (strcmp(pUser->user, "root") == 0) {
+ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = false;
- } else if (strcmp(pOperUser->user, "root") == 0) {
+ } else if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
hasRight = true;
} else if (strcmp(pUser->user, pOperUser->user) == 0) {
hasRight = false;
@@ -497,10 +507,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
}
if (hasRight) {
- code = mnodeDropUser(pUser);
- if (code == TSDB_CODE_SUCCESS) {
- mLPrint("user:%s, is dropped by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
- }
+ code = mnodeDropUser(pUser, pMsg);
} else {
code = TSDB_CODE_MND_NO_RIGHTS;
}
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index bc6c2665c875d17c832cbe75008dc028a9321433..9553ade3aa4ff2f245f0b194e2301ca8c6a91900 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -74,6 +74,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
pVgroup->pDb = pDb;
pVgroup->prev = NULL;
pVgroup->next = NULL;
+ pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
@@ -132,7 +133,7 @@ static void mnodeVgroupUpdateIdPool(SVgObj *pVgroup) {
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
- memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj **));
+ memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj *));
}
}
}
@@ -149,8 +150,7 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
}
}
- memcpy(pVgroup, pNew, pOper->rowSize);
- free(pNew);
+ memcpy(pVgroup, pNew, tsVgUpdateSize);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mnodeGetDnode(pVgroup->vnodeGid[i].dnodeId);
@@ -252,10 +252,44 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
.pObj = pVgroup
};
- sdbUpdateRow(&oper);
+ if (sdbUpdateRow(&oper) != TSDB_CODE_SUCCESS) {
+ mError("vgId:%d, failed to update vgroup", pVgroup->vgId);
+ }
mnodeSendCreateVgroupMsg(pVgroup, NULL);
}
+/*
+ Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode
+*/
+void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
+ SVnodeLoad *pNextV = NULL;
+
+ void *pIter = NULL;
+ while (1) {
+ SVgObj *pVgroup;
+ pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+ if (pVgroup == NULL) break;
+
+ pNextV = pVloads;
+ int32_t i;
+ for (i = 0; i < openVnodes; ++i) {
+ if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
+ break;
+ }
+ pNextV++;
+ }
+
+ if (i == openVnodes) {
+ mnodeSendCreateVgroupMsg(pVgroup, NULL);
+ }
+
+ mnodeDecVgroupRef(pVgroup);
+ }
+
+ sdbFreeIter(pIter);
+ return;
+}
+
void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
bool dnodeExist = false;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
@@ -299,11 +333,35 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup);
}
+static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
+ if (code != TSDB_CODE_SUCCESS) {
+ pMsg->pVgroup = NULL;
+ return code;
+ }
+
+ SVgObj *pVgroup = pMsg->pVgroup;
+ SDbObj *pDb = pMsg->pDb;
+
+ mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+ for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+ mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
+ }
+
+ mnodeIncVgroupRef(pVgroup);
+ pMsg->expected = pVgroup->numOfVnodes;
+ mnodeSendCreateVgroupMsg(pVgroup, pMsg);
+
+ return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
+ if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
- strcpy(pVgroup->dbName, pDb->name);
+ tstrncpy(pVgroup->dbName, pDb->name, TSDB_DB_NAME_LEN);
pVgroup->numOfVnodes = pDb->cfg.replications;
pVgroup->createdTime = taosGetTimestampMs();
+ pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
if (balanceAllocVnodes(pVgroup) != 0) {
mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
free(pVgroup);
@@ -314,26 +372,22 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
- .rowSize = sizeof(SVgObj)
+ .rowSize = sizeof(SVgObj),
+ .pMsg = pMsg,
+ .cb = mnodeCreateVgroupCb
};
+ pMsg->pVgroup = pVgroup;
+
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
+ pMsg->pVgroup = NULL;
tfree(pVgroup);
- return TSDB_CODE_MND_SDB_ERROR;
- }
-
- mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
- for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
- mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
+ } else {
+ code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
- mnodeIncVgroupRef(pVgroup);
- pMsg->pVgroup = pVgroup;
- pMsg->expected = pVgroup->numOfVnodes;
- mnodeSendCreateVgroupMsg(pVgroup, pMsg);
-
- return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+ return code;
}
void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
@@ -596,7 +650,6 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) {
}
void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
- mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p db:%s", pVgroup->vgId, pVgroup->vgId, ahandle, pVgroup->dbName);
SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.handle = ahandle,
@@ -609,9 +662,12 @@ void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
}
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
- mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+ mTrace("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes,
+ pVgroup->dbName);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
+ mTrace("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId,
+ i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle);
mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
}
}
@@ -717,7 +773,7 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = mnodeGetDnode(pCfg->dnodeId);
if (pDnode == NULL) {
- mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
+ mTrace("dnode:%s, vgId:%d, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
return TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
mnodeDecDnodeRef(pDnode);
@@ -729,6 +785,7 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
}
mnodeDecVgroupRef(pVgroup);
+ mTrace("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp);
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp);
mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
@@ -834,4 +891,4 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
sdbFreeIter(pIter);
mPrint("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
-}
\ No newline at end of file
+}
diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c
index 29b2e6c82b3b028e87ac027ac0f6b5478d1a750d..cd3d722480981e6a10329020861d45c29cde3fd7 100644
--- a/src/mnode/src/mnodeWrite.c
+++ b/src/mnode/src/mnodeWrite.c
@@ -43,7 +43,7 @@ void mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg))
int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
if (pMsg->rpcMsg.pCont == NULL) {
- mError("%p, msg:%s in mwrite queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+ mError("app:%p:%p, msg:%s content is null", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
return TSDB_CODE_MND_INVALID_MSG_LEN;
}
@@ -54,27 +54,31 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
rpcRsp->rsp = ipSet;
rpcRsp->len = sizeof(SRpcIpSet);
- mTrace("%p, msg:%s in mwrite queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse);
+ mTrace("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+ ipSet->inUse);
for (int32_t i = 0; i < ipSet->numOfIps; ++i) {
- mTrace("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i]));
+ mTrace("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i],
+ htons(ipSet->port[i]));
}
return TSDB_CODE_RPC_REDIRECT;
}
if (tsMnodeProcessWriteMsgFp[pMsg->rpcMsg.msgType] == NULL) {
- mError("%p, msg:%s in mwrite queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+ mError("app:%p:%p, msg:%s not processed", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
return TSDB_CODE_MND_MSG_NOT_PROCESSED;
}
int32_t code = mnodeInitMsg(pMsg);
if (code != TSDB_CODE_SUCCESS) {
- mError("%p, msg:%s in mwrite queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code));
+ mError("app:%p:%p, msg:%s not processed, reason:%s", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+ tstrerror(code));
return code;
}
if (!pMsg->pUser->writeAuth) {
- mError("%p, msg:%s in mwrite queue, not processed, no write auth", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+ mError("app:%p:%p, msg:%s not processed, no write auth", pMsg->rpcMsg.ahandle, pMsg,
+ taosMsg[pMsg->rpcMsg.msgType]);
return TSDB_CODE_MND_NO_RIGHTS;
}
diff --git a/src/plugins/http/src/gcHandle.c b/src/plugins/http/src/gcHandle.c
index 176e16301bb764faae12c8329ea410d9546d6268..20cd19df9d115e296e379f8ff03c49fa3a7dd5b6 100644
--- a/src/plugins/http/src/gcHandle.c
+++ b/src/plugins/http/src/gcHandle.c
@@ -52,7 +52,7 @@ bool gcGetUserFromUrl(HttpContext* pContext) {
return false;
}
- strcpy(pContext->user, pParser->path[GC_USER_URL_POS].pos);
+ tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].pos, TSDB_USER_LEN);
return true;
}
@@ -62,7 +62,7 @@ bool gcGetPassFromUrl(HttpContext* pContext) {
return false;
}
- strcpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos);
+ tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
return true;
}
diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c
index 0439083f31e5d4d7c9434535b6dd3f43a7fe6a65..69630336a25eb999c2ae8749d32fb4696a68fe67 100644
--- a/src/plugins/http/src/httpAuth.c
+++ b/src/plugins/http/src/httpAuth.c
@@ -29,6 +29,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) {
char *base64 = (char *)base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, ip:%s, basic token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
+ free(base64);
return false;
}
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index 981492681a8b27c9f9b3cde402ee548f83d26a14..a9082be708d34e45f9b15269f8fae8052ebe7277 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -70,7 +70,7 @@ bool httpInitContexts() {
void httpCleanupContexts() {
if (tsHttpServer.contextCache != NULL) {
SCacheObj *cache = tsHttpServer.contextCache;
- httpPrint("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
+ httpPrint("context cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable));
taosCacheCleanup(tsHttpServer.contextCache);
tsHttpServer.contextCache = NULL;
}
diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c
index 76cc90c48f2629b8a2d7709c8a22201279d20d17..7b49b065718b28427593bbe30c28c9ad352cf5a5 100644
--- a/src/plugins/http/src/httpJson.c
+++ b/src/plugins/http/src/httpJson.c
@@ -139,7 +139,7 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
return 0; // there is no data to dump.
}
} else {
- httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s",
+ httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, isTheLast, ret, buf->buf);
return 0;
}
@@ -442,14 +442,13 @@ void httpJsonPairStatus(JsonBuf* buf, int code) {
httpJsonPair(buf, "status", 6, "error", 5);
httpJsonItemToken(buf);
httpJsonPairIntVal(buf, "code", 4, code);
- if (code >= 0) {
- httpJsonItemToken(buf);
- if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
- httpJsonPair(buf, "desc", 4, "failed to create database", 23);
- } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
- httpJsonPair(buf, "desc", 4, "failed to create table", 22);
- } else
- httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));
+ httpJsonItemToken(buf);
+ if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
+ httpJsonPair(buf, "desc", 4, "failed to create database", 23);
+ } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
+ httpJsonPair(buf, "desc", 4, "failed to create table", 22);
+ } else {
+ httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));
}
}
}
diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c
index c1e33706d5d53df112408e73867dde2cede3642b..e0fc708fa5248bb249db91623e2af68cffc6d447 100644
--- a/src/plugins/http/src/httpSession.c
+++ b/src/plugins/http/src/httpSession.c
@@ -108,7 +108,7 @@ static void httpDestroySession(void *data) {
void httpCleanUpSessions() {
if (tsHttpServer.sessionCache != NULL) {
SCacheObj *cache = tsHttpServer.sessionCache;
- httpPrint("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
+ httpPrint("session cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable));
taosCacheCleanup(tsHttpServer.sessionCache);
tsHttpServer.sessionCache = NULL;
}
diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c
index ce2f7a83bdd4a5a132a31b3a5de1f2ba372b6329..9d584544609b9a31274868b8f981a6329f93b65a 100644
--- a/src/plugins/http/src/httpSql.c
+++ b/src/plugins/http/src/httpSql.c
@@ -205,6 +205,10 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num
}
}
+ if (tscResultsetFetchCompleted(result)) {
+ isContinue = false;
+ }
+
if (isContinue) {
// retrieve next batch of rows
httpTrace("context:%p, fd:%d, ip:%s, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd,
diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c
index 694cdec0a0c84f52d61aff03311f7b355edd664a..d1a0eb90f03de4108ad16b6d2f836058f7273eb2 100644
--- a/src/plugins/http/src/httpUtil.c
+++ b/src/plugins/http/src/httpUtil.c
@@ -202,8 +202,7 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) {
pContext->user, cmdSize);
return false;
}
- memset(multiCmds->cmds + multiCmds->maxSize * (int16_t)sizeof(HttpSqlCmd), 0,
- (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
+ memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
multiCmds->maxSize = (int16_t)cmdSize;
return true;
diff --git a/src/plugins/http/src/restHandle.c b/src/plugins/http/src/restHandle.c
index 93094fa287b9eb272ed1c9224026594e7d12772f..2e04f562ea618db98c5a8d151365649001c44f15 100644
--- a/src/plugins/http/src/restHandle.c
+++ b/src/plugins/http/src/restHandle.c
@@ -65,7 +65,7 @@ bool restGetUserFromUrl(HttpContext* pContext) {
return false;
}
- strcpy(pContext->user, pParser->path[REST_USER_URL_POS].pos);
+ tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].pos, TSDB_USER_LEN);
return true;
}
@@ -75,7 +75,7 @@ bool restGetPassFromUrl(HttpContext* pContext) {
return false;
}
- strcpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos);
+ tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
return true;
}
diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c
index fae11127e1634038a4c5d18a80cf04d07bb82f49..17192db3caa2284bd026f9129fa3e8c8440f860f 100644
--- a/src/plugins/http/src/tgHandle.c
+++ b/src/plugins/http/src/tgHandle.c
@@ -268,17 +268,24 @@ int tgReadSchema(char *fileName) {
httpPrint("open telegraf schema file:%s success", fileName);
fseek(fp, 0, SEEK_END);
- size_t contentSize = (size_t)ftell(fp);
+ int32_t contentSize = (int32_t)ftell(fp);
+ if (contentSize <= 0) {
+ fclose(fp);
+ return 0;
+ }
+
rewind(fp);
- char *content = (char *)calloc(contentSize * sizeof(char) + 1, 1);
- size_t result = fread(content, 1, contentSize, fp);
+ char * content = (char *)calloc(contentSize + 1, 1);
+ int32_t result = fread(content, 1, contentSize, fp);
+
if (result != contentSize) {
httpError("failed to read telegraf schema file:%s", fileName);
fclose(fp);
free(content);
- return -1;
+ return 0;
}
+ content[contentSize] = 0;
int schemaNum = tgParseSchema(content, fileName);
free(content);
diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c
index 735c77ae2148758a13faf1d22ffea6ceb7e1d024..4ea27f4839cad1608df01eeca0874ce5f35b1d69 100644
--- a/src/plugins/monitor/src/monitorMain.c
+++ b/src/plugins/monitor/src/monitorMain.c
@@ -61,7 +61,7 @@ typedef struct {
char ep[TSDB_EP_LEN];
int8_t cmdIndex;
int8_t state;
- char sql[SQL_LENGTH];
+ char sql[SQL_LENGTH + 1];
void * initTimer;
void * diskTimer;
} SMonitorConn;
@@ -177,8 +177,8 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
") tags (acctId binary(%d))",
tsMonitorDbName, TSDB_USER_LEN);
} else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) {
- snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root",
- tsMonitorDbName, "root");
+ snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, TSDB_DEFAULT_USER,
+ tsMonitorDbName, TSDB_DEFAULT_USER);
} else if (cmd == MONITOR_CMD_CREATE_TB_SLOWQUERY) {
snprintf(sql, SQL_LENGTH,
"create table if not exists %s.slowquery(ts timestamp, username "
@@ -208,7 +208,7 @@ static void monitorInitDatabase() {
static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) {
if (-code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || -code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) {
- monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
+ monitorTrace("monitor:%p, sql success, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) {
monitorPrint("dnode:%s is started", tsLocalEp);
}
diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c
index 3266579e337c5b63032e8bcf253f87cf66252b6a..f123d153d199eaa4f8b3475590004ebd4831b185 100644
--- a/src/plugins/mqtt/src/mqttSystem.c
+++ b/src/plugins/mqtt/src/mqttSystem.c
@@ -122,12 +122,11 @@ void mqttCleanUpSystem() {
}
void mqtt_PublishCallback(void** unused, struct mqtt_response_publish* published) {
- mqttPrint("mqtt_PublishCallback");
/* note that published->topic_name is NOT null-terminated (here we'll change it to a c-string) */
char* topic_name = (char*)malloc(published->topic_name_size + 1);
memcpy(topic_name, published->topic_name, published->topic_name_size);
topic_name[published->topic_name_size] = '\0';
- mqttPrint("Received publish('%s'): %s", topic_name, (const char*)published->application_message);
+ mqttPrint("received publish('%s'): %s", topic_name, (const char*)published->application_message);
char _token[128] = {0};
char _dbname[128] = {0};
char _tablename[128] = {0};
@@ -164,14 +163,14 @@ void mqtt_PublishCallback(void** unused, struct mqtt_response_publish* published
void* mqttClientRefresher(void* client) {
while (mttIsRuning) {
mqtt_sync((struct mqtt_client*)client);
- usleep(100000U);
+ taosMsleep(100);
}
- mqttPrint("Exit mqttClientRefresher");
+ mqttTrace("quit refresher");
return NULL;
}
void mqttCleanup(int status, int sockfd, pthread_t* client_daemon) {
- mqttPrint("mqttCleanup");
+ mqttPrint("clean up mqtt module");
if (sockfd != -1) close(sockfd);
if (client_daemon != NULL) pthread_cancel(*client_daemon);
}
@@ -197,7 +196,7 @@ void mqttQueryInsertCallback(void* param, TAOS_RES* result, int32_t code) {
}
void mqttReconnectClient(struct mqtt_client* client, void** reconnect_state_vptr) {
- mqttPrint("mqttReconnectClient");
+ mqttPrint("reconnect client");
struct reconnect_state_t* reconnect_state = *((struct reconnect_state_t**)reconnect_state_vptr);
/* Close the clients socket if this isn't the initial reconnect call */
@@ -213,7 +212,7 @@ void mqttReconnectClient(struct mqtt_client* client, void** reconnect_state_vptr
/* Open a new socket. */
int sockfd = open_nb_socket(reconnect_state->hostname, reconnect_state->port);
if (sockfd == -1) {
- mqttError("Failed to open socket: ");
+ mqttError("failed to open socket: ");
mqttCleanup(EXIT_FAILURE, sockfd, NULL);
}
diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qextbuffer.h
index 23b67083e593f5eab88b11f1f960d6af7138f3e9..2cbef2b1bea66654b262a3fb37061477969960f2 100644
--- a/src/query/inc/qextbuffer.h
+++ b/src/query/inc/qextbuffer.h
@@ -149,7 +149,7 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
* @param pMemBuffer
* @return
*/
-bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer);
+int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer);
/**
*
diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h
index c83c3dbe25d0ede36fbaedca357efa7082013710..e437e1c4e59a7c6fdd99b141de59977dc49e242b 100644
--- a/src/query/inc/qtsbuf.h
+++ b/src/query/inc/qtsbuf.h
@@ -100,10 +100,10 @@ typedef struct STSBuf {
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
uint32_t numOfVnode; // number of vnode stored in current file
- uint32_t tsOrder; // timestamp order in current file
+ int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
-STSBuf* tsBufCreate(bool autoDelete);
+STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder);
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index 4a20502e4ec516de7bddc682f33437ebfa102b68..b4ea1254b7094a8f9a3f83d55da63d990dc21b09 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -251,12 +251,20 @@ alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLeve
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
-typename(A) ::= ids(X). { tSQLSetColumnType (&A, &X); }
+typename(A) ::= ids(X). {
+ X.type = 0;
+ tSQLSetColumnType (&A, &X);
+}
//define binary type, e.g., binary(10), nchar(10)
typename(A) ::= ids(X) LP signed(Y) RP. {
- X.type = -Y; // negative value of name length
- tSQLSetColumnType(&A, &X);
+ if (Y <= 0) {
+ X.type = 0;
+ tSQLSetColumnType(&A, &X);
+ } else {
+ X.type = -Y; // negative value of name length
+ tSQLSetColumnType(&A, &X);
+ }
}
%type signed {int64_t}
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 3e0ff479e9c4afce1b91dfa42464589bd6bf4536..9f7a9dace4ce7279704a26fd8e37e56c597c8724 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -12,6 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+#include
#include "os.h"
#include "qfill.h"
@@ -272,7 +273,7 @@ static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) {
if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) {
pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total;
- qTrace("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%d, total:%"PRId64,
+ qTrace("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64,
pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
assert(pQuery->rec.rows >= 0);
setQueryStatus(pQuery, QUERY_COMPLETED);
@@ -1089,7 +1090,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
// from top to bottom in desc
// from bottom to top in asc order
if (pRuntimeEnv->pTSBuf != NULL) {
- SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery);
+ SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv);
qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows,
pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order);
}
@@ -1215,7 +1216,6 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
// interval query with limit applied
int32_t numOfRes = 0;
-
if (isIntervalQuery(pQuery)) {
numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
} else {
@@ -1399,7 +1399,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pCtx->inputType = pQuery->colList[index].type;
}
- assert(isValidDataType(pCtx->inputType, pCtx->inputBytes));
+ assert(isValidDataType(pCtx->inputType));
pCtx->ptsOutputBuf = NULL;
pCtx->outputBytes = pQuery->pSelectExpr[i].bytes;
@@ -2060,7 +2060,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa
pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data;
}
- qTrace("QInfo:%p realloc output buffer to inc output buffer from: %d rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
+ qTrace("QInfo:%p realloc output buffer to inc output buffer from: %" PRId64 " rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
pQuery->rec.capacity, capacity);
pQuery->rec.capacity = capacity;
@@ -2096,7 +2096,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
}
}
- qTrace("QInfo:%p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv),
+ qTrace("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv),
newSize, pRec->capacity, newSize - pRec->rows);
pRec->capacity = newSize;
@@ -2188,8 +2188,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
* set tag value in SQLFunctionCtx
* e.g.,tag information into input buffer
*/
-static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type,
- int16_t bytes) {
+static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
tVariantDestroy(tag);
if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
@@ -2205,8 +2204,18 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
}
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+ if (isNull(varDataVal(val), type)) {
+ tag->nType = TSDB_DATA_TYPE_NULL;
+ return;
+ }
+
tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
} else {
+ if (isNull(val, type)) {
+ tag->nType = TSDB_DATA_TYPE_NULL;
+ return;
+ }
+
tVariantCreateFromBinary(tag, val, bytes, type);
}
}
@@ -2214,35 +2223,55 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
SQuery *pQuery = pRuntimeEnv->pQuery;
+ SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
SExprInfo *pExprInfo = &pQuery->pSelectExpr[0];
if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) {
-
assert(pExprInfo->base.numOfParams == 1);
- doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag,
- pExprInfo->type, pExprInfo->bytes);
+
+ // todo refactor extract function.
+ int16_t type = -1, bytes = -1;
+ for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+ if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) {
+ type = pQuery->tagColList[i].type;
+ bytes = pQuery->tagColList[i].bytes;
+ }
+ }
+
+ doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
} else {
// set tag value, by which the results are aggregated.
for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
- SExprInfo* pExprInfo = &pQuery->pSelectExpr[idx];
+ SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx];
// ts_comp column required the tag value for join filter
- if (!TSDB_COL_IS_TAG(pExprInfo->base.colInfo.flag)) {
+ if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) {
continue;
}
// todo use tag column index to optimize performance
- doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
- pExprInfo->type, pExprInfo->bytes);
+ doSetTagValueInParam(tsdb, pTableId, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
+ pLocalExprInfo->type, pLocalExprInfo->bytes);
}
// set the join tag for first column
SSqlFuncMsg *pFuncMsg = &pExprInfo->base;
- if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
+ if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
pRuntimeEnv->pTSBuf != NULL) {
assert(pFuncMsg->numOfParams == 1);
- assert(0); // to do fix me
- // doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag);
+
+ // todo refactor
+ int16_t type = -1, bytes = -1;
+ for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+ if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) {
+ type = pQuery->tagColList[i].type;
+ bytes = pQuery->tagColList[i].bytes;
+ }
+ }
+
+ doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
+ qTrace("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64,
+ pRuntimeEnv->pCtx[0].tag.i64Key)
}
}
}
@@ -2465,7 +2494,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1);
}
- qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", pQInfo,
+ qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo,
pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st);
return TSDB_CODE_SUCCESS;
@@ -2923,7 +2952,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
}
if (pQuery->rec.rows <= pQuery->limit.offset) {
- qTrace("QInfo:%p skip rows:%d, new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows,
+ qTrace("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows,
pQuery->limit.offset - pQuery->rec.rows);
pQuery->limit.offset -= pQuery->rec.rows;
@@ -3335,7 +3364,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
- assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
+ //assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb);
@@ -3617,6 +3646,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
int32_t bytes = pQuery->pSelectExpr[col].bytes;
@@ -3666,7 +3696,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
}
if (pQuery->limit.offset < ret) {
- qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, remain:%d, new offset:%d",
+ qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d",
pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0);
ret -= pQuery->limit.offset;
@@ -3680,8 +3710,8 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
pQuery->limit.offset = 0;
return ret;
} else {
- qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, "
- "remain:%d, new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0,
+ qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, "
+ "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0,
pQuery->limit.offset - ret);
pQuery->limit.offset -= ret;
@@ -4229,8 +4259,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
while (pQInfo->groupIndex < numOfGroups) {
SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
- qTrace("QInfo:%p last_row query on group:%d, total group:%d, current group:%d", pQInfo, pQInfo->groupIndex,
- numOfGroups);
+ qTrace("QInfo:%p last_row query on group:%d, total group:%zu, current group:%p", pQInfo, pQInfo->groupIndex,
+ numOfGroups, group);
STsdbQueryCond cond = {
.twindow = pQuery->window,
@@ -4263,12 +4293,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
-
- taosArrayDestroy(s);
if (isFirstLastRowQuery(pQuery)) {
assert(taosArrayGetSize(s) == 1);
}
-
+
+ taosArrayDestroy(s);
+
// here we simply set the first table as current table
pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
@@ -4294,7 +4324,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
while (pQInfo->groupIndex < numOfGroups) {
SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
- qTrace("QInfo:%p group by normal columns group:%d, total group:%d", pQInfo, pQInfo->groupIndex, numOfGroups);
+ qTrace("QInfo:%p group by normal columns group:%d, total group:%zu", pQInfo, pQInfo->groupIndex, numOfGroups);
STsdbQueryCond cond = {
.twindow = pQuery->window,
@@ -4454,6 +4484,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
}
}
+
+ if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+ setQueryStatus(pQuery, QUERY_COMPLETED);
+ }
}
/*
@@ -4476,7 +4510,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
qTrace(
- "QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
+ "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%zu, %"PRId64" points returned, total:%"PRId64", offset:%" PRId64,
pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
pQuery->limit.offset);
}
@@ -4563,7 +4597,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
}
- qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+ qTrace("QInfo:%p current:%"PRId64", total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total);
return;
}
@@ -4572,7 +4606,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
// do check all qualified data blocks
int64_t el = scanMultiTableDataBlocks(pQInfo);
- qTrace("QInfo:%p master scan completed, elapsed time: %lldms, reverse scan start", pQInfo, el);
+ qTrace("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el);
// query error occurred or query is killed, abort current execution
if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
@@ -4587,7 +4621,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
doSaveContext(pQInfo);
el = scanMultiTableDataBlocks(pQInfo);
- qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el);
+ qTrace("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el);
doRestoreContext(pQInfo);
} else {
@@ -4614,7 +4648,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
}
// handle the limitation of output buffer
- qTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
+ qTrace("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
}
/*
@@ -4686,8 +4720,8 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
break;
}
- qTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
- pQInfo, pQuery->limit.offset, pQuery->current->lastKey);
+ qTrace("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
+ pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey);
resetCtxOutputBuf(pRuntimeEnv);
}
@@ -4815,7 +4849,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
limitResults(pRuntimeEnv);
}
- qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+ qTrace("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total);
return;
}
@@ -4835,13 +4869,13 @@ static void tableQueryImpl(SQInfo *pQInfo) {
clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex);
if (pQuery->rec.rows > 0) {
- qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+ qTrace("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total);
return;
}
}
}
- qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total);
+ qTrace("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total);
return;
}
@@ -4897,7 +4931,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st);
if (pQuery->rec.rows == 0) {
- qTrace("QInfo:%p over, %d tables queried, %d rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
+ qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
}
}
@@ -5021,7 +5055,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pQueryMsg->order = htons(pQueryMsg->order);
pQueryMsg->orderColId = htons(pQueryMsg->orderColId);
- pQueryMsg->queryType = htons(pQueryMsg->queryType);
+ pQueryMsg->queryType = htonl(pQueryMsg->queryType);
pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType);
pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols);
@@ -5040,7 +5074,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
}
char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols;
-
for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) {
SColumnInfo *pColInfo = &pQueryMsg->colList[col];
@@ -5190,17 +5223,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pMsg += len;
}
- qTrace("qmsg:%p query %d tables, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, "
- "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64,
- pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
+ qTrace("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, "
+ "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64,
+ pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
- pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset);
+ pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset);
return 0;
}
static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
- qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz);
+ qTrace("qmsg:%p create arithmetic expr from binary string: %s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz);
tExprNode* pExprNode = NULL;
TRY(32) {
@@ -5280,7 +5313,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) {
tagLen += pExprs[i].bytes;
}
- assert(isValidDataType(pExprs[i].type, pExprs[i].bytes));
+ assert(isValidDataType(pExprs[i].type));
}
// TODO refactor
@@ -5380,6 +5413,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) &&
(upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) {
+ assert(rangeFilterArray != NULL);
if (lower == TSDB_RELATION_GREATER_EQUAL) {
if (upper == TSDB_RELATION_LESS_EQUAL) {
pSingleColFilter->fp = rangeFilterArray[4];
@@ -5394,11 +5428,12 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
}
}
} else { // set callback filter function
+ assert(filterArray != NULL);
if (lower != TSDB_RELATION_INVALID) {
pSingleColFilter->fp = filterArray[lower];
if (upper != TSDB_RELATION_INVALID) {
- qError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type);
+ qError("pQInfo:%p failed to get filter function, invalid filter condition: %d", pQInfo, type);
return TSDB_CODE_QRY_INVALID_MSG;
}
} else {
@@ -5640,7 +5675,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
STSBuf *pTSBuf = NULL;
if (pQueryMsg->tsLen > 0) { // open new file to save the result
- char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset;
+ char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset;
pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder);
tsBufResetPos(pTSBuf);
@@ -5818,7 +5853,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
}
pQuery->rec.total += pQuery->rec.rows;
- qTrace("QInfo:%p current numOfRes rows:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+ qTrace("QInfo:%p current numOfRes rows:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total);
if (pQuery->limit.limit > 0 && pQuery->limit.limit == pQuery->rec.total) {
qTrace("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit);
@@ -5871,23 +5906,20 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
bool isSTableQuery = false;
STableGroupInfo groupInfo = {0};
- //todo multitable_query??
- if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) {
- isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
-
+ if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(pTableIdList, 0);
- qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
-
+
+ qTrace("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
- } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) {
+ } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
isSTableQuery = true;
- // TODO: need a macro from TSDB to check if table is super table,
- // also note there's possiblity that only one table in the super table
- if (taosArrayGetSize(pTableIdList) == 1) {
+ // TODO: need a macro from TSDB to check if table is super table
+
+ // also note there's possibility that only one table in the super table
+ if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(pTableIdList, 0);
- // if array size is 1 and assert super table
// group by normal column, do not pass the group by condition to tsdb to group table into different group
int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
@@ -5901,15 +5933,13 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
goto _over;
}
} else {
- SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
+ groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES);
+ groupInfo.numOfTables = taosArrayGetSize(pTableIdList);
- SArray* sa = taosArrayInit(groupInfo.numOfTables, sizeof(STableId));
- for(int32_t i = 0; i < groupInfo.numOfTables; ++i) {
- STableIdInfo* tableId = taosArrayGet(pTableIdList, i);
- taosArrayPush(sa, tableId);
- }
- taosArrayPush(pTableGroup, &sa);
- groupInfo.pGroupList = pTableGroup;
+ SArray* p = taosArrayClone(pTableIdList);
+ taosArrayPush(groupInfo.pGroupList, &p);
+
+ qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, groupInfo.numOfTables);
}
} else {
assert(0);
@@ -5950,7 +5980,7 @@ static void doDestoryQueryInfo(SQInfo* pQInfo) {
freeQInfo(pQInfo);
}
-void qDestroyQueryInfo(qinfo_t qHandle) {
+void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) {
SQInfo* pQInfo = (SQInfo*) qHandle;
if (!isValidQInfo(pQInfo)) {
return;
@@ -5961,10 +5991,14 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
if (ref == 0) {
doDestoryQueryInfo(pQInfo);
+
+ if (fp != NULL) {
+ fp(param);
+ }
}
}
-void qTableQuery(qinfo_t qinfo) {
+void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || pQInfo->signature != pQInfo) {
@@ -5974,7 +6008,7 @@ void qTableQuery(qinfo_t qinfo) {
if (isQueryKilled(pQInfo)) {
qTrace("QInfo:%p it is already killed, abort", pQInfo);
- qDestroyQueryInfo(pQInfo);
+ qDestroyQueryInfo(pQInfo, fp, param);
return;
}
@@ -5990,7 +6024,7 @@ void qTableQuery(qinfo_t qinfo) {
}
sem_post(&pQInfo->dataReady);
- qDestroyQueryInfo(pQInfo);
+ qDestroyQueryInfo(pQInfo, fp, param);
}
int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
@@ -6007,7 +6041,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
}
sem_wait(&pQInfo->dataReady);
- qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
+ qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
pQInfo->code);
return pQInfo->code;
@@ -6083,7 +6117,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
return code;
}
-int32_t qKillQuery(qinfo_t qinfo) {
+int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6091,7 +6125,7 @@ int32_t qKillQuery(qinfo_t qinfo) {
}
setQueryKilled(pQInfo);
- qDestroyQueryInfo(pQInfo);
+ qDestroyQueryInfo(pQInfo, fp, param);
return TSDB_CODE_SUCCESS;
}
@@ -6121,6 +6155,17 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
int32_t rsize = pExprInfo->bytes;
count = 0;
+ int16_t bytes = pExprInfo->bytes;
+ int16_t type = pExprInfo->type;
+
+ for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+ if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) {
+ bytes = pQuery->tagColList[i].bytes;
+ type = pQuery->tagColList[i].type;
+ break;
+ }
+ }
+
while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
int32_t i = pQInfo->tableIndex++;
SGroupItem *item = taosArrayGet(pa, i);
@@ -6138,9 +6183,6 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
*(int32_t *)output = pQInfo->vgId;
output += sizeof(pQInfo->vgId);
- int16_t bytes = pExprInfo->bytes;
- int16_t type = pExprInfo->type;
-
if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char *data = tsdbGetTableName(pQInfo->tsdb, &item->id);
memcpy(output, data, varDataTLen(data));
@@ -6157,7 +6199,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
} else {
if (val == NULL) {
setNull(output, type, bytes);
- } else {
+ } else { // todo here stop will cause client crash
memcpy(output, val, bytes);
}
}
diff --git a/src/query/src/qast.c b/src/query/src/qast.c
index 8d2202563d324de926629e41ad562f8bd4f3873b..dc3b1499bbe02fd389cb1502361fd5cdc46bc54e 100644
--- a/src/query/src/qast.c
+++ b/src/query/src/qast.c
@@ -577,8 +577,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
assert(0);
}
} else {
- int32_t optr = cond.end->optr;
-
+ int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID;
if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) {
bool comp = true;
int32_t ret = 0;
@@ -775,7 +774,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
char * pData = SL_GET_NODE_DATA(pNode);
// todo refactor:
- tstr *name = ((STableIndexElem *)pData)->pTable->name;
+ tstr *name = (*(STable **)pData)->name;
// todo speed up by using hash
if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) {
diff --git a/src/query/src/qextbuffer.c b/src/query/src/qextbuffer.c
index e0a90e04082706eb720960e88356df466dbd017f..35f5e22ed51b9261fa621520d5757a777e9973c8 100644
--- a/src/query/src/qextbuffer.c
+++ b/src/query/src/qextbuffer.c
@@ -245,30 +245,29 @@ static void tExtMemBufferClearFlushoutInfo(tExtMemBuffer *pMemBuffer) {
memset(pFileMeta->flushoutData.pFlushoutInfo, 0, sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize);
}
-bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
+int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
+ int32_t ret = 0;
if (pMemBuffer->numOfTotalElems == 0) {
- return true;
+ return ret;
}
if (pMemBuffer->file == NULL) {
if ((pMemBuffer->file = fopen(pMemBuffer->path, "wb+")) == NULL) {
- return false;
+ ret = TAOS_SYSTEM_ERROR(errno);
+ return ret;
}
}
/* all data has been flushed to disk, ignore flush operation */
if (pMemBuffer->numOfElemsInBuffer == 0) {
- return true;
+ return ret;
}
- bool ret = true;
-
tFilePagesItem *first = pMemBuffer->pHead;
-
while (first != NULL) {
size_t retVal = fwrite((char *)&(first->item), pMemBuffer->pageSize, 1, pMemBuffer->file);
if (retVal <= 0) { // failed to write to buffer, may be not enough space
- ret = false;
+ ret = TAOS_SYSTEM_ERROR(errno);
}
pMemBuffer->fileMeta.numOfElemsInFile += first->item.num;
diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c
index 87add1e69e6f5ad1f7422a0dcb8d69c4b57e95c8..928b9eb8738e5c3ee299ed595fb163182c76e5a8 100644
--- a/src/query/src/qparserImpl.c
+++ b/src/query/src/qparserImpl.c
@@ -497,10 +497,18 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) {
* number of bytes in UCS-4 format, which is 4 times larger than the
* number of characters
*/
- pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+ if (type->type == 0) {
+ pField->bytes = 0;
+ } else {
+ pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+ }
} else if (i == TSDB_DATA_TYPE_BINARY) {
/* for binary, the TOKENTYPE is the length of binary */
- pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+ if (type->type == 0) {
+ pField->bytes = 0;
+ } else {
+ pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+ }
}
break;
}
diff --git a/src/query/src/qpercentile.c b/src/query/src/qpercentile.c
index 367028928b71c345123eaf432fb72178b81957f3..577fddb8080d0949d4799e59e607ff7a7ebd6568 100644
--- a/src/query/src/qpercentile.c
+++ b/src/query/src/qpercentile.c
@@ -283,7 +283,7 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE
break;
};
default: {
- uError("MemBucket:%p,not support data type %d,failed", *pBucket, pBucket->dataType);
+ uError("MemBucket:%p,not support data type %d,failed", pBucket, pBucket->dataType);
tfree(pBucket);
return NULL;
}
diff --git a/src/query/src/qtokenizer.c b/src/query/src/qtokenizer.c
index e6340cc678f25cc2c82a1da9deae5c01b3504863..80d59a384eb54309229aa6e6f05cb6d1301f62aa 100644
--- a/src/query/src/qtokenizer.c
+++ b/src/query/src/qtokenizer.c
@@ -25,7 +25,7 @@
// All the keywords of the SQL language are stored in a hash table
typedef struct SKeyword {
const char* name; // The keyword name
- uint8_t type; // type
+ uint16_t type; // type
uint8_t len; // length
} SKeyword;
@@ -655,4 +655,4 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
return t0;
}
-bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
\ No newline at end of file
+bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
diff --git a/src/query/src/qtsbuf.c b/src/query/src/qtsbuf.c
index 834f648b7a83ac12d2cc3eb0e6b7a387d5aa98e8..85aa2e79032fb5c359403117db685eb009b21c18 100644
--- a/src/query/src/qtsbuf.c
+++ b/src/query/src/qtsbuf.c
@@ -15,7 +15,7 @@ static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader);
* @param path
* @return
*/
-STSBuf* tsBufCreate(bool autoDelete) {
+STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
STSBuf* pTSBuf = calloc(1, sizeof(STSBuf));
if (pTSBuf == NULL) {
return NULL;
@@ -40,7 +40,7 @@ STSBuf* tsBufCreate(bool autoDelete) {
pTSBuf->cur.order = TSDB_ORDER_ASC;
pTSBuf->autoDelete = autoDelete;
- pTSBuf->tsOrder = -1;
+ pTSBuf->tsOrder = order;
return pTSBuf;
}
@@ -66,8 +66,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number
STSBufFileHeader header = {0};
fseek(pTSBuf->f, 0, SEEK_SET);
- fread(&header, 1, sizeof(header), pTSBuf->f);
-
+ fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
+
// invalid file
if (header.magic != TS_COMP_FILE_MAGIC) {
return NULL;
@@ -119,7 +119,6 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// ascending by default
pTSBuf->cur.order = TSDB_ORDER_ASC;
-
pTSBuf->autoDelete = autoDelete;
// tscTrace("create tsBuf from file:%s, fd:%d, size:%d, numOfVnode:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
@@ -536,7 +535,9 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
-
+
+ assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
+
int64_t r = fseek(pTSBuf->f, 0, SEEK_SET);
if (r != 0) {
return -1;
@@ -754,7 +755,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
}
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order) {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, order);
STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info);
pBlockInfo->numOfBlocks = numOfBlocks;
@@ -845,7 +846,9 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) {
if (pTSBuf == NULL) {
return NULL;
}
-
+
+ tsBufFlush(pTSBuf);
+
return tsBufCreateFromFile(pTSBuf->path, false);
}
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index 545cef408216e597b1f48e6e6da0968baaf7983c..e75802a98f4e94c23034072d908b8b5ece3acd7a 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -25,17 +25,17 @@
#include
/************ Begin %include sections from the grammar ************************/
-#include
-#include
+#include
#include
#include
+#include
+#include
+#include "tutil.h"
#include "qsqlparser.h"
#include "tstoken.h"
-#include "tutil.h"
#include "tvariant.h"
#include "ttokendef.h"
#include "qsqltype.h"
-
/**************** End of %include directives **********************************/
/* These constants specify the various numeric values for terminal symbols
** in a format understandable to "makeheaders". This section is blank unless
@@ -2262,13 +2262,21 @@ static void yy_reduce(
{ setDefaultCreateDbOption(&yymsp[1].minor.yy374);}
break;
case 102: /* typename ::= ids */
-{ tSQLSetColumnType (&yylhsminor.yy325, &yymsp[0].minor.yy0); }
+{
+ yymsp[0].minor.yy0.type = 0;
+ tSQLSetColumnType (&yylhsminor.yy325, &yymsp[0].minor.yy0);
+}
yymsp[0].minor.yy325 = yylhsminor.yy325;
break;
case 103: /* typename ::= ids LP signed RP */
{
- yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy279; // negative value of name length
- tSQLSetColumnType(&yylhsminor.yy325, &yymsp[-3].minor.yy0);
+ if (yymsp[-1].minor.yy279 <= 0) {
+ yymsp[-3].minor.yy0.type = 0;
+ tSQLSetColumnType(&yylhsminor.yy325, &yymsp[-3].minor.yy0);
+ } else {
+ yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy279; // negative value of name length
+ tSQLSetColumnType(&yylhsminor.yy325, &yymsp[-3].minor.yy0);
+ }
}
yymsp[-3].minor.yy325 = yylhsminor.yy325;
break;
diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c
index 32d06225c513dfef071113278666d451bd9a0ae3..1b64a7aefa786c69f0386186e7f2231580dd14b1 100644
--- a/src/query/src/tvariant.c
+++ b/src/query/src/tvariant.c
@@ -184,7 +184,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_NCHAR: {
dst[0] = '\'';
- taosUcs4ToMbs(pVar->wpz, (wcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
+ taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
int32_t len = strlen(dst);
dst[len] = '\'';
dst[len + 1] = 0;
@@ -416,7 +416,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
}
pVariant->wpz = pWStr;
- *pDestSize = wcslen(pVariant->wpz);
+ *pDestSize = twcslen(pVariant->wpz);
// shrink the allocate memory, no need to check here.
char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE);
@@ -474,6 +474,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
free(pVariant->pz);
pVariant->nLen = 0;
}
+
setNull((char *)result, type, tDataTypeDesc[type].nSize);
return 0;
}
@@ -597,10 +598,10 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
* todo handle the return value
*/
int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
- if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType, pVariant->nLen))) {
+ if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
return -1;
}
-
+
errno = 0; // reset global error code
switch (type) {
@@ -887,7 +888,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
free(pVariant->pz);
pVariant->dKey = v;
} else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) {
- pVariant->dKey = pVariant->i64Key;
+ pVariant->dKey = (double)(pVariant->i64Key);
}
pVariant->nType = TSDB_DATA_TYPE_DOUBLE;
diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp
index f50e1a5a718b0a197c602acec7dc6249410e3e21..1d2d94f3c085ac9429704f7b36a8df4b4cdb67d0 100644
--- a/src/query/tests/tsBufTest.cpp
+++ b/src/query/tests/tsBufTest.cpp
@@ -29,7 +29,7 @@ int64_t* createTsList(int32_t num, int64_t start, int32_t step) {
// simple test
void simpleTest() {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
// write 10 ts points
int32_t num = 10;
@@ -52,7 +52,7 @@ void simpleTest() {
// one large list of ts, the ts list need to be split into several small blocks
void largeTSTest() {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
// write 10 ts points
int32_t num = 1000000;
@@ -75,7 +75,7 @@ void largeTSTest() {
}
void multiTagsTest() {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
int64_t tag = 1;
@@ -105,7 +105,7 @@ void multiTagsTest() {
}
void multiVnodeTagsTest() {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
int64_t start = 10000000;
@@ -143,7 +143,7 @@ void multiVnodeTagsTest() {
}
void loadDataTest() {
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
int64_t oldStart = 10000000;
@@ -221,7 +221,7 @@ void TSTraverse() {
int32_t step = 30;
int32_t numOfVnode = 2;
- STSBuf* pTSBuf = tsBufCreate(true);
+ STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
@@ -359,8 +359,8 @@ void invalidFileTest() {
}
void mergeDiffVnodeBufferTest() {
- STSBuf* pTSBuf1 = tsBufCreate(true);
- STSBuf* pTSBuf2 = tsBufCreate(true);
+ STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC);
+ STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t step = 30;
int32_t num = 1000;
@@ -391,8 +391,8 @@ void mergeDiffVnodeBufferTest() {
}
void mergeIdenticalVnodeBufferTest() {
- STSBuf* pTSBuf1 = tsBufCreate(true);
- STSBuf* pTSBuf2 = tsBufCreate(true);
+ STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC);
+ STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t step = 30;
int32_t num = 1000;
diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp
index c5b1cbf858a69f2d905f6aa655da18eefec3a2b6..59fd326ef47aea53fcd5ba7981f6255f4bd28d88 100644
--- a/src/query/tests/unitTest.cpp
+++ b/src/query/tests/unitTest.cpp
@@ -66,7 +66,7 @@ static void _init_tvariant_nchar(tVariant* t) {
t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE);
t->nType = TSDB_DATA_TYPE_NCHAR;
wcscpy(t->wpz, L"-2000000.8765");
- t->nLen = wcslen(t->wpz);
+ t->nLen = twcslen(t->wpz);
}
int main(int argc, char** argv) {
diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h
index 40fab000564f3730fa2b7e06afec673c9faf3d5e..6ef8fc2d921a3379532bbc0efd2f226ef3389fc5 100644
--- a/src/rpc/inc/rpcTcp.h
+++ b/src/rpc/inc/rpcTcp.h
@@ -21,9 +21,11 @@ extern "C" {
#endif
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
+void taosStopTcpServer(void *param);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
+void taosStopTcpClient(void *chandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
diff --git a/src/rpc/inc/rpcUdp.h b/src/rpc/inc/rpcUdp.h
index fd60f4a089a8fa2259924897754ef0935f47f73a..c1da6a92402367d23677eb5eb3753782248fb46e 100644
--- a/src/rpc/inc/rpcUdp.h
+++ b/src/rpc/inc/rpcUdp.h
@@ -23,6 +23,7 @@ extern "C" {
#include "taosdef.h"
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
+void taosStopUdpConnection(void *handle);
void taosCleanUpUdpConnection(void *handle);
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 989021eb52aec0c02e39f7b21da2b2758df3cea9..f62df5677167d50bfe3c39717efa47c69debc487 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -108,7 +108,7 @@ typedef struct SRpcConn {
uint16_t outTranId; // outgoing transcation ID
uint16_t inTranId; // transcation ID for incoming msg
uint8_t outType; // message type for outgoing request
- char inType; // message type for incoming request
+ uint8_t inType; // message type for incoming request
void *chandle; // handle passed by TCP/UDP connection layer
void *ahandle; // handle provided by upper app layter
int retry; // number of retry for sending request
@@ -153,6 +153,13 @@ void (*taosCleanUpConn[])(void *thandle) = {
taosCleanUpTcpClient
};
+void (*taosStopConn[])(void *thandle) = {
+ taosStopUdpConnection,
+ taosStopUdpConnection,
+ taosStopTcpServer,
+ taosStopTcpClient,
+};
+
int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
taosSendUdpData,
taosSendUdpData,
@@ -289,12 +296,18 @@ void *rpcOpen(const SRpcInit *pInit) {
void rpcClose(void *param) {
SRpcInfo *pRpc = (SRpcInfo *)param;
+ // stop connection to outside first
+ (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+ (*taosStopConn[pRpc->connType])(pRpc->udphandle);
+
+ // close all connections
for (int i = 0; i < pRpc->sessions; ++i) {
if (pRpc->connList && pRpc->connList[i].user[0]) {
rpcCloseConn((void *)(pRpc->connList + i));
}
}
+ // clean up
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
@@ -394,6 +407,8 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
if ( pConn->inType == 0 || pConn->user[0] == 0 ) {
tTrace("%s, connection is already released, rsp wont be sent", pConn->info);
rpcUnlockConn(pConn);
+ rpcFreeCont(pMsg->pCont);
+ rpcDecRef(pRpc);
return;
}
@@ -574,20 +589,26 @@ static void rpcReleaseConn(SRpcConn *pConn) {
char hashstr[40] = {0};
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
taosHashRemove(pRpc->hash, hashstr, size);
-
rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
pConn->pRspMsg = NULL;
- pConn->inType = 0;
- pConn->inTranId = 0;
- } else {
- pConn->outType = 0;
- pConn->outTranId = 0;
- pConn->pReqMsg = NULL;
- }
+
+ if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg);
+ }
- taosFreeId(pRpc->idPool, pConn->sid);
+ // memset could not be used, since lockeBy can not be reset
+ pConn->inType = 0;
+ pConn->outType = 0;
+ pConn->inTranId = 0;
+ pConn->outTranId = 0;
+ pConn->secured = 0;
+ pConn->peerId = 0;
+ pConn->peerIp = 0;
+ pConn->peerPort = 0;
+ pConn->pReqMsg = NULL;
+ pConn->reqMsgLen = 0;
pConn->pContext = NULL;
+ taosFreeId(pRpc->idPool, pConn->sid);
tTrace("%s, rpc connection is released", pConn->info);
}
@@ -611,7 +632,6 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
terrno = TSDB_CODE_RPC_MAX_SESSIONS;
} else {
pConn = pRpc->connList + sid;
- memset(pConn, 0, sizeof(SRpcConn));
pConn->pRpc = pRpc;
pConn->sid = sid;
@@ -621,6 +641,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
pConn->spi = pRpc->spi;
pConn->encrypt = pRpc->encrypt;
if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
+ tTrace("%s %p client connection is allocated", pRpc->label, pConn);
}
return pConn;
@@ -647,7 +668,6 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
terrno = TSDB_CODE_RPC_MAX_SESSIONS;
} else {
pConn = pRpc->connList + sid;
- memset(pConn, 0, sizeof(SRpcConn));
memcpy(pConn->user, pHead->user, tListLen(pConn->user));
pConn->pRpc = pRpc;
pConn->sid = sid;
@@ -676,6 +696,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
+ tTrace("%s %p server connection is allocated", pRpc->label, pConn);
}
return pConn;
@@ -701,7 +722,7 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
if (pConn) {
if (pConn->linkUid != pHead->linkUid) {
terrno = TSDB_CODE_RPC_MISMATCHED_LINK_ID;
- tError("%s %p %p, linkUid:0x%x is not matched with received:0x%x", pRpc->label, pConn, pHead->ahandle, pConn->linkUid, pHead->linkUid);
+ tError("%s %p %p, linkUid:0x%x is not matched with received:0x%x", pRpc->label, pConn, (void*)pHead->ahandle, pConn->linkUid, pHead->linkUid);
pConn = NULL;
}
}
@@ -888,8 +909,10 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
SRpcInfo *pRpc = pConn->pRpc;
+ if (pConn->pReqMsg == NULL) return;
// if there are pending request, notify the app
+ rpcAddRef(pRpc);
tTrace("%s, notify the server app, connection is gone", pConn->info);
SRpcMsg rpcMsg;
@@ -899,6 +922,8 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
rpcMsg.handle = pConn;
rpcMsg.msgType = pConn->inType;
rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ pConn->pReqMsg = NULL;
+ pConn->reqMsgLen = 0;
if (pRpc->cfp) (*(pRpc->cfp))(&rpcMsg, NULL);
}
@@ -939,11 +964,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
terrno = 0;
pConn = rpcProcessMsgHead(pRpc, pRecv);
- if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
- tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
+ tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno,
pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
- }
int32_t code = terrno;
if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) {
@@ -1171,16 +1194,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
msgLen = rpcAddAuthPart(pConn, msg, msgLen);
if ( rpcIsReq(pHead->msgType)) {
- if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
- tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
- pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
- msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+ tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
+ pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
+ msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
- if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
- tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
- pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort,
- htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+ tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+ pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort,
+ htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
//tTrace("connection type is: %d", pConn->connType);
diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index 7035b30b66ec1d0b0d1a1e50cfb1d74f83ff904f..82168f0b0e6da3809cc6820243d30e9635dfcc3f 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -117,7 +117,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
if (code < 0) {
tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
- break;;
+ break;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
@@ -190,22 +190,28 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
}
}
-
-void taosCleanUpTcpServer(void *handle) {
+void taosStopTcpServer(void *handle) {
SServerObj *pServerObj = handle;
- SThreadObj *pThreadObj;
if (pServerObj == NULL) return;
if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
+ tTrace("%s TCP server is stopped", pServerObj->label);
+}
+
+void taosCleanUpTcpServer(void *handle) {
+ SServerObj *pServerObj = handle;
+ SThreadObj *pThreadObj;
+ if (pServerObj == NULL) return;
+
for (int i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
taosStopTcpThread(pThreadObj);
pthread_mutex_destroy(&(pThreadObj->mutex));
}
- tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
+ tTrace("%s TCP server is cleaned up", pServerObj->label);
tfree(pServerObj->pThreadObj);
tfree(pServerObj);
@@ -226,7 +232,7 @@ static void *taosAcceptTcpConnection(void *arg) {
connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
if (connFd == -1) {
if (errno == EINVAL) {
- tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
+ tTrace("%s TCP server stop accepting new connections, exiting", pServerObj->label);
break;
}
@@ -242,13 +248,13 @@ static void *taosAcceptTcpConnection(void *arg) {
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
- pFdObj->port = caddr.sin_port;
- tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
- inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
+ pFdObj->port = htons(caddr.sin_port);
+ tTrace("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
+ inet_ntoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
} else {
close(connFd);
tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
- inet_ntoa(caddr.sin_addr), caddr.sin_port);
+ inet_ntoa(caddr.sin_addr), htons(caddr.sin_port));
}
// pick up next thread for next connection
@@ -299,17 +305,24 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
return NULL;
}
- tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port);
+ tTrace("%s TCP client is initialized, ip:%u:%hu", label, ip, port);
return pThreadObj;
}
+void taosStopTcpClient(void *chandle) {
+ SThreadObj *pThreadObj = chandle;
+ if (pThreadObj == NULL) return;
+
+ tTrace ("%s TCP client is stopped", pThreadObj->label);
+}
+
void taosCleanUpTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
taosStopTcpThread(pThreadObj);
- tTrace ("%s, all connections are cleaned up", pThreadObj->label);
+ tTrace ("%s TCP client is cleaned up", pThreadObj->label);
tfree(pThreadObj);
}
@@ -320,14 +333,22 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
if (fd < 0) return NULL;
+ struct sockaddr_in sin;
+ uint16_t localPort = 0;
+ unsigned int addrlen = sizeof(sin);
+ if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
+ sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
+ localPort = (uint16_t)ntohs(sin.sin_port);
+ }
+
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = ip;
- tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d",
- pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
+ tTrace("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
+ pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
@@ -340,7 +361,10 @@ void taosCloseTcpConnection(void *chandle) {
SFdObj *pFdObj = chandle;
if (pFdObj == NULL) return;
- pFdObj->thandle = NULL;
+ SThreadObj *pThreadObj = pFdObj->pThreadObj;
+ tTrace("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj);
+
+ // pFdObj->thandle = NULL;
pFdObj->closedByApp = 1;
shutdown(pFdObj->fd, SHUT_WR);
}
@@ -367,7 +391,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pThreadObj->shandle;
- recvInfo.thandle = pFdObj->thandle;;
+ recvInfo.thandle = pFdObj->thandle;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
(*(pThreadObj->processData))(&recvInfo);
@@ -385,14 +409,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
- tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
+ tTrace("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
return -1;
}
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
- tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
+ tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
}
@@ -401,8 +425,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
- tError("%s %p, read error, leftLen:%d retLen:%d",
- pThreadObj->label, pFdObj->thandle, leftLen, retLen);
+ tError("%s %p read error, leftLen:%d retLen:%d FD:%p",
+ pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
free(buffer);
return -1;
}
@@ -414,7 +438,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
pInfo->ip = pFdObj->ip;
pInfo->port = pFdObj->port;
pInfo->shandle = pThreadObj->shandle;
- pInfo->thandle = pFdObj->thandle;;
+ pInfo->thandle = pFdObj->thandle;
pInfo->chandle = pFdObj;
pInfo->connType = RPC_CONN_TCP;
@@ -437,7 +461,7 @@ static void *taosProcessTcpData(void *param) {
while (1) {
int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
if (pThreadObj->stop) {
- tTrace("%s, tcp thread get stop event, exiting...", pThreadObj->label);
+ tTrace("%s TCP thread get stop event, exiting...", pThreadObj->label);
break;
}
if (fdNum < 0) continue;
@@ -446,19 +470,19 @@ static void *taosProcessTcpData(void *param) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
- tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle);
+ tTrace("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLRDHUP) {
- tTrace("%s %p, FD RD hang up", pThreadObj->label, pFdObj->thandle);
+ tTrace("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
- tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle);
+ tTrace("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
@@ -527,7 +551,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
- tError("%s %p, TCP thread:%d, number of FDs is negative!!!",
+ tError("%s %p TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
if (pFdObj->prev) {
@@ -542,7 +566,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
- tTrace("%s %p, FD:%p is cleaned, numOfFds:%d",
+ tTrace("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
tfree(pFdObj);
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 279cf7ed49be70d31b3afc86d53b07453578806f..a4c7d6c1459ba7c5c29160df38cebaddb5651771 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -30,7 +30,6 @@
#define RPC_MAX_UDP_SIZE 65480
typedef struct {
- void *signature;
int index;
int fd;
uint16_t port; // peer port
@@ -111,7 +110,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pConn->processData = fp;
pConn->index = i;
pConn->pSet = pSet;
- pConn->signature = pConn;
int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
if (code != 0) {
@@ -132,7 +130,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
return pSet;
}
-void taosCleanUpUdpConnection(void *handle) {
+void taosStopUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn *pConn;
@@ -140,20 +138,32 @@ void taosCleanUpUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
- pConn->signature = NULL;
-
- // shutdown to signal the thread to exit
- if ( pConn->fd >=0) shutdown(pConn->fd, SHUT_RD);
+ if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
+ if (pConn->fd >=0) taosCloseSocket(pConn->fd);
}
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->thread) pthread_join(pConn->thread, NULL);
- if (pConn->fd >=0) taosCloseSocket(pConn->fd);
tfree(pConn->buffer);
- tTrace("UDP chandle:%p is closed", pConn);
+ // tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
+ }
+
+ tTrace("%s UDP is stopped", pSet->label);
+}
+
+void taosCleanUpUdpConnection(void *handle) {
+ SUdpConnSet *pSet = (SUdpConnSet *)handle;
+ SUdpConn *pConn;
+
+ if (pSet == NULL) return;
+
+ for (int i = 0; i < pSet->threads; ++i) {
+ pConn = pSet->udpConn + i;
+ if (pConn->fd >=0) taosCloseSocket(pConn->fd);
}
+ tTrace("%s UDP is cleaned up", pSet->label);
tfree(pSet);
}
@@ -165,7 +175,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
SUdpConn *pConn = pSet->udpConn + pSet->index;
pConn->port = port;
- tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
+ tTrace("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort);
return pConn;
}
@@ -185,15 +195,15 @@ static void *taosRecvUdpData(void *param) {
while (1) {
dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
- if(dataLen == 0) {
- tTrace("data length is 0, socket was closed, exiting");
+ if(dataLen <= 0) {
+ tTrace("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno));
break;
}
port = ntohs(sourceAdd.sin_port);
if (dataLen < sizeof(SRpcHead)) {
- tError("%s recvfrom failed, reason:%s\n", pConn->label, strerror(errno));
+ tError("%s recvfrom failed(%s)", pConn->label, strerror(errno));
continue;
}
@@ -222,7 +232,7 @@ static void *taosRecvUdpData(void *param) {
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
SUdpConn *pConn = (SUdpConn *)chandle;
- if (pConn == NULL || pConn->signature != pConn) return -1;
+ if (pConn == NULL) return -1;
struct sockaddr_in destAdd;
memset(&destAdd, 0, sizeof(destAdd));
diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h
index f52022d79dbd553e333ef26e8d59b86ea871db94..cd64e48347f8fb99cd70000042fa010b325831a5 100644
--- a/src/tsdb/inc/tsdbMain.h
+++ b/src/tsdb/inc/tsdbMain.h
@@ -15,13 +15,16 @@
#ifndef _TD_TSDB_MAIN_H_
#define _TD_TSDB_MAIN_H_
+#include "hash.h"
+#include "tcoding.h"
#include "tglobal.h"
+#include "tkvstore.h"
#include "tlist.h"
+#include "tlog.h"
+#include "tref.h"
#include "tsdb.h"
#include "tskiplist.h"
#include "tutil.h"
-#include "tlog.h"
-#include "tcoding.h"
#ifdef __cplusplus
extern "C" {
@@ -34,174 +37,101 @@ extern int tsdbDebugFlag;
#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbPrint(...) { taosPrintLog("TDB ", 255, __VA_ARGS__); }
-// ------------------------------ TSDB META FILE INTERFACES ------------------------------
-#define TSDB_META_FILE_NAME "meta"
-#define TSDB_META_HASH_FRACTION 1.1
-
-typedef int (*iterFunc)(void *, void *cont, int contLen);
-typedef void (*afterFunc)(void *);
-
-typedef struct {
- int fd; // File descriptor
- int nDel; // number of deletions
- int tombSize; // deleted size
- int64_t size; // Total file size
- void * map; // Map from uid ==> position
- iterFunc iFunc;
- afterFunc aFunc;
- void * appH;
-} SMetaFile;
-
-SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH);
-int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen);
-int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid);
-int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen);
-void tsdbCloseMetaFile(SMetaFile *mfh);
-
-// ------------------------------ TSDB META INTERFACES ------------------------------
-#define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)
-
-typedef struct {
- TSKEY keyFirst;
- TSKEY keyLast;
- int32_t numOfRows;
- void * pData;
-} SMemTable;
-
-// ---------- TSDB TABLE DEFINITION
#define TSDB_MAX_TABLE_SCHEMAS 16
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+
+// Definitions
+// ------------------ tsdbMeta.c
typedef struct STable {
- int8_t type;
+ ETableType type;
+ tstr* name; // NOTE: there a flexible string here
STableId tableId;
- uint64_t superUid; // Super table UID
- int16_t numOfSchemas;
- STSchema ** schema;
- STSchema * tagSchema;
+ uint64_t suid;
+ struct STable* pSuper; // super table pointer
+ uint8_t numOfSchemas;
+ STSchema* schema[TSDB_MAX_TABLE_SCHEMAS];
+ STSchema* tagSchema;
SKVRow tagVal;
- SMemTable * mem;
- SMemTable * imem;
- void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index
- void * eventHandler; // TODO
- void * streamHandler; // TODO
+ SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index
+ void* eventHandler; // TODO
+ void* streamHandler; // TODO
TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure
- struct STable *next; // TODO: remove the next
- struct STable *prev;
- tstr * name; // NOTE: there a flexible string here
- char * sql;
- void * cqhandle;
+ char* sql;
+ void* cqhandle;
+ T_REF_DECLARE();
} STable;
-#define TSDB_GET_TABLE_LAST_KEY(tb) ((tb)->lastKey)
-
-void tsdbEncodeTable(STable *pTable, char *buf, int *contLen);
-STable *tsdbDecodeTable(void *cont, int contLen);
-void tsdbFreeEncode(void *cont);
-
-// ---------- TSDB META HANDLE DEFINITION
typedef struct {
- int32_t maxTables; // Max number of tables
-
- int32_t nTables; // Tables created
-
- STable **tables; // table array
-
- STable *superList; // super table list TODO: change it to list container
-
- void *map; // table map of (uid ===> table)
-
- SMetaFile *mfh; // meta file handle
- int maxRowBytes;
- int maxCols;
- void * pRepo;
+ pthread_rwlock_t rwLock;
+
+ int32_t nTables;
+ STable** tables;
+ SList* superList;
+ SHashObj* uidMap;
+ SKVStore* pStore;
+ int maxRowBytes;
+ int maxCols;
} STsdbMeta;
-// element put in skiplist for each table
-typedef struct STableIndexElem {
- STsdbMeta* pMeta;
- STable* pTable;
-} STableIndexElem;
-
-STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables, void *pRepo);
-int32_t tsdbFreeMeta(STsdbMeta *pMeta);
-STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable);
-
-// ---- Operation on STable
-#define TSDB_TABLE_ID(pTable) ((pTable)->tableId)
-#define TSDB_TABLE_UID(pTable) ((pTable)->uid)
-#define TSDB_TABLE_NAME(pTable) ((pTable)->tableName)
-#define TSDB_TABLE_TYPE(pTable) ((pTable)->type)
-#define TSDB_TABLE_SUPER_TABLE_UID(pTable) ((pTable)->stableUid)
-#define TSDB_TABLE_IS_SUPER_TABLE(pTable) (TSDB_TABLE_TYPE(pTable) == TSDB_SUPER_TABLE)
-#define TSDB_TABLE_TAG_VALUE(pTable) ((pTable)->pTagVal)
-#define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData)
-#define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex)
-
-// ---- Operation on SMetaHandle
-#define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables)
-#define TSDB_NUM_OF_SUPER_TABLES(pHandle) ((pHandle)->numOfSuperTables)
-#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
-#define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
-
-STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo);
-
-STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
-// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
-STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid);
-char *getTSTupleKey(const void * data);
-
+// ------------------ tsdbBuffer.c
typedef struct {
- int blockId;
- int offset;
- int remain;
- int padding;
- char data[];
-} STsdbCacheBlock;
+ int64_t blockId;
+ int offset;
+ int remain;
+ char data[];
+} STsdbBufBlock;
typedef struct {
- int64_t index;
- int numOfCacheBlocks;
- SList * memPool;
-} STsdbBufferPool;
-
+ pthread_cond_t poolNotEmpty;
+ int bufBlockSize;
+ int tBufBlocks;
+ int nBufBlocks;
+ int64_t index;
+ SList* bufBlockList;
+} STsdbBufPool;
+
+// ------------------ tsdbMemTable.c
typedef struct {
- TSKEY keyFirst;
- TSKEY keyLast;
- int64_t numOfRows;
- SList * list;
-} SCacheMem;
+ uint64_t uid;
+ TSKEY keyFirst;
+ TSKEY keyLast;
+ int64_t numOfRows;
+ SSkipList* pData;
+} STableData;
typedef struct {
- int cacheBlockSize;
- int totalCacheBlocks;
- STsdbBufferPool pool;
- STsdbCacheBlock *curBlock;
- SCacheMem * mem;
- SCacheMem * imem;
- TsdbRepoT * pRepo;
-} STsdbCache;
-
-STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo);
-void tsdbFreeCache(STsdbCache *pCache);
-void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
-
-// ------------------------------ TSDB FILE INTERFACES ------------------------------
-#define TSDB_FILE_HEAD_SIZE 512
-#define TSDB_FILE_DELIMITER 0xF00AFA0F
+ T_REF_DECLARE();
+ TSKEY keyFirst;
+ TSKEY keyLast;
+ int64_t numOfRows;
+ STableData** tData;
+ SList* actList;
+ SList* bufBlockList;
+} SMemTable;
-#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
-#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
+enum { TSDB_UPDATE_META, TSDB_DROP_META };
+typedef struct __attribute__((packed)){
+ char act;
+ uint64_t uid;
+} SActObj;
+
+typedef struct {
+ int len;
+ char cont[];
+} SActCont;
+// ------------------ tsdbFile.c
+extern const char* tsdbFileSuffix[];
typedef enum {
- TSDB_FILE_TYPE_HEAD = 0, // .head file type
- TSDB_FILE_TYPE_DATA, // .data file type
- TSDB_FILE_TYPE_LAST, // .last file type
- TSDB_FILE_TYPE_MAX
+ TSDB_FILE_TYPE_HEAD = 0,
+ TSDB_FILE_TYPE_DATA,
+ TSDB_FILE_TYPE_LAST,
+ TSDB_FILE_TYPE_MAX,
+ TSDB_FILE_TYPE_NHEAD,
+ TSDB_FILE_TYPE_NLAST
} TSDB_FILE_TYPE;
-#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX)
-
-extern const char *tsdbFileSuffix[];
-
typedef struct {
uint32_t offset;
uint32_t len;
@@ -211,46 +141,26 @@ typedef struct {
uint32_t totalSubBlocks;
} STsdbFileInfo;
-void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo);
-void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo);
-
typedef struct {
- int fd;
- char fname[128];
+ char fname[TSDB_FILENAME_LEN];
+ int fd;
+
STsdbFileInfo info;
} SFile;
-#define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1)
-
typedef struct {
- int32_t fileId;
- SFile files[TSDB_FILE_TYPE_MAX];
+ int fileId;
+ SFile files[TSDB_FILE_TYPE_MAX];
} SFileGroup;
-// TSDB file handle
typedef struct {
- int maxFGroups;
- int numOfFGroups;
+ pthread_rwlock_t fhlock;
- SFileGroup *fGroup;
+ int maxFGroups;
+ int nFGroups;
+ SFileGroup* pFGroup;
} STsdbFileH;
-#define TSDB_MIN_FILE_ID(fh) (fh)->fGroup[0].fileId
-#define TSDB_MAX_FILE_ID(fh) (fh)->fGroup[(fh)->numOfFGroups - 1].fileId
-
-STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg);
-void tsdbCloseFileH(STsdbFileH *pFileH);
-int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile);
-SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
-int tsdbOpenFile(SFile *pFile, int oflag);
-int tsdbCloseFile(SFile *pFile);
-SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid);
-int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid);
-int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
-
-#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
-
typedef struct {
int numOfFGroups;
SFileGroup *base;
@@ -258,49 +168,49 @@ typedef struct {
int direction;
} SFileGroupIter;
-void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction);
-void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid);
-SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter);
+// ------------------ tsdbMain.c
+typedef struct {
+ int8_t state;
+
+ char* rootDir;
+ STsdbCfg config;
+ STsdbAppH appH;
+ STsdbStat stat;
+ STsdbMeta* tsdbMeta;
+ STsdbBufPool* pPool;
+ SMemTable* mem;
+ SMemTable* imem;
+ STsdbFileH* tsdbFileH;
+ int commit;
+ pthread_t commitThread;
+ pthread_mutex_t mutex;
+ bool repoLocked;
+} STsdbRepo;
+// ------------------ tsdbRWHelper.c
typedef struct {
uint32_t len;
uint32_t offset;
- uint32_t padding; // For padding purpose
+ // uint32_t padding;
uint32_t hasLast : 2;
uint32_t numOfBlocks : 30;
uint64_t uid;
TSKEY maxKey;
-} SCompIdx; /* sizeof(SCompIdx) = 28 */
-
-void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx);
-void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx);
-
-/**
- * if numOfSubBlocks == 0, then the SCompBlock is a sub-block
- * if numOfSubBlocks >= 1, then the SCompBlock is a super-block
- * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to
- * the data block offset and length
- * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the
- * binary
- */
+} SCompIdx;
+
typedef struct {
- int64_t last : 1; // If the block in data file or last file
- int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks
- int32_t algorithm : 8; // Compression algorithm
- int32_t numOfRows : 24; // Number of total points
- int32_t sversion; // Schema version
- int32_t len; // Data block length or nothing
- int16_t numOfSubBlocks; // Number of sub-blocks;
+ int64_t last : 1;
+ int64_t offset : 63;
+ int32_t algorithm : 8;
+ int32_t numOfRows : 24;
+ int32_t sversion;
+ int32_t len;
+ int16_t numOfSubBlocks;
int16_t numOfCols;
TSKEY keyFirst;
TSKEY keyLast;
} SCompBlock;
-// Maximum number of sub-blocks a super-block can have
-#define TSDB_MAX_SUBBLOCKS 8
-#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1)
-#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
-
typedef struct {
int32_t delimiter; // For recovery usage
int32_t checksum; // TODO: decide if checksum logic in this file or make it one API
@@ -308,21 +218,9 @@ typedef struct {
SCompBlock blocks[];
} SCompInfo;
-#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx))
-#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size) \
- do { \
- if (pCompBlock->numOfSubBlocks > 1) { \
- pCompBlock = pCompInfo->blocks + pCompBlock->offset; \
- size = pCompBlock->numOfSubBlocks; \
- } else { \
- size = 1; \
- } \
- } while (0)
-
-// TODO: take pre-calculation into account
typedef struct {
- int16_t colId; // Column ID
- int16_t len; // Column length // TODO: int16_t is not enough
+ int16_t colId;
+ int16_t len;
int32_t type : 8;
int32_t offset : 24;
int64_t sum;
@@ -334,7 +232,6 @@ typedef struct {
char padding[2];
} SCompCol;
-// TODO: Take recover into account
typedef struct {
int32_t delimiter; // For recovery usage
int32_t numOfCols; // For recovery usage
@@ -342,88 +239,10 @@ typedef struct {
SCompCol cols[];
} SCompData;
-STsdbFileH *tsdbGetFile(TsdbRepoT *pRepo);
-
-int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast,
- SDataCols *pCols);
-SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid);
-void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey);
-
-// TSDB repository definition
-typedef struct STsdbRepo {
- char *rootDir;
- // TSDB configuration
- STsdbCfg config;
-
- STsdbAppH appH;
-
- STsdbStat stat;
-
- // The meter meta handle of this TSDB repository
- STsdbMeta *tsdbMeta;
-
- // The cache Handle
- STsdbCache *tsdbCache;
-
- // The TSDB file handle
- STsdbFileH *tsdbFileH;
-
- // Disk tier handle for multi-tier storage
- void *diskTier;
-
- pthread_mutex_t mutex;
-
- int commit;
- pthread_t commitThread;
-
- // A limiter to monitor the resources used by tsdb
- void *limiter;
-
- int8_t state;
-
-} STsdbRepo;
-
-typedef struct {
- int32_t totalLen;
- int32_t len;
- SDataRow row;
-} SSubmitBlkIter;
-
-int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
-SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
-
-#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
-
-// SSubmitMsg Iterator
-typedef struct {
- int32_t totalLen;
- int32_t len;
- SSubmitBlk *pBlock;
-} SSubmitMsgIter;
-
-int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
-SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
-
-int32_t tsdbTriggerCommit(TsdbRepoT *repo);
-int32_t tsdbLockRepo(TsdbRepoT *repo);
-int32_t tsdbUnLockRepo(TsdbRepoT *repo);
-
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
typedef struct {
- tsdb_rw_helper_t type; // helper type
-
- int maxTables;
- int maxRowSize;
- int maxRows;
- int maxCols;
- int minRowsPerFileBlock;
- int maxRowsPerFileBlock;
- int8_t compress;
-} SHelperCfg;
-
-typedef struct {
- int fid;
+ int fid;
TSKEY minKey;
TSKEY maxKey;
// For read/write purpose
@@ -442,82 +261,139 @@ typedef struct {
} SHelperTable;
typedef struct {
- // Global configuration
- SHelperCfg config;
-
- int8_t state;
+ tsdb_rw_helper_t type;
+ STsdbRepo* pRepo;
+ int8_t state;
// For file set usage
SHelperFile files;
- SCompIdx * pCompIdx;
-
+ SCompIdx* pCompIdx;
// For table set usage
SHelperTable tableInfo;
- SCompInfo * pCompInfo;
+ SCompInfo* pCompInfo;
bool hasOldLastBlock;
-
// For block set usage
- SCompData *pCompData;
- SDataCols *pDataCols[2];
-
- void *pBuffer; // Buffer to hold the whole data block
- void *compBuffer; // Buffer for temperary compress/decompress purpose
+ SCompData* pCompData;
+ SDataCols* pDataCols[2];
+ void* pBuffer; // Buffer to hold the whole data block
+ void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
-// --------- Helper state
+
+// Operations
+// ------------------ tsdbMeta.c
+#define TABLE_TYPE(t) (t)->type
+#define TABLE_NAME(t) (t)->name
+#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
+#define TABLE_UID(t) (t)->tableId.uid
+#define TABLE_TID(t) (t)->tableId.tid
+#define TABLE_SUID(t) (t)->suid
+#define TABLE_LASTKEY(t) (t)->lastKey
+
+STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
+void tsdbFreeMeta(STsdbMeta* pMeta);
+int tsdbOpenMeta(STsdbRepo* pRepo);
+int tsdbCloseMeta(STsdbRepo* pRepo);
+STSchema* tsdbGetTableSchema(STable* pTable);
+STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
+STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
+STSchema* tsdbGetTableTagSchema(STable* pTable);
+int tsdbUpdateTable(STsdbRepo* pRepo, STable* pTable, STableCfg* pCfg);
+int tsdbWLockRepoMeta(STsdbRepo* pRepo);
+int tsdbRLockRepoMeta(STsdbRepo* pRepo);
+int tsdbUnlockRepoMeta(STsdbRepo* pRepo);
+void tsdbRefTable(STable* pTable);
+void tsdbUnRefTable(STable* pTable);
+
+// ------------------ tsdbBuffer.c
+STsdbBufPool* tsdbNewBufPool();
+void tsdbFreeBufPool(STsdbBufPool* pBufPool);
+int tsdbOpenBufPool(STsdbRepo* pRepo);
+void tsdbCloseBufPool(STsdbRepo* pRepo);
+SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
+
+// ------------------ tsdbMemTable.c
+int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
+int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
+void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
+int tsdbAsyncCommit(STsdbRepo* pRepo);
+
+// ------------------ tsdbFile.c
+#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
+#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
+#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
+#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
+#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
+#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
+void tsdbFreeFileH(STsdbFileH* pFileH);
+int tsdbOpenFileH(STsdbRepo* pRepo);
+void tsdbCloseFileH(STsdbRepo* pRepo);
+SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid, int maxTables);
+void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
+void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
+SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
+int tsdbOpenFile(SFile* pFile, int oflag);
+void tsdbCloseFile(SFile* pFile);
+int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
+SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
+void tsdbFitRetention(STsdbRepo* pRepo);
+int tsdbUpdateFileHeader(SFile* pFile, uint32_t version);
+int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
+void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
+void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
+
+// ------------------ tsdbRWHelper.c
#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set
#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded
#define TSDB_HELPER_TABLE_SET 0x4 // Table is set
#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded
-
-#define TSDB_HELPER_TYPE(h) ((h)->config.type)
-
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
+#define TSDB_MAX_SUBBLOCKS 8
+#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
+#define helperType(h) (h)->type
+#define helperRepo(h) (h)->pRepo
+#define helperState(h) (h)->state
+
+int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
+int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
+void tsdbDestroyHelper(SRWHelper* pHelper);
+void tsdbResetHelper(SRWHelper* pHelper);
+int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
+int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError);
+void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
+int tsdbWriteDataBlock(SRWHelper* pHelper, SDataCols* pDataCols);
+int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
+int tsdbWriteCompInfo(SRWHelper* pHelper);
+int tsdbWriteCompIdx(SRWHelper* pHelper);
+int tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
+int tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
+int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
+void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
+int tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds);
+int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target);
+
+// ------------------ tsdbMain.c
+#define REPO_ID(r) (r)->config.tsdbId
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
-int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
-int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
-void tsdbDestroyHelper(SRWHelper *pHelper);
-void tsdbResetHelper(SRWHelper *pHelper);
-
-// --------- For set operations
-int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
-void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
-int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
-
-// --------- For read operations
-int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
-int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
-int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
-int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
-int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
-void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols);
-
-// --------- For write operations
-int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);
-int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper);
-int tsdbWriteCompInfo(SRWHelper *pHelper);
-int tsdbWriteCompIdx(SRWHelper *pHelper);
-
-// --------- Other functions need to further organize
-void tsdbFitRetention(STsdbRepo *pRepo);
-int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
-void tsdbAdjustCacheBlocks(STsdbCache *pCache);
-int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
-int tsdbUpdateFileHeader(SFile *pFile, uint32_t version);
-int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg);
-int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
-int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
-STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version);
-STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);
-
-#define DEFAULT_TAG_INDEX_COLUMN 0 // skip list built based on the first column of tags
-
-int compFGroupKey(const void *key, const void *fgroup);
+char* tsdbGetMetaFileName(char* rootDir);
+void tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname);
+int tsdbLockRepo(STsdbRepo* pRepo);
+int tsdbUnlockRepo(STsdbRepo* pRepo);
+char* tsdbGetDataDirName(char* rootDir);
+STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo);
+STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
#ifdef __cplusplus
}
diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c
new file mode 100644
index 0000000000000000000000000000000000000000..16f06c82bbf310eb94ab389d6c8d028d8d4b160c
--- /dev/null
+++ b/src/tsdb/src/tsdbBuffer.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "tsdb.h"
+#include "tsdbMain.h"
+
+#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
+
+static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize);
+static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock);
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+STsdbBufPool *tsdbNewBufPool() {
+ STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool));
+ if (pBufPool == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
+ }
+
+ pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *));
+ if (pBufPool->bufBlockList == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ return pBufPool;
+
+_err:
+ tsdbFreeBufPool(pBufPool);
+ return NULL;
+}
+
+void tsdbFreeBufPool(STsdbBufPool *pBufPool) {
+ if (pBufPool) {
+ if (pBufPool->bufBlockList) {
+ ASSERT(listNEles(pBufPool->bufBlockList) == 0);
+ tdListFree(pBufPool->bufBlockList);
+ }
+
+ pthread_cond_destroy(&pBufPool->poolNotEmpty);
+
+ free(pBufPool);
+ }
+}
+
+int tsdbOpenBufPool(STsdbRepo *pRepo) {
+ STsdbCfg * pCfg = &(pRepo->config);
+ STsdbBufPool *pPool = pRepo->pPool;
+
+ ASSERT(pPool != NULL);
+
+ pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB
+ pPool->tBufBlocks = pCfg->totalBlocks;
+ pPool->nBufBlocks = 0;
+ pPool->index = 0;
+
+ for (int i = 0; i < pCfg->totalBlocks; i++) {
+ STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
+ if (pBufBlock == NULL) goto _err;
+
+ if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
+ tsdbFreeBufBlock(pBufBlock);
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pPool->nBufBlocks++;
+ }
+
+ tsdbTrace("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo),
+ pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks);
+
+ return 0;
+
+_err:
+ tsdbCloseBufPool(pRepo);
+ return -1;
+}
+
+void tsdbCloseBufPool(STsdbRepo *pRepo) {
+ if (pRepo == NULL) return;
+
+ STsdbBufPool * pBufPool = pRepo->pPool;
+ STsdbBufBlock *pBufBlock = NULL;
+
+ if (pBufPool) {
+ SListNode *pNode = NULL;
+ while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) {
+ tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+ tsdbFreeBufBlock(pBufBlock);
+ free(pNode);
+ }
+ }
+
+ tsdbTrace("vgId:%d buffer pool is closed", REPO_ID(pRepo));
+}
+
+SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
+ ASSERT(pRepo != NULL && pRepo->pPool != NULL);
+ ASSERT(IS_REPO_LOCKED(pRepo));
+
+ STsdbBufPool *pBufPool = pRepo->pPool;
+
+ while (POOL_IS_EMPTY(pBufPool)) {
+ pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
+ }
+
+ SListNode * pNode = tdListPopHead(pBufPool->bufBlockList);
+ ASSERT(pNode != NULL);
+ STsdbBufBlock *pBufBlock = NULL;
+ tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+
+ pBufBlock->blockId = pBufPool->index++;
+ pBufBlock->offset = 0;
+ pBufBlock->remain = pBufPool->bufBlockSize;
+
+ tsdbTrace("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
+ return pNode;
+}
+
+// ---------------- LOCAL FUNCTIONS ----------------
+static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) {
+ STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize);
+ if (pBufBlock == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pBufBlock->blockId = 0;
+ pBufBlock->offset = 0;
+ pBufBlock->remain = bufBlockSize;
+
+ return pBufBlock;
+
+_err:
+ tsdbFreeBufBlock(pBufBlock);
+ return NULL;
+}
+
+static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c
deleted file mode 100644
index 24476d89975bd420f53d7a40ac1cb6dd0c961bf0..0000000000000000000000000000000000000000
--- a/src/tsdb/src/tsdbCache.c
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-#include
-
-#include "tsdb.h"
-#include "tsdbMain.h"
-
-static int tsdbAllocBlockFromPool(STsdbCache *pCache);
-static void tsdbFreeBlockList(SList *list);
-static void tsdbFreeCacheMem(SCacheMem *mem);
-static int tsdbAddCacheBlockToPool(STsdbCache *pCache);
-
-STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo) {
- STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
- if (pCache == NULL) return NULL;
-
- if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
- cacheBlockSize *= (1024 * 1024);
-
- if (totalBlocks <= 1) totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
-
- pCache->cacheBlockSize = cacheBlockSize;
- pCache->totalCacheBlocks = totalBlocks;
- pCache->pRepo = pRepo;
-
- STsdbBufferPool *pPool = &(pCache->pool);
- pPool->index = 0;
- pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *));
- if (pPool->memPool == NULL) goto _err;
-
- for (int i = 0; i < totalBlocks; i++) {
- if (tsdbAddCacheBlockToPool(pCache) < 0) goto _err;
- }
-
- pCache->mem = NULL;
- pCache->imem = NULL;
-
- return pCache;
-
-_err:
- tsdbFreeCache(pCache);
- return NULL;
-}
-
-void tsdbFreeCache(STsdbCache *pCache) {
- tsdbFreeCacheMem(pCache->imem);
- tsdbFreeCacheMem(pCache->mem);
- tsdbFreeBlockList(pCache->pool.memPool);
- free(pCache);
-}
-
-void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) {
- if (pCache == NULL) return NULL;
- if (bytes > pCache->cacheBlockSize) return NULL;
-
- if (pCache->curBlock == NULL || pCache->curBlock->remain < bytes) {
- if (pCache->curBlock !=NULL && listNEles(pCache->mem->list) >= pCache->totalCacheBlocks/2) {
- tsdbTriggerCommit(pCache->pRepo);
- }
-
- while (tsdbAllocBlockFromPool(pCache) < 0) {
- // TODO: deal with the error
- // printf("Failed to allocate from cache pool\n");
- }
- }
-
- void *ptr = (void *)(pCache->curBlock->data + pCache->curBlock->offset);
- pCache->curBlock->offset += bytes;
- pCache->curBlock->remain -= bytes;
- memset(ptr, 0, bytes);
- if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key;
- if (key > pCache->mem->keyLast) pCache->mem->keyLast = key;
- pCache->mem->numOfRows++;
-
- return ptr;
-}
-
-static void tsdbFreeBlockList(SList *list) {
- SListNode * node = NULL;
- STsdbCacheBlock *pBlock = NULL;
- while ((node = tdListPopHead(list)) != NULL) {
- tdListNodeGetData(list, node, (void *)(&pBlock));
- free(pBlock);
- listNodeFree(node);
- }
- tdListFree(list);
-}
-
-static void tsdbFreeCacheMem(SCacheMem *mem) {
- if (mem == NULL) return;
- SList *list = mem->list;
- tsdbFreeBlockList(list);
- free(mem);
-}
-
-static int tsdbAllocBlockFromPool(STsdbCache *pCache) {
- STsdbBufferPool *pPool = &(pCache->pool);
-
- tsdbLockRepo(pCache->pRepo);
- if (listNEles(pPool->memPool) == 0) {
- tsdbUnLockRepo(pCache->pRepo);
- return -1;
- }
-
- SListNode *node = tdListPopHead(pPool->memPool);
-
- STsdbCacheBlock *pBlock = NULL;
- tdListNodeGetData(pPool->memPool, node, (void *)(&pBlock));
- pBlock->blockId = pPool->index++;
- pBlock->offset = 0;
- pBlock->remain = pCache->cacheBlockSize;
-
- if (pCache->mem == NULL) { // Create a new one
- pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem));
- if (pCache->mem == NULL) return -1;
- pCache->mem->keyFirst = INT64_MAX;
- pCache->mem->keyLast = 0;
- pCache->mem->numOfRows = 0;
- pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *));
- }
-
- tdListAppendNode(pCache->mem->list, node);
- pCache->curBlock = pBlock;
-
- tsdbUnLockRepo(pCache->pRepo);
-
- return 0;
-}
-
-int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
- STsdbCache *pCache = pRepo->tsdbCache;
- int oldNumOfBlocks = pCache->totalCacheBlocks;
-
- tsdbLockRepo((TsdbRepoT *)pRepo);
-
- ASSERT(pCache->totalCacheBlocks != totalBlocks);
-
- if (pCache->totalCacheBlocks < totalBlocks) {
- ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks);
- int blocksToAdd = pCache->totalCacheBlocks - totalBlocks;
- pCache->totalCacheBlocks = totalBlocks;
- for (int i = 0; i < blocksToAdd; i++) {
- if (tsdbAddCacheBlockToPool(pCache) < 0) {
- tsdbUnLockRepo((TsdbRepoT *)pRepo);
- tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId);
- return -1;
- }
- }
- } else {
- pCache->totalCacheBlocks = totalBlocks;
- tsdbAdjustCacheBlocks(pCache);
- }
- pRepo->config.totalBlocks = totalBlocks;
-
- tsdbUnLockRepo((TsdbRepoT *)pRepo);
- tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);
- return 0;
-}
-
-static int tsdbAddCacheBlockToPool(STsdbCache *pCache) {
- STsdbBufferPool *pPool = &pCache->pool;
-
- STsdbCacheBlock *pBlock = malloc(sizeof(STsdbCacheBlock) + pCache->cacheBlockSize);
- if (pBlock == NULL) return -1;
-
- pBlock->offset = 0;
- pBlock->remain = pCache->cacheBlockSize;
- tdListAppend(pPool->memPool, (void *)(&pBlock));
- pPool->numOfCacheBlocks++;
-
- return 0;
-}
-
-static int tsdbRemoveCacheBlockFromPool(STsdbCache *pCache) {
- STsdbBufferPool *pPool = &pCache->pool;
- STsdbCacheBlock *pBlock = NULL;
-
- ASSERT(pCache->totalCacheBlocks >= 0);
-
- SListNode *node = tdListPopHead(pPool->memPool);
- if (node == NULL) return -1;
-
- tdListNodeGetData(pPool->memPool, node, &pBlock);
- free(pBlock);
- listNodeFree(node);
- pPool->numOfCacheBlocks--;
-
- return 0;
-}
-
-void tsdbAdjustCacheBlocks(STsdbCache *pCache) {
- while (pCache->totalCacheBlocks < pCache->pool.numOfCacheBlocks) {
- if (tsdbRemoveCacheBlockFromPool(pCache) < 0) break;
- }
-}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbCompactor.c b/src/tsdb/src/tsdbCompactor.c
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index f8c00033b2d6a1597b4b322d05e0af77fcfeaac1..8ebb62f341e77b0449a21942a59894b2f2a924cf 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -29,178 +29,172 @@
#include "tutil.h"
#include "ttime.h"
-const char *tsdbFileSuffix[] = {
- ".head", // TSDB_FILE_TYPE_HEAD
- ".data", // TSDB_FILE_TYPE_DATA
- ".last" // TSDB_FILE_TYPE_LAST
-};
-
-static int compFGroup(const void *arg1, const void *arg2);
-static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
-
-STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) {
- STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH));
- if (pFileH == NULL) { // TODO: deal with ERROR here
- return NULL;
+const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".h"};
+
+static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
+static void tsdbDestroyFile(SFile *pFile);
+static int compFGroup(const void *arg1, const void *arg2);
+static int keyFGroupCompFunc(const void *key, const void *fgroup);
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) {
+ STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH));
+ if (pFileH == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- pFileH->maxFGroups = pCfg->keep / pCfg->daysPerFile + 3;
-
- pFileH->fGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup));
- if (pFileH->fGroup == NULL) {
- free(pFileH);
- return NULL;
+ int code = pthread_rwlock_init(&(pFileH->fhlock), NULL);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
}
- DIR *dir = opendir(dataDir);
- if (dir == NULL) {
- free(pFileH);
- return NULL;
- }
+ pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile);
- struct dirent *dp = NULL;
- while ((dp = readdir(dir)) != NULL) {
- if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue;
- int fid = 0;
- sscanf(dp->d_name, "f%d", &fid);
- if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) {
- break;
- // TODO
- }
+ pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup));
+ if (pFileH->pFGroup == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- closedir(dir);
return pFileH;
+
+_err:
+ tsdbFreeFileH(pFileH);
+ return NULL;
}
-void tsdbCloseFileH(STsdbFileH *pFileH) {
+void tsdbFreeFileH(STsdbFileH *pFileH) {
if (pFileH) {
- tfree(pFileH->fGroup);
+ pthread_rwlock_destroy(&pFileH->fhlock);
+ tfree(pFileH->pFGroup);
free(pFileH);
}
}
-static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) {
- uint32_t version;
- char buf[512] = "\0";
+int tsdbOpenFileH(STsdbRepo *pRepo) {
+ ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL);
- tsdbGetFileName(dataDir, fid, suffix, pFile->fname);
- if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1;
- pFile->fd = -1;
- if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1;
+ char *tDataDir = NULL;
+ DIR * dir = NULL;
+ int fid = 0;
+ int vid = 0;
- if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1;
- if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1;
+ SFileGroup fileGroup = {0};
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
- void *pBuf = buf;
- pBuf = taosDecodeFixedU32(pBuf, &version);
- pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info));
+ tDataDir = tsdbGetDataDirName(pRepo->rootDir);
+ if (tDataDir == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
- tsdbCloseFile(pFile);
+ dir = opendir(tDataDir);
+ if (dir == NULL) {
+ tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
- return 0;
-}
+ struct dirent *dp = NULL;
+ while ((dp = readdir(dir)) != NULL) {
+ if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue;
+ sscanf(dp->d_name, "v%df%d", &vid, &fid);
-static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) {
- if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0;
+ if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) continue;
- SFileGroup fGroup = {0};
- fGroup.fileId = fid;
+ memset((void *)(&fileGroup), 0, sizeof(SFileGroup));
+ fileGroup.fileId = fid;
+ for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+ if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) {
+ tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type);
+ goto _err;
+ }
+ }
- for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1;
+ tsdbTrace("vgId:%d file group %d init", REPO_ID(pRepo), fid);
+
+ pFileH->pFGroup[pFileH->nFGroups++] = fileGroup;
+ qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
}
- pFileH->fGroup[pFileH->numOfFGroups++] = fGroup;
- qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup);
+
+ tfree(tDataDir);
+ closedir(dir);
return 0;
+
+_err:
+ for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
+
+ tfree(tDataDir);
+ if (dir != NULL) closedir(dir);
+ tsdbCloseFileH(pRepo);
+ return -1;
}
-/**
- * Create the file group if the file group not exists.
- *
- * @return A pointer to
- */
-SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) {
- if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL;
+void tsdbCloseFileH(STsdbRepo *pRepo) {
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+
+ for (int i = 0; i < pFileH->nFGroups; i++) {
+ SFileGroup *pFGroup = pFileH->pFGroup + i;
+ for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+ tsdbDestroyFile(&pFGroup->files[type]);
+ }
+ }
+}
+
+SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int maxTables) {
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+
+ if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL;
SFileGroup fGroup;
SFileGroup *pFGroup = &fGroup;
- SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid);
+ SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
if (pGroup == NULL) { // if not exists, create one
pFGroup->fileId = fid;
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], &(pFGroup->files[type])) < 0)
+ if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
goto _err;
}
- pFileH->fGroup[pFileH->numOfFGroups++] = fGroup;
- qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup);
- return tsdbSearchFGroup(pFileH, fid);
+ pFileH->pFGroup[pFileH->nFGroups++] = fGroup;
+ qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
+ return tsdbSearchFGroup(pFileH, fid, TD_EQ);
}
return pGroup;
_err:
- // TODO: deal with the err here
+ for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
return NULL;
}
-int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) {
- SFileGroup *pGroup =
- bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey);
- if (pGroup == NULL) return -1;
-
- // Remove from disk
- for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- remove(pGroup->files[type].fname);
- }
-
- // Adjust the memory
- int filesBehind = pFileH->numOfFGroups - (((char *)pGroup - (char *)(pFileH->fGroup)) / sizeof(SFileGroup) + 1);
- if (filesBehind > 0) {
- memmove((void *)pGroup, (void *)((char *)pGroup + sizeof(SFileGroup)), sizeof(SFileGroup) * filesBehind);
- }
- pFileH->numOfFGroups--;
-
- return 0;
-}
-
-void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) {
+void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { // TODO
pIter->direction = direction;
- pIter->base = pFileH->fGroup;
- pIter->numOfFGroups = pFileH->numOfFGroups;
- if (pFileH->numOfFGroups == 0){
+ pIter->base = pFileH->pFGroup;
+ pIter->numOfFGroups = pFileH->nFGroups;
+ if (pFileH->nFGroups == 0) {
pIter->pFileGroup = NULL;
} else {
if (direction == TSDB_FGROUP_ITER_FORWARD) {
- pIter->pFileGroup = pFileH->fGroup;
+ pIter->pFileGroup = pFileH->pFGroup;
} else {
- pIter->pFileGroup = pFileH->fGroup + pFileH->numOfFGroups - 1;
+ pIter->pFileGroup = pFileH->pFGroup + pFileH->nFGroups - 1;
}
}
}
-void tsdbFitRetention(STsdbRepo *pRepo) {
- STsdbFileH *pFileH = pRepo->tsdbFileH;
- SFileGroup *pGroup = pFileH->fGroup;
-
- int mfid =
- tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision) - pFileH->maxFGroups + 3;
-
- while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) {
- tsdbRemoveFileGroup(pFileH, pGroup[0].fileId);
- }
-}
-
-void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) {
+void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { // TODO
if (pIter->numOfFGroups == 0) {
assert(pIter->pFileGroup == NULL);
return;
}
-
- int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE;
- void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), compFGroupKey, flags);
+
+ int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE;
+ void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags);
if (ptr == NULL) {
pIter->pFileGroup = NULL;
} else {
@@ -208,7 +202,7 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) {
}
}
-SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
+SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {//TODO
SFileGroup *ret = pIter->pFileGroup;
if (ret == NULL) return NULL;
@@ -228,147 +222,204 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
return ret;
}
-// int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
-// SCompBlock *pBlock = pStartBlock;
-// for (int i = 0; i < numOfBlocks; i++) {
-// if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
-// pCols->numOfRows += (pCompData->cols[0].len / 8);
-// for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
-// SCompCol *pCompCol = &(pCompData->cols[iCol]);
-// // pCols->numOfRows += pBlock->numOfRows;
-// int k = 0;
-// for (; k < pCols->numOfCols; k++) {
-// if (pCompCol->colId == pCols->cols[k].colId) break;
-// }
-
-// if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
-// (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
-// return -1;
-// }
-// pStartBlock++;
-// }
-// return 0;
-// }
-
-int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) {
- SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx);
- SCompBlock *pStartBlock = NULL;
- SCompBlock *pBlock = NULL;
- int numOfBlocks = pSuperBlock->numOfSubBlocks;
-
- if (numOfBlocks == 1)
- pStartBlock = pSuperBlock;
- else
- pStartBlock = TSDB_COMPBLOCK_AT(pCompInfo, pSuperBlock->offset);
-
- int maxNumOfCols = 0;
- pBlock = pStartBlock;
- for (int i = 0; i < numOfBlocks; i++) {
- if (pBlock->numOfCols > maxNumOfCols) maxNumOfCols = pBlock->numOfCols;
- pBlock++;
+int tsdbOpenFile(SFile *pFile, int oflag) {
+ ASSERT(!TSDB_IS_FILE_OPENED(pFile));
+
+ pFile->fd = open(pFile->fname, oflag, 0755);
+ if (pFile->fd < 0) {
+ tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ return 0;
+}
+
+void tsdbCloseFile(SFile *pFile) {
+ if (TSDB_IS_FILE_OPENED(pFile)) {
+ close(pFile->fd);
+ pFile->fd = -1;
}
+}
- SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * maxNumOfCols);
- if (pCompData == NULL) return -1;
+int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
+ memset((void *)pFile, 0, sizeof(SFile));
+ pFile->fd = -1;
- // Load data from the block
- // if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData));
+ tsdbGetDataFileName(pRepo, fid, type, pFile->fname);
- // Write data block to the file
- {
- // TODO
+ if (access(pFile->fname, F_OK) == 0) {
+ tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname);
+ terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS;
+ goto _err;
}
+ if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) {
+ goto _err;
+ }
- if (pCompData) free(pCompData);
- return 0;
-}
+ pFile->info.size = TSDB_FILE_HEAD_SIZE;
-int compFGroupKey(const void *key, const void *fgroup) {
- int fid = *(int *)key;
- SFileGroup *pFGroup = (SFileGroup *)fgroup;
- if (fid == pFGroup->fileId) {
- return 0;
- } else {
- return fid > pFGroup->fileId? 1:-1;
+ if (tsdbUpdateFileHeader(pFile, 0) < 0) {
+ tsdbCloseFile(pFile);
+ return -1;
}
+
+ tsdbCloseFile(pFile);
+
+ return 0;
+
+_err:
+ return -1;
}
-static int compFGroup(const void *arg1, const void *arg2) {
- return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId;
+SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) {
+ void *ptr =
+ taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags);
+ if (ptr == NULL) return NULL;
+ return (SFileGroup *)ptr;
}
-int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
- if (dataDir == NULL || fname == NULL) return -1;
+void tsdbFitRetention(STsdbRepo *pRepo) {
+ STsdbCfg *pCfg = &(pRepo->config);
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+ SFileGroup *pGroup = pFileH->pFGroup;
+
+ int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) -
+ TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile);
- sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix);
+ pthread_rwlock_wrlock(&(pFileH->fhlock));
- return 0;
+ while (pFileH->nFGroups > 0 && pGroup[0].fileId < mfid) {
+ tsdbRemoveFileGroup(pRepo, pGroup);
+ }
+
+ pthread_rwlock_unlock(&(pFileH->fhlock));
}
-int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function
- if (TSDB_IS_FILE_OPENED(pFile)) return -1;
+int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
+ char buf[TSDB_FILE_HEAD_SIZE] = "\0";
- pFile->fd = open(pFile->fname, oflag, 0755);
- if (pFile->fd < 0) return -1;
+ void *pBuf = (void *)buf;
+ taosEncodeFixedU32((void *)(&pBuf), version);
+ tsdbEncodeSFileInfo((void *)(&pBuf), &(pFile->info));
+
+ taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+
+ if (lseek(pFile->fd, 0, SEEK_SET) < 0) {
+ tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+ if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
return 0;
}
-int tsdbCloseFile(SFile *pFile) {
- int ret = close(pFile->fd);
- pFile->fd = -1;
- return ret;
+int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
+ int tlen = 0;
+ tlen += taosEncodeFixedU32(buf, pInfo->offset);
+ tlen += taosEncodeFixedU32(buf, pInfo->len);
+ tlen += taosEncodeFixedU64(buf, pInfo->size);
+ tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
+ tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
+ tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
+
+ return tlen;
}
-SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) {
- SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid);
- if (pGroup == NULL) return NULL;
+void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
+ buf = taosDecodeFixedU32(buf, &(pInfo->offset));
+ buf = taosDecodeFixedU32(buf, &(pInfo->len));
+ buf = taosDecodeFixedU64(buf, &(pInfo->size));
+ buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
+ buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
+ buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
- for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- tsdbOpenFile(&(pGroup->files[type]), O_RDWR);
- }
- return pGroup;
+ return buf;
}
-int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) {
- memset((void *)pFile, 0, sizeof(SFile));
- pFile->fd = -1;
+void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
+ ASSERT(pFGroup != NULL);
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
- tsdbGetFileName(dataDir, fileId, suffix, pFile->fname);
-
- if (access(pFile->fname, F_OK) == 0) {
- // File already exists
- return -1;
+ SFileGroup fileGroup = *pFGroup;
+
+ int nFilesLeft = pFileH->nFGroups - (POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1);
+ if (nFilesLeft > 0) {
+ memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft);
}
- if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) {
- // TODO: deal with the ERROR here
- return -1;
+ pFileH->nFGroups--;
+ ASSERT(pFileH->nFGroups >= 0);
+
+ for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+ if (remove(fileGroup.files[type].fname) < 0) {
+ tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
+ }
+ tsdbDestroyFile(&fileGroup.files[type]);
}
+}
- pFile->info.size = TSDB_FILE_HEAD_SIZE;
+// ---------------- LOCAL FUNCTIONS ----------------
+static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
+ uint32_t version;
+ char buf[512] = "\0";
- if (tsdbUpdateFileHeader(pFile, 0) < 0) {
- tsdbCloseFile(pFile);
- return -1;
+ tsdbGetDataFileName(pRepo, fid, type, pFile->fname);
+
+ pFile->fd = -1;
+ if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
+
+ if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE,
+ pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+ tsdbError("vgId:%d file %s head part is corrupted", REPO_ID(pRepo), pFile->fname);
+ terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+ goto _err;
}
+ void *pBuf = buf;
+ pBuf = taosDecodeFixedU32(pBuf, &version);
+ pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info));
+
tsdbCloseFile(pFile);
return 0;
+_err:
+ tsdbDestroyFile(pFile);
+ return -1;
}
-void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey,
- TSKEY *maxKey) {
- *minKey = fileId * daysPerFile * tsMsPerDay[precision];
- *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
+static void tsdbDestroyFile(SFile *pFile) { tsdbCloseFile(pFile); }
+
+static int compFGroup(const void *arg1, const void *arg2) {
+ int val1 = ((SFileGroup *)arg1)->fileId;
+ int val2 = ((SFileGroup *)arg2)->fileId;
+
+ if (val1 < val2) {
+ return -1;
+ } else if (val1 > val2) {
+ return 1;
+ } else {
+ return 0;
+ }
}
-SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) {
- if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId)
- return NULL;
- void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey);
- if (ptr == NULL) return NULL;
- return (SFileGroup *)ptr;
-}
\ No newline at end of file
+static int keyFGroupCompFunc(const void *key, const void *fgroup) {
+ int fid = *(int *)key;
+ SFileGroup *pFGroup = (SFileGroup *)fgroup;
+ if (fid == pFGroup->fileId) {
+ return 0;
+ } else {
+ return fid > pFGroup->fileId ? 1 : -1;
+ }
+}
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index 5526ad0d6e9bb8071a823fa3aac13b5d8803964c..ac334e8ed1be576057ee11b8ae8bb65b5080759e 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -1,320 +1,284 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "tsdbMain.h"
#include "os.h"
-#include "taosdef.h"
-#include "tulog.h"
#include "talgo.h"
-#include "tsdb.h"
-#include "tsdbMain.h"
-#include "tscompression.h"
+#include "taosdef.h"
#include "tchecksum.h"
+#include "tscompression.h"
+#include "tsdb.h"
#include "ttime.h"
-#include
+#include "tulog.h"
-#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
-#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
-#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
-#define TSDB_MIN_ID 0
-#define TSDB_MAX_ID INT_MAX
+#include
+#include
#define TSDB_CFG_FILE_NAME "config"
#define TSDB_DATA_DIR_NAME "data"
-#define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7
-#define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M
-
-enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING };
-
-static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
-static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
-static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
-// static int tsdbOpenMetaFile(char *tsdbDir);
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows);
-static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
-static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
-static void * tsdbCommitData(void *arg);
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper,
- SDataCols *pDataCols);
-static TSKEY tsdbNextIterKey(SSkipListIterator *pIter);
-static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
-static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
-static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
-static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
-static int32_t tsdbSaveConfig(STsdbRepo *pRepo);
-
-#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
-#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
-#define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE)
-#define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED)
-
-/**
- * Set the default TSDB configuration
- */
-void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
- if (pCfg == NULL) return;
-
- pCfg->precision = -1;
- pCfg->tsdbId = 0;
- pCfg->maxTables = -1;
- pCfg->daysPerFile = -1;
- pCfg->minRowsPerFileBlock = -1;
- pCfg->maxRowsPerFileBlock = -1;
- pCfg->keep = -1;
- pCfg->compression = TWO_STAGE_COMP;
-}
-
-/**
- * Create a configuration for TSDB default
- * @return a pointer to a configuration. the configuration object
- * must call tsdbFreeCfg to free memory after usage
- */
-STsdbCfg *tsdbCreateDefaultCfg() {
- STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg));
- if (pCfg == NULL) return NULL;
+#define TSDB_META_FILE_NAME "meta"
+#define TSDB_META_FILE_INDEX 10000000
+#define IS_VALID_PRECISION(precision) \
+ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
+#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
+#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
- tsdbSetDefaultCfg(pCfg);
+typedef struct {
+ int32_t totalLen;
+ int32_t len;
+ SDataRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+ int32_t totalLen;
+ int32_t len;
+ SSubmitBlk *pBlock;
+} SSubmitMsgIter;
+
+static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
+static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg);
+static int32_t tsdbUnsetRepoEnv(char *rootDir);
+static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg);
+static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg);
+static char * tsdbGetCfgFname(char *rootDir);
+static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg);
+static void tsdbFreeRepo(STsdbRepo *pRepo);
+static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows);
+static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
+static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static int tsdbRestoreInfo(STsdbRepo *pRepo);
+static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
+static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
+static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
+static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
+static int keyFGroupCompFunc(const void *key, const void *fgroup);
+static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
+static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
+
+// Function declaration
+int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) {
+ if (mkdir(rootDir, 0755) < 0) {
+ tsdbError("vgId:%d failed to create rootDir %s since %s", pCfg->tsdbId, rootDir, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
- return pCfg;
-}
+ if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
-void tsdbFreeCfg(STsdbCfg *pCfg) {
- if (pCfg != NULL) free(pCfg);
-}
+ if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1;
-STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo) {
- assert(repo != NULL);
- return &((STsdbRepo*)repo)->config;
+ tsdbTrace(
+ "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep "
+ "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d",
+ pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep,
+ pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression);
+ return 0;
}
-/**
- * Create a new TSDB repository
- * @param rootDir the TSDB repository root directory
- * @param pCfg the TSDB repository configuration, upper layer need to free the pointer
- * @param limiter the limitation tracker will implement in the future, make it void now
- *
- * @return a TSDB repository handle on success, NULL for failure
- */
-int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) {
- if (mkdir(rootDir, 0755) != 0) {
- tsdbError("vgId:%d, failed to create rootDir! rootDir:%s, reason:%s", pCfg->tsdbId, rootDir, strerror(errno));
- if (errno == EACCES) {
- return TSDB_CODE_TDB_NO_DISK_PERMISSIONS;
- } else if (errno == ENOSPC) {
- return TSDB_CODE_TDB_NO_DISKSPACE;
- } else if (errno == EEXIST) {
- } else {
- return TSDB_CODE_TDB_INIT_FAILED;
- }
- }
+int32_t tsdbDropRepo(char *rootDir) { return tsdbUnsetRepoEnv(rootDir); }
- if (access(rootDir, F_OK | R_OK | W_OK) == -1) return -1;
+TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) {
+ STsdbCfg config = {0};
+ STsdbRepo *pRepo = NULL;
- if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) {
- return -1;
+ if (tsdbLoadConfig(rootDir, &config) < 0) {
+ tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno));
+ return NULL;
}
- STsdbRepo *pRepo = (STsdbRepo *)malloc(sizeof(STsdbRepo));
+ pRepo = tsdbNewRepo(rootDir, pAppH, &config);
if (pRepo == NULL) {
- return -1;
+ tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno));
+ return NULL;
}
- pRepo->rootDir = strdup(rootDir);
- pRepo->config = *pCfg;
- pRepo->limiter = limiter;
+ if (tsdbOpenMeta(pRepo) < 0) {
+ tsdbError("vgId:%d failed to open meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
- // Create the environment files and directories
- int32_t code = tsdbSetRepoEnv(pRepo);
-
- free(pRepo->rootDir);
- free(pRepo);
- return code;
-}
+ if (tsdbOpenBufPool(pRepo) < 0) {
+ tsdbError("vgId:%d failed to open buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
-/**
- * Close and free all resources taken by the repository
- * @param repo the TSDB repository handle. The interface will free the handle too, so upper
- * layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbDropRepo(TsdbRepoT *repo) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- int id = pRepo->config.tsdbId;
+ if (tsdbOpenFileH(pRepo) < 0) {
+ tsdbError("vgId:%d failed to open file handle since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
- pRepo->state = TSDB_REPO_STATE_CLOSED;
+ if (tsdbRestoreInfo(pRepo) < 0) {
+ tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
- // Free the metaHandle
- tsdbFreeMeta(pRepo->tsdbMeta);
+ // pRepo->state = TSDB_REPO_STATE_ACTIVE;
- // Free the cache
- tsdbFreeCache(pRepo->tsdbCache);
+ tsdbTrace("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo));
- // Destroy the repository info
- tsdbDestroyRepoEnv(pRepo);
+ return (TSDB_REPO_T *)pRepo;
- free(pRepo->rootDir);
- free(pRepo);
+_err:
+ tsdbCloseRepo(pRepo, false);
+ return NULL;
+}
- tsdbTrace("vgId:%d, tsdb repository is dropped!", id);
+// Note: all working thread and query thread must stopped when calling this function
+void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
+ if (repo == NULL) return;
- return 0;
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ int vgId = REPO_ID(pRepo);
+
+ if (toCommit) {
+ tsdbAsyncCommit(pRepo);
+ if (pRepo->commit) pthread_join(pRepo->commitThread, NULL);
+ }
+ tsdbUnRefMemTable(pRepo, pRepo->mem);
+ tsdbUnRefMemTable(pRepo, pRepo->imem);
+ pRepo->mem = NULL;
+ pRepo->imem = NULL;
+
+ tsdbCloseFileH(pRepo);
+ tsdbCloseBufPool(pRepo);
+ tsdbCloseMeta(pRepo);
+ tsdbFreeRepo(pRepo);
+ tsdbTrace("vgId:%d repository is closed", vgId);
}
-static int tsdbRestoreInfo(STsdbRepo *pRepo) {
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- STsdbFileH *pFileH = pRepo->tsdbFileH;
- SFileGroup *pFGroup = NULL;
+int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+ STsdbRepo * pRepo = (STsdbRepo *)repo;
+ SSubmitMsgIter msgIter = {0};
- SFileGroupIter iter;
- SRWHelper rhelper = {{0}};
+ if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) {
+ tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
+ return -1;
+ }
- if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
- tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
- while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
- if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
- for (int i = 1; i < pRepo->config.maxTables; i++) {
- STable * pTable = pMeta->tables[i];
- if (pTable == NULL) continue;
- SCompIdx *pIdx = &rhelper.pCompIdx[i];
+ SSubmitBlk *pBlock = NULL;
+ int32_t affectedrows = 0;
- if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey;
+ TSKEY now = taosGetTimestamp(pRepo->config.precision);
+
+ while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
+ if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) {
+ return -1;
}
}
-
- tsdbDestroyHelper(&rhelper);
+ pRsp->affectedRows = htonl(affectedrows);
return 0;
-
-_err:
- tsdbDestroyHelper(&rhelper);
- return -1;
}
-/**
- * Open an existing TSDB storage repository
- * @param tsdbDir the existing TSDB root directory
- *
- * @return a TSDB repository handle on success, NULL for failure and the error number is set
- */
-TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) {
- char dataDir[128] = "\0";
- if (access(rootDir, F_OK | W_OK | R_OK) < 0) {
- return NULL;
- }
+uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ // STsdbMeta *pMeta = pRepo->tsdbMeta;
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+ uint32_t magic = 0;
+ char * fname = NULL;
- STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo));
- if (pRepo == NULL) {
- return NULL;
- }
+ struct stat fState;
- pRepo->rootDir = strdup(rootDir);
+ tsdbTrace("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex);
+ ASSERT(*index <= eindex);
- tsdbRestoreCfg(pRepo, &(pRepo->config));
- if (pAppH) pRepo->appH = *pAppH;
+ char *sdup = strdup(pRepo->rootDir);
+ char *prefix = dirname(sdup);
- pRepo->tsdbMeta = tsdbInitMeta(rootDir, pRepo->config.maxTables, pRepo);
- if (pRepo->tsdbMeta == NULL) {
- free(pRepo->rootDir);
- free(pRepo);
- return NULL;
- }
+ if (name[0] == 0) { // get the file from index or after, but not larger than eindex
+ int fid = (*index) / 3;
- pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TsdbRepoT *)pRepo);
- if (pRepo->tsdbCache == NULL) {
- tsdbFreeMeta(pRepo->tsdbMeta);
- free(pRepo->rootDir);
- free(pRepo);
- return NULL;
- }
+ if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
+ if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
+ fname = tsdbGetMetaFileName(pRepo->rootDir);
+ *index = TSDB_META_FILE_INDEX;
+ } else {
+ tfree(sdup);
+ return 0;
+ }
+ } else {
+ SFileGroup *pFGroup =
+ taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
+ if (pFGroup->fileId == fid) {
+ strcpy(fname, pFGroup->files[(*index) % 3].fname);
+ } else {
+ if (pFGroup->fileId * 3 + 2 < eindex) {
+ strcpy(fname, pFGroup->files[0].fname);
+ *index = pFGroup->fileId * 3;
+ } else {
+ tfree(sdup);
+ return 0;
+ }
+ }
+ }
+ strcpy(name, fname + strlen(prefix));
+ } else { // get the named file at the specified index. If not there, return 0
+ if (*index == TSDB_META_FILE_INDEX) { // get meta file
+ fname = tsdbGetMetaFileName(pRepo->rootDir);
+ } else {
+ int fid = (*index) / 3;
+ SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
+ if (pFGroup == NULL) { // not found
+ tfree(sdup);
+ return 0;
+ }
- tsdbGetDataDirName(pRepo, dataDir);
- pRepo->tsdbFileH = tsdbInitFileH(dataDir, &(pRepo->config));
- if (pRepo->tsdbFileH == NULL) {
- tsdbFreeCache(pRepo->tsdbCache);
- tsdbFreeMeta(pRepo->tsdbMeta);
- free(pRepo->rootDir);
- free(pRepo);
- return NULL;
+ SFile *pFile = &pFGroup->files[(*index) % 3];
+ fname = strdup(pFile->fname);
+ }
}
- // Restore key from file
- if (tsdbRestoreInfo(pRepo) < 0) {
- tsdbFreeCache(pRepo->tsdbCache);
- tsdbFreeMeta(pRepo->tsdbMeta);
- tsdbCloseFileH(pRepo->tsdbFileH);
- free(pRepo->rootDir);
- free(pRepo);
- return NULL;
+ if (stat(fname, &fState) < 0) {
+ tfree(sdup);
+ tfree(fname);
+ return 0;
}
- pRepo->state = TSDB_REPO_STATE_ACTIVE;
+ tfree(sdup);
+ *size = fState.st_size;
+ magic = *size;
- tsdbTrace("vgId:%d, open tsdb repository successfully!", pRepo->config.tsdbId);
- return (TsdbRepoT *)pRepo;
+ tfree(fname);
+ return magic;
}
-// static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
-// // TODO
-// return 0;
-// }
-
-/**
- * Close a TSDB repository. Only free memory resources, and keep the files.
- * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
- * layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit) {
+void tsdbStartStream(TSDB_REPO_T *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
- if (pRepo == NULL) return 0;
- int id = pRepo->config.tsdbId;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
- pRepo->state = TSDB_REPO_STATE_CLOSED;
- tsdbLockRepo(repo);
- if (pRepo->commit) {
- tsdbUnLockRepo(repo);
- return -1;
- }
- pRepo->commit = 1;
- // Loop to move pData to iData
- for (int i = 1; i < pRepo->config.maxTables; i++) {
- STable *pTable = pRepo->tsdbMeta->tables[i];
- if (pTable != NULL && pTable->mem != NULL) {
- pTable->imem = pTable->mem;
- pTable->mem = NULL;
+ for (int i = 0; i < pRepo->config.maxTables; i++) {
+ STable *pTable = pMeta->tables[i];
+ if (pTable && pTable->type == TSDB_STREAM_TABLE) {
+ pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql,
+ tsdbGetTableSchema(pTable));
}
}
- // TODO: Loop to move mem to imem
- pRepo->tsdbCache->imem = pRepo->tsdbCache->mem;
- pRepo->tsdbCache->mem = NULL;
- pRepo->tsdbCache->curBlock = NULL;
- tsdbUnLockRepo(repo);
-
- if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
- if (toCommit) tsdbCommitData((void *)repo);
-
- tsdbCloseFileH(pRepo->tsdbFileH);
-
- tsdbFreeMeta(pRepo->tsdbMeta);
-
- tsdbFreeCache(pRepo->tsdbCache);
-
- tfree(pRepo->rootDir);
- tfree(pRepo);
-
- tsdbTrace("vgId:%d, repository is closed!", id);
+}
- return 0;
+STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
+ ASSERT(repo != NULL);
+ return &((STsdbRepo *)repo)->config;
}
-/**
- * Change the configuration of a repository
- * @param pCfg the repository configuration, the upper layer should free the pointer
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
+int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
+ // TODO: think about multithread cases
STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbCfg * pRCfg = &pRepo->config;
- if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG;
+ if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
ASSERT(pRCfg->tsdbId == pCfg->tsdbId);
ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize);
@@ -325,422 +289,138 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
bool configChanged = false;
if (pRCfg->compression != pCfg->compression) {
- configChanged = true;
tsdbAlterCompression(pRepo, pCfg->compression);
+ configChanged = true;
}
if (pRCfg->keep != pCfg->keep) {
+ if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) {
+ tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno));
+ return -1;
+ }
configChanged = true;
- tsdbAlterKeep(pRepo, pCfg->keep);
}
if (pRCfg->totalBlocks != pCfg->totalBlocks) {
- configChanged = true;
tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+ configChanged = true;
}
if (pRCfg->maxTables != pCfg->maxTables) {
+ if (tsdbAlterMaxTables(pRepo, pCfg->maxTables) < 0) {
+ tsdbError("vgId:%d failed to configure repo when alter maxTables since %s", REPO_ID(pRepo), tstrerror(terrno));
+ return -1;
+ }
configChanged = true;
- tsdbAlterMaxTables(pRepo, pCfg->maxTables);
}
- if (configChanged) tsdbSaveConfig(pRepo);
-
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
-
- if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
-
- tsdbLockRepo(repo);
- if (pRepo->commit) {
- tsdbUnLockRepo(repo);
- return -1;
- }
- pRepo->commit = 1;
- // Loop to move pData to iData
- for (int i = 1; i < pRepo->config.maxTables; i++) {
- STable *pTable = pRepo->tsdbMeta->tables[i];
- if (pTable != NULL && pTable->mem != NULL) {
- pTable->imem = pTable->mem;
- pTable->mem = NULL;
+ if (configChanged) {
+ if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) {
+ tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno));
+ return -1;
}
}
- // TODO: Loop to move mem to imem
- pRepo->tsdbCache->imem = pRepo->tsdbCache->mem;
- pRepo->tsdbCache->mem = NULL;
- pRepo->tsdbCache->curBlock = NULL;
- tsdbUnLockRepo(repo);
-
- // TODO: here should set as detached or use join for memory leak
- pthread_attr_t thattr;
- pthread_attr_init(&thattr);
- pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
- pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo);
- tsdbTrace("vgId:%d, start to commit!", pRepo->config.tsdbId);
return 0;
}
-int32_t tsdbLockRepo(TsdbRepoT *repo) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- return pthread_mutex_lock(&(pRepo->mutex));
+void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) {
+ ASSERT(repo != NULL);
+ STsdbRepo *pRepo = repo;
+ *totalPoints = pRepo->stat.pointsWritten;
+ *totalStorage = pRepo->stat.totalStorage;
+ *compStorage = pRepo->stat.compStorage;
}
-int32_t tsdbUnLockRepo(TsdbRepoT *repo) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- return pthread_mutex_unlock(&(pRepo->mutex));
+// ----------------- INTERNAL FUNCTIONS -----------------
+char *tsdbGetMetaFileName(char *rootDir) {
+ int tlen = strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2;
+ char *fname = calloc(1, tlen);
+ if (fname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ snprintf(fname, tlen, "%s/%s", rootDir, TSDB_META_FILE_NAME);
+ return fname;
}
-/**
- * Get the TSDB repository information, including some statistics
- * @param pRepo the TSDB repository handle
- * @param error the error number to set when failure occurs
- *
- * @return a info struct handle on success, NULL for failure and the error number is set. The upper
- * layers should free the info handle themselves or memory leak will occur
- */
-STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) {
- // TODO
- return NULL;
+void tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type, char *fname) {
+ snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]);
}
-int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
- // TODO
+int tsdbLockRepo(STsdbRepo *pRepo) {
+ int code = pthread_mutex_lock(&pRepo->mutex);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ pRepo->repoLocked = true;
return 0;
}
-int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- STsdbMeta *pMeta = pRepo->tsdbMeta;
- int16_t tversion = htons(pMsg->tversion);
-
- STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid));
- if (pTable == NULL) return TSDB_CODE_TDB_INVALID_TABLE_ID;
- if (pTable->tableId.tid != htonl(pMsg->tid)) return TSDB_CODE_TDB_INVALID_TABLE_ID;
-
- if (pTable->type != TSDB_CHILD_TABLE) {
- tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", pRepo->config.tsdbId,
- varDataVal(pTable->name), pTable->type);
- return TSDB_CODE_TDB_INVALID_TABLE_TYPE;
+int tsdbUnlockRepo(STsdbRepo *pRepo) {
+ ASSERT(IS_REPO_LOCKED(pRepo));
+ pRepo->repoLocked = false;
+ int code = pthread_mutex_unlock(&pRepo->mutex);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
}
+ return 0;
+}
- if (schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)) < tversion) {
- tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", pRepo->config.tsdbId,
- schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)), tversion);
- void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid));
- if (msg == NULL) {
- return terrno;
- }
- // Deal with error her
- STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg);
- STable *super = tsdbGetTableByUid(pMeta, pTableCfg->superUid);
- ASSERT(super != NULL);
-
- int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- tsdbClearTableCfg(pTableCfg);
- rpcFreeCont(msg);
+char *tsdbGetDataDirName(char *rootDir) {
+ int tlen = strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2;
+ char *fname = calloc(1, tlen);
+ if (fname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
}
- STSchema *pTagSchema = tsdbGetTableTagSchema(pMeta, pTable);
-
- if (schemaVersion(pTagSchema) > tversion) {
- tsdbError(
- "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag "
- "version:%d",
- pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema));
- return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE;
- }
- if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) {
- tsdbRemoveTableFromIndex(pMeta, pTable);
- }
- // TODO: remove table from index if it is the first column of tag
- tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data);
- if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) {
- tsdbAddTableIntoIndex(pMeta, pTable);
- }
- return TSDB_CODE_SUCCESS;
+ snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME);
+ return fname;
}
-TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
-
- STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
- if (pTable == NULL) return -1;
-
- return TSDB_GET_TABLE_LAST_KEY(pTable);
-}
+STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; }
+STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; }
+STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; }
-void tsdbStartStream(TsdbRepoT *repo) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- STsdbMeta *pMeta = pRepo->tsdbMeta;
+// ----------------- LOCAL FUNCTIONS -----------------
+static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
+ // Check precision
+ if (pCfg->precision == -1) {
+ pCfg->precision = TSDB_DEFAULT_PRECISION;
+ } else {
+ if (!IS_VALID_PRECISION(pCfg->precision)) {
+ tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision);
+ goto _err;
+ }
+ }
- for (int i = 0; i < pRepo->config.maxTables; i++) {
- STable *pTable = pMeta->tables[i];
- if (pTable && pTable->type == TSDB_STREAM_TABLE) {
- pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable));
- }
- }
-}
-
-STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
- // TODO
- return NULL;
-}
-
-// TODO: need to return the number of data inserted
-int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) {
- SSubmitMsgIter msgIter;
- STsdbRepo *pRepo = (STsdbRepo *)repo;
-
- tsdbInitSubmitMsgIter(pMsg, &msgIter);
- SSubmitBlk *pBlock = NULL;
- int32_t code = TSDB_CODE_SUCCESS;
- int32_t affectedrows = 0;
-
- TSKEY now = taosGetTimestamp(pRepo->config.precision);
-
- while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
- if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) {
- return code;
- }
- }
- pRsp->affectedRows = htonl(affectedrows);
- return code;
-}
-
-/**
- * Initialize a table configuration
- */
-int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) {
- if (config == NULL) return -1;
- if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) return -1;
-
- memset((void *)config, 0, sizeof(STableCfg));
-
- config->type = type;
- config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
- config->tableId.uid = uid;
- config->tableId.tid = tid;
- config->name = NULL;
- config->sql = NULL;
- return 0;
-}
-
-/**
- * Set the super table UID of the created table
- */
-int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) {
- if (config->type != TSDB_CHILD_TABLE) return -1;
- if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1;
-
- config->superUid = uid;
- return 0;
-}
-
-/**
- * Set the table schema in the configuration
- * @param config the configuration to set
- * @param pSchema the schema to set
- * @param dup use the schema directly or duplicate one for use
- *
- * @return 0 for success and -1 for failure
- */
-int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) {
- if (dup) {
- config->schema = tdDupSchema(pSchema);
- } else {
- config->schema = pSchema;
- }
- return 0;
-}
-
-/**
- * Set the table schema in the configuration
- * @param config the configuration to set
- * @param pSchema the schema to set
- * @param dup use the schema directly or duplicate one for use
- *
- * @return 0 for success and -1 for failure
- */
-int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) {
- if (config->type != TSDB_CHILD_TABLE) return -1;
-
- if (dup) {
- config->tagSchema = tdDupSchema(pSchema);
- } else {
- config->tagSchema = pSchema;
- }
- return 0;
-}
-
-int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) {
- if (config->type != TSDB_CHILD_TABLE) return -1;
-
- if (dup) {
- config->tagValues = tdKVRowDup(row);
- } else {
- config->tagValues = row;
- }
-
- return 0;
-}
-
-int tsdbTableSetName(STableCfg *config, char *name, bool dup) {
- if (dup) {
- config->name = strdup(name);
- if (config->name == NULL) return -1;
- } else {
- config->name = name;
- }
-
- return 0;
-}
-
-int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) {
- if (config->type != TSDB_CHILD_TABLE) return -1;
-
- if (dup) {
- config->sname = strdup(sname);
- if (config->sname == NULL) return -1;
- } else {
- config->sname = sname;
- }
- return 0;
-}
-
-int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) {
- if (config->type != TSDB_STREAM_TABLE) return -1;
-
- if (dup) {
- config->sql = strdup(sql);
- if (config->sql == NULL) return -1;
- } else {
- config->sql = sql;
- }
-
- return 0;
-}
-
-void tsdbClearTableCfg(STableCfg *config) {
- if (config) {
- if (config->schema) tdFreeSchema(config->schema);
- if (config->tagSchema) tdFreeSchema(config->tagSchema);
- if (config->tagValues) kvRowFree(config->tagValues);
- tfree(config->name);
- tfree(config->sname);
- tfree(config->sql);
- free(config);
- }
-}
-
-int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
- if (pBlock->len <= 0) return -1;
- pIter->totalLen = pBlock->len;
- pIter->len = 0;
- pIter->row = (SDataRow)(pBlock->data);
- return 0;
-}
-
-SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
- SDataRow row = pIter->row;
- if (row == NULL) return NULL;
-
- pIter->len += dataRowLen(row);
- if (pIter->len >= pIter->totalLen) {
- pIter->row = NULL;
- } else {
- pIter->row = (char *)row + dataRowLen(row);
- }
-
- return row;
-}
-
-int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
- if (pMsg == NULL || pIter == NULL) return -1;
-
- pMsg->length = htonl(pMsg->length);
- pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
- pMsg->compressed = htonl(pMsg->compressed);
-
- pIter->totalLen = pMsg->length;
- pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE;
- if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
- pIter->pBlock = NULL;
- } else {
- pIter->pBlock = pMsg->blocks;
- }
-
- return 0;
-}
-
-SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
- SSubmitBlk *pBlock = pIter->pBlock;
- if (pBlock == NULL) return NULL;
-
- pBlock->len = htonl(pBlock->len);
- pBlock->numOfRows = htons(pBlock->numOfRows);
- pBlock->uid = htobe64(pBlock->uid);
- pBlock->tid = htonl(pBlock->tid);
-
- pBlock->sversion = htonl(pBlock->sversion);
- pBlock->padding = htonl(pBlock->padding);
-
- pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len;
- if (pIter->len >= pIter->totalLen) {
- pIter->pBlock = NULL;
- } else {
- pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->len + sizeof(SSubmitBlk));
- }
-
- return pBlock;
-}
-
-STsdbMeta* tsdbGetMeta(TsdbRepoT* pRepo) {
- STsdbRepo *tsdb = (STsdbRepo *)pRepo;
- return tsdb->tsdbMeta;
-}
-
-STsdbFileH* tsdbGetFile(TsdbRepoT* pRepo) {
- STsdbRepo* tsdb = (STsdbRepo*) pRepo;
- return tsdb->tsdbFileH;
-}
-
-// Check the configuration and set default options
-static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
- // Check precision
- if (pCfg->precision == -1) {
- pCfg->precision = TSDB_DEFAULT_PRECISION;
- } else {
- if (!IS_VALID_PRECISION(pCfg->precision)) {
- tsdbError("vgId:%d, invalid precision configuration! precision:%d", pCfg->tsdbId, pCfg->precision);
- return -1;
- }
- }
-
- // Check compression
- if (pCfg->compression == -1) {
- pCfg->compression = TSDB_DEFAULT_COMPRESSION;
- } else {
- if (!IS_VALID_COMPRESSION(pCfg->compression)) {
- tsdbError("vgId:%d: invalid compression configuration! compression:%d", pCfg->tsdbId, pCfg->precision);
- return -1;
+ // Check compression
+ if (pCfg->compression == -1) {
+ pCfg->compression = TSDB_DEFAULT_COMPRESSION;
+ } else {
+ if (!IS_VALID_COMPRESSION(pCfg->compression)) {
+ tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision);
+ goto _err;
}
}
// Check tsdbId
- if (pCfg->tsdbId < 0) return -1;
+ if (pCfg->tsdbId < 0) {
+ tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId);
+ goto _err;
+ }
// Check maxTables
if (pCfg->maxTables == -1) {
pCfg->maxTables = TSDB_DEFAULT_TABLES;
} else {
if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) {
- tsdbError("vgId:%d: invalid maxTables configuration! maxTables:%d TSDB_MIN_TABLES:%d TSDB_MAX_TABLES:%d",
+ tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d",
pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES);
- return -1;
+ goto _err;
}
}
@@ -750,10 +430,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
} else {
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
tsdbError(
- "vgId:%d, invalid daysPerFile configuration! daysPerFile:%d TSDB_MIN_DAYS_PER_FILE:%d TSDB_MAX_DAYS_PER_FILE:"
+ "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE "
"%d",
pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
- return -1;
+ goto _err;
}
}
@@ -763,10 +443,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
} else {
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
tsdbError(
- "vgId:%d, invalid minRowsPerFileBlock configuration! minRowsPerFileBlock:%d TSDB_MIN_MIN_ROW_FBLOCK:%d "
- "TSDB_MAX_MIN_ROW_FBLOCK:%d",
+ "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d "
+ "TSDB_MAX_MIN_ROW_FBLOCK %d",
pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
- return -1;
+ goto _err;
}
}
@@ -775,14 +455,18 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
} else {
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
tsdbError(
- "vgId:%d, invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock:%d TSDB_MIN_MAX_ROW_FBLOCK:%d "
- "TSDB_MAX_MAX_ROW_FBLOCK:%d",
+ "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d "
+ "TSDB_MAX_MAX_ROW_FBLOCK %d",
pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
- return -1;
+ goto _err;
}
}
- if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) return -1;
+ if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
+ tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId,
+ pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock);
+ goto _err;
+ }
// Check keep
if (pCfg->keep == -1) {
@@ -790,216 +474,269 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
} else {
if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) {
tsdbError(
- "vgId:%d, invalid keep configuration! keep:%d TSDB_MIN_KEEP:%d "
- "TSDB_MAX_KEEP:%d",
+ "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d "
+ "TSDB_MAX_KEEP %d",
pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
- return -1;
+ goto _err;
}
}
return 0;
-}
-static int32_t tsdbGetCfgFname(STsdbRepo *pRepo, char *fname) {
- if (pRepo == NULL) return -1;
- sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_CFG_FILE_NAME);
- return 0;
+_err:
+ terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+ return -1;
}
-static int32_t tsdbSaveConfig(STsdbRepo *pRepo) {
- char fname[128] = "\0"; // TODO: get rid of the literal 128
+static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) {
+ if (tsdbSaveConfig(rootDir, pCfg) < 0) {
+ tsdbError("vgId:%d failed to set TSDB environment since %s", pCfg->tsdbId, tstrerror(terrno));
+ return -1;
+ }
- if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+ char *dirName = tsdbGetDataDirName(rootDir);
+ if (dirName == NULL) return -1;
- int fd = open(fname, O_WRONLY | O_CREAT, 0755);
- if (fd < 0) {
+ if (mkdir(dirName, 0755) < 0) {
+ tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ free(dirName);
return -1;
}
- if (write(fd, (void *)(&(pRepo->config)), sizeof(STsdbCfg)) < 0) {
+ free(dirName);
+
+ char *fname = tsdbGetMetaFileName(rootDir);
+ if (fname == NULL) return -1;
+ if (tdCreateKVStore(fname) < 0) {
+ tsdbError("vgId:%d failed to open KV store since %s", pCfg->tsdbId, tstrerror(terrno));
+ free(fname);
return -1;
}
- close(fd);
+ free(fname);
return 0;
}
-static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) {
- char fname[128] = "\0";
+static int32_t tsdbUnsetRepoEnv(char *rootDir) {
+ taosRemoveDir(rootDir);
+ tsdbTrace("repository %s is removed", rootDir);
+ return 0;
+}
- if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
+ int fd = -1;
+ char *fname = NULL;
+ char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+ char *pBuf = buf;
- int fd = open(fname, O_RDONLY);
+ fname = tsdbGetCfgFname(rootDir);
+ if (fname == NULL) {
+ tsdbError("vgId:%d failed to save configuration since %s", pCfg->tsdbId, tstrerror(terrno));
+ goto _err;
+ }
+
+ fd = open(fname, O_WRONLY | O_CREAT, 0755);
if (fd < 0) {
- return -1;
+ tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
- if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) {
- close(fd);
- return -1;
+ int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg);
+ ASSERT((tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE) && (POINTER_DISTANCE(pBuf, buf) == tlen));
+
+ taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+
+ if (twrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
+ strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
- close(fd);
+ if (fsync(fd) < 0) {
+ tsdbError("vgId:%d failed to fsync file %s since %s", pCfg->tsdbId, fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ free(fname);
+ close(fd);
return 0;
-}
-static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname) {
- if (pRepo == NULL || pRepo->rootDir == NULL) return -1;
- sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_DATA_DIR_NAME);
- return 0;
+_err:
+ tfree(fname);
+ if (fd >= 0) close(fd);
+ return -1;
}
-static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
- STsdbCfg *pCfg = &pRepo->config;
- if (tsdbSaveConfig(pRepo) < 0) return -1;
+static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
+ char *fname = NULL;
+ int fd = -1;
+ char buf[TSDB_FILE_HEAD_SIZE] = "\0";
- char dirName[128] = "\0";
- if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
+ fname = tsdbGetCfgFname(rootDir);
+ if (fname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
- if (mkdir(dirName, 0755) < 0) {
- tsdbError("vgId:%d, failed to create repository directory! reason:%s", pRepo->config.tsdbId, strerror(errno));
- return -1;
+ fd = open(fname, O_RDONLY);
+ if (fd < 0) {
+ tsdbError("failed to open file %s since %s", fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
- tsdbTrace(
- "vgId:%d, set up tsdb environment succeed! cacheBlockSize:%d, totalBlocks:%d, maxTables:%d, daysPerFile:%d, keep:"
- "%d, minRowsPerFileBlock:%d, maxRowsPerFileBlock:%d, precision:%d, compression:%d",
- pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep,
- pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression);
+ if (tread(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+ tsdbError("file %s is corrupted", fname);
+ terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+ goto _err;
+ }
+
+ tsdbDecodeCfg(buf, pCfg);
+
+ tfree(fname);
+ close(fd);
+
return 0;
+
+_err:
+ tfree(fname);
+ if (fd >= 0) close(fd);
+ return -1;
}
-static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) {
- char fname[260];
- if (pRepo == NULL) return 0;
- char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2);
- if (dirName == NULL) {
- return -1;
+static char *tsdbGetCfgFname(char *rootDir) {
+ int tlen = strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2;
+ char *fname = calloc(1, tlen);
+ if (fname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
}
- sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb");
+ snprintf(fname, tlen, "%s/%s", rootDir, TSDB_CFG_FILE_NAME);
+ return fname;
+}
- DIR *dir = opendir(dirName);
- if (dir == NULL) return -1;
+static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
+ STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo));
+ if (pRepo == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
- struct dirent *dp;
- while ((dp = readdir(dir)) != NULL) {
- if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) continue;
- sprintf(fname, "%s/%s", pRepo->rootDir, dp->d_name);
- remove(fname);
+ int code = pthread_mutex_init(&pRepo->mutex, NULL);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
}
- closedir(dir);
+ pRepo->repoLocked = false;
- rmdir(dirName);
+ pRepo->rootDir = strdup(rootDir);
+ if (pRepo->rootDir == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
- return 0;
-}
+ pRepo->config = *pCfg;
+ pRepo->appH = *pAppH;
-// static int tsdbOpenMetaFile(char *tsdbDir) {
-// // TODO
-// return 0;
-// }
+ pRepo->tsdbMeta = tsdbNewMeta(pCfg);
+ if (pRepo->tsdbMeta == NULL) {
+ tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
-static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
- // TODO
- int32_t level = 0;
- int32_t headSize = 0;
+ pRepo->pPool = tsdbNewBufPool(pCfg);
+ if (pRepo->pPool == NULL) {
+ tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
- if (pTable->mem == NULL) {
- pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
- if (pTable->mem == NULL) return -1;
- pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
- pTable->mem->keyFirst = INT64_MAX;
- pTable->mem->keyLast = 0;
+ pRepo->tsdbFileH = tsdbNewFileH(pCfg);
+ if (pRepo->tsdbFileH == NULL) {
+ tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
}
- tSkipListNewNodeInfo(pTable->mem->pData, &level, &headSize);
+ return pRepo;
- TSKEY key = dataRowKey(row);
- // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfRows);
-
- // Copy row into the memory
- SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key);
- if (pNode == NULL) {
- // TODO: deal with allocate failure
- }
-
- pNode->level = level;
- dataRowCpy(SL_GET_NODE_DATA(pNode), row);
-
- // Insert the skiplist node into the data
- if (pTable->mem == NULL) {
- pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
- if (pTable->mem == NULL) return -1;
- pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
- pTable->mem->keyFirst = INT64_MAX;
- pTable->mem->keyLast = 0;
- }
- tSkipListPut(pTable->mem->pData, pNode);
- if (key > pTable->mem->keyLast) pTable->mem->keyLast = key;
- if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key;
- if (key > pTable->lastKey) pTable->lastKey = key;
-
- pTable->mem->numOfRows = tSkipListGetSize(pTable->mem->pData);
+_err:
+ tsdbFreeRepo(pRepo);
+ return NULL;
+}
+
+static void tsdbFreeRepo(STsdbRepo *pRepo) {
+ if (pRepo) {
+ tsdbFreeFileH(pRepo->tsdbFileH);
+ tsdbFreeBufPool(pRepo->pPool);
+ tsdbFreeMeta(pRepo->tsdbMeta);
+ // tsdbFreeMemTable(pRepo->mem);
+ // tsdbFreeMemTable(pRepo->imem);
+ tfree(pRepo->rootDir);
+ pthread_mutex_destroy(&pRepo->mutex);
+ free(pRepo);
+ }
+}
- tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId,
- pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row));
+static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+ if (pMsg == NULL) {
+ terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+ return -1;
+ }
+
+ pMsg->length = htonl(pMsg->length);
+ pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+ pMsg->compressed = htonl(pMsg->compressed);
+
+ pIter->totalLen = pMsg->length;
+ pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE;
+ if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
+ terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+ return -1;
+ } else {
+ pIter->pBlock = pMsg->blocks;
+ }
return 0;
}
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
+static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
int64_t points = 0;
- STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
- STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
- if (pTable == NULL) {
- tsdbError("vgId:%d, failed to get table for insert, uid:" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid,
+ STable *pTable = tsdbGetTableByUid(pMeta, pBlock->uid);
+ if (pTable == NULL || TABLE_TID(pTable) != pBlock->tid) {
+ tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
pBlock->tid);
- return TSDB_CODE_TDB_INVALID_TABLE_ID;
- }
-
- // Check schema version
- int32_t tversion = pBlock->sversion;
- STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable);
- ASSERT(pSchema != NULL);
- int16_t nversion = schemaVersion(pSchema);
- if (tversion > nversion) {
- tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.",
- pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion);
- void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, pTable->tableId.tid);
- if (msg == NULL) {
- return terrno;
- }
- // Deal with error her
- STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg);
- STable *pTableUpdate = NULL;
- if (pTable->type == TSDB_CHILD_TABLE) {
- pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid);
- } else {
- pTableUpdate = pTable;
- }
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
+ }
- int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- tsdbClearTableCfg(pTableCfg);
- rpcFreeCont(msg);
- } else {
- if (tsdbGetTableSchemaByVersion(pMeta, pTable, tversion) == NULL) {
- tsdbError("vgId:%d table:%s tid:%d invalid schema version %d from client", pRepo->config.tsdbId,
- varDataVal(pTable->name), pTable->tableId.tid, tversion);
- return TSDB_CODE_TDB_TABLE_SCHEMA_VERSION;
- }
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
+ }
+
+ // Check schema version and update schema if needed
+ if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
+ tsdbError("vgId:%d failed to insert data to table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ return -1;
}
SSubmitBlkIter blkIter = {0};
- SDataRow row = NULL;
+ SDataRow row = NULL;
TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
@@ -1007,391 +744,317 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
- tsdbError("vgId:%d, table:%s, tid:%d, talbe uid:%ld timestamp is out of range. now:" PRId64 ", maxKey:" PRId64
- ", minKey:" PRId64,
- pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey);
- return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
- }
-
- if (tdInsertRowToTable(pRepo, row, pTable) < 0) {
+ tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
+ " maxKey %" PRId64,
+ REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey);
+ terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
return -1;
}
- (*affectedrows)++;
- points++;
- }
- atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols));
- atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen);
- return TSDB_CODE_SUCCESS;
-}
+ if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
-static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) {
- ASSERT(maxRowsToRead > 0);
- if (pIter == NULL) return 0;
- STSchema *pSchema = NULL;
-
- int numOfRows = 0;
-
- do {
- if (numOfRows >= maxRowsToRead) break;
-
- SSkipListNode *node = tSkipListIterGet(pIter);
- if (node == NULL) break;
-
- SDataRow row = SL_GET_NODE_DATA(node);
- if (dataRowKey(row) > maxKey) break;
-
- if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
- pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row));
- if (pSchema == NULL) {
- // TODO: deal with the error here
- ASSERT(false);
- }
- }
-
- tdAppendDataRowToDataCol(row, pSchema, pCols);
- numOfRows++;
- } while (tSkipListIterNext(pIter));
-
- return numOfRows;
-}
-
-static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) {
- if (iters == NULL) return;
-
- for (int tid = 1; tid < maxTables; tid++) {
- if (iters[tid] == NULL) continue;
- tSkipListDestroyIter(iters[tid]);
- }
-
- free(iters);
-}
-
-static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) {
- SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *));
- if (iters == NULL) return NULL;
-
- for (int tid = 1; tid < maxTables; tid++) {
- STable *pTable = pMeta->tables[tid];
- if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue;
-
- iters[tid] = tSkipListCreateIter(pTable->imem->pData);
- if (iters[tid] == NULL) goto _err;
-
- if (!tSkipListIterNext(iters[tid])) goto _err;
+ (*affectedrows)++;
+ points++;
}
- return iters;
-
- _err:
- tsdbDestroyTableIters(iters, maxTables);
- return NULL;
-}
+ STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
+ pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
+ pRepo->stat.totalStorage += points * schemaVLen(pSchema);
-static void tsdbFreeMemTable(SMemTable *pMemTable) {
- if (pMemTable) {
- tSkipListDestroy(pMemTable->pData);
- free(pMemTable);
- }
+ return 0;
}
-// Commit to file
-static void *tsdbCommitData(void *arg) {
- STsdbRepo * pRepo = (STsdbRepo *)arg;
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- STsdbCache *pCache = pRepo->tsdbCache;
- STsdbCfg * pCfg = &(pRepo->config);
- SDataCols * pDataCols = NULL;
- SRWHelper whelper = {{0}};
- if (pCache->imem == NULL) return NULL;
-
- tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId);
-
- // Create the iterator to read from cache
- SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables);
- if (iters == NULL) {
- ASSERT(0);
- return NULL;
- }
+static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
+ SSubmitBlk *pBlock = pIter->pBlock;
+ if (pBlock == NULL) return NULL;
- if (tsdbInitWriteHelper(&whelper, pRepo) < 0) goto _exit;
- if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit;
+ pBlock->len = htonl(pBlock->len);
+ pBlock->numOfRows = htons(pBlock->numOfRows);
+ pBlock->uid = htobe64(pBlock->uid);
+ pBlock->tid = htonl(pBlock->tid);
- int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision);
- int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision);
+ pBlock->sversion = htonl(pBlock->sversion);
+ pBlock->padding = htonl(pBlock->padding);
- // Loop to commit to each file
- for (int fid = sfid; fid <= efid; fid++) {
- if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
- ASSERT(false);
- goto _exit;
- }
+ pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len;
+ if (pIter->len >= pIter->totalLen) {
+ pIter->pBlock = NULL;
+ } else {
+ pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->len + sizeof(SSubmitBlk));
}
- // Do retention actions
- tsdbFitRetention(pRepo);
- if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
+ return pBlock;
+}
-_exit:
- tdFreeDataCols(pDataCols);
- tsdbDestroyTableIters(iters, pCfg->maxTables);
- tsdbDestroyHelper(&whelper);
+static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+ SDataRow row = pIter->row;
+ if (row == NULL) return NULL;
- tsdbLockRepo(arg);
- tdListMove(pCache->imem->list, pCache->pool.memPool);
- tsdbAdjustCacheBlocks(pCache);
- tdListFree(pCache->imem->list);
- free(pCache->imem);
- pCache->imem = NULL;
- pRepo->commit = 0;
- for (int i = 1; i < pCfg->maxTables; i++) {
- STable *pTable = pMeta->tables[i];
- if (pTable && pTable->imem) {
- tsdbFreeMemTable(pTable->imem);
- pTable->imem = NULL;
- }
+ pIter->len += dataRowLen(row);
+ if (pIter->len >= pIter->totalLen) {
+ pIter->row = NULL;
+ } else {
+ pIter->row = (char *)row + dataRowLen(row);
}
- tsdbUnLockRepo(arg);
- tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId);
- return NULL;
+ return row;
}
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) {
- char dataDir[128] = {0};
+static int tsdbRestoreInfo(STsdbRepo *pRepo) {
+ // TODO
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
- STsdbCfg * pCfg = &pRepo->config;
- SFileGroup *pGroup = NULL;
-
- TSKEY minKey = 0, maxKey = 0;
- tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
-
- // Check if there are data to commit to this file
- int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey);
- if (!hasDataToCommit) return 0; // No data to commit, just return
-
- // Create and open files for commit
- tsdbGetDataDirName(pRepo, dataDir);
- if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) {
- tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid);
- goto _err;
- }
-
- // Open files for write/read
- if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
- tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId);
- goto _err;
- }
-
- // Loop to commit data in each table
- for (int tid = 1; tid < pCfg->maxTables; tid++) {
- STable * pTable = pMeta->tables[tid];
- if (pTable == NULL) continue;
-
- SSkipListIterator *pIter = iters[tid];
-
- // Set the helper and the buffer dataCols object to help to write this table
- tsdbSetHelperTable(pHelper, pTable, pRepo);
- tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable));
-
- // Loop to write the data in the cache to files. If no data to write, just break the loop
- int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5;
- int nLoop = 0;
- while (true) {
- int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols);
- assert(rowsRead >= 0);
- if (pDataCols->numOfRows == 0) break;
- nLoop++;
-
- ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
- ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
-
- int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
- ASSERT(rowsWritten != 0);
- if (rowsWritten < 0) goto _err;
- ASSERT(rowsWritten <= pDataCols->numOfRows);
+ SFileGroup *pFGroup = NULL;
- tdPopDataColsPoints(pDataCols, rowsWritten);
- maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows;
- }
+ SFileGroupIter iter;
+ SRWHelper rhelper = {0};
- ASSERT(pDataCols->numOfRows == 0);
+ if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
- // Move the last block to the new .l file if neccessary
- if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
- tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId);
- goto _err;
- }
+ tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC);
+ while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
+ if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
+ for (int i = 1; i < pRepo->config.maxTables; i++) {
+ STable *pTable = pMeta->tables[i];
+ if (pTable == NULL) continue;
+ SCompIdx *pIdx = &rhelper.pCompIdx[i];
- // Write the SCompBlock part
- if (tsdbWriteCompInfo(pHelper) < 0) {
- tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId);
- goto _err;
+ if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey;
}
}
- if (tsdbWriteCompIdx(pHelper) < 0) {
- tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId);
- goto _err;
- }
-
- tsdbCloseHelperFile(pHelper, 0);
- // TODO: make it atomic with some methods
- pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF;
- pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF;
- pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF;
-
+ tsdbDestroyHelper(&rhelper);
return 0;
- _err:
- ASSERT(false);
- tsdbCloseHelperFile(pHelper, 1);
+_err:
+ tsdbDestroyHelper(&rhelper);
return -1;
}
-/**
- * Return the next iterator key.
- *
- * @return the next key if iter has
- * -1 if iter not
- */
-static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
- if (pIter == NULL) return -1;
-
- SSkipListNode *node = tSkipListIterGet(pIter);
- if (node == NULL) return -1;
-
- SDataRow row = SL_GET_NODE_DATA(node);
- return dataRowKey(row);
-}
-
-static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) {
- TSKEY nextKey;
- for (int i = 0; i < nIters; i++) {
- SSkipListIterator *pIter = iters[i];
- nextKey = tsdbNextIterKey(pIter);
- if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
- }
+static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+ if (pBlock->len <= 0) return -1;
+ pIter->totalLen = pBlock->len;
+ pIter->len = 0;
+ pIter->row = (SDataRow)(pBlock->data);
return 0;
}
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
- int8_t oldCompRession = pRepo->config.compression;
+ int8_t ocompression = pRepo->config.compression;
pRepo->config.compression = compression;
- tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression);
+ tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", REPO_ID(pRepo), ocompression, compression);
}
-static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
- STsdbCfg *pCfg = &pRepo->config;
- int oldKeep = pCfg->keep;
+static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+ int okeep = pCfg->keep;
+ SFileGroup *pFGroup = NULL;
- int maxFiles = keep / pCfg->maxTables + 3;
- if (pRepo->config.keep > keep) {
- pRepo->config.keep = keep;
- pRepo->tsdbFileH->maxFGroups = maxFiles;
- } else {
- pRepo->config.keep = keep;
- pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup));
- if (pRepo->tsdbFileH->fGroup == NULL) {
- // TODO: deal with the error
+ ASSERT(pCfg->keep != keep);
+ int maxFiles = TSDB_MAX_FILE(keep, pCfg->daysPerFile);
+
+ if (maxFiles != pFileH->maxFGroups) {
+ pthread_rwlock_wrlock(&(pFileH->fhlock));
+
+ pCfg->keep = keep;
+ pFGroup = (SFileGroup *)calloc(maxFiles, sizeof(SFileGroup));
+ if (pFGroup == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ pthread_rwlock_unlock(&(pFileH->fhlock));
+ return -1;
}
- pRepo->tsdbFileH->maxFGroups = maxFiles;
+
+ int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) -
+ TSDB_MAX_FILE(keep, pCfg->daysPerFile);
+
+ int i = 0;
+ for (; i < pFileH->nFGroups; i++) {
+ if (pFileH->pFGroup[i].fileId >= mfid) break;
+ tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[i]));
+ }
+
+ for (int j = 0; i < pFileH->nFGroups; i++, j++) {
+ pFGroup[j] = pFileH->pFGroup[i];
+ }
+
+ free(pFileH->pFGroup);
+ pFileH->pFGroup = pFGroup;
+
+ pthread_rwlock_unlock(&(pFileH->fhlock));
}
- tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep);
+
+ tsdbTrace("vgId:%d keep is changed from %d to %d", REPO_ID(pRepo), okeep, keep);
+
+ return 0;
}
-static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
+static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
+ // TODO
int oldMaxTables = pRepo->config.maxTables;
if (oldMaxTables < pRepo->config.maxTables) {
- // TODO
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
}
STsdbMeta *pMeta = pRepo->tsdbMeta;
- pMeta->maxTables = maxTables;
pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *));
- memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables));
+ memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables));
pRepo->config.maxTables = maxTables;
+ if (pRepo->mem) {
+ pRepo->mem->tData = realloc(pRepo->mem->tData, maxTables * sizeof(STableData *));
+ memset(POINTER_SHIFT(pRepo->mem->tData, sizeof(STableData *) * oldMaxTables), 0,
+ sizeof(STableData *) * (maxTables - oldMaxTables));
+ }
+
+ if (pRepo->imem) {
+ pRepo->imem->tData = realloc(pRepo->imem->tData, maxTables * sizeof(STableData *));
+ memset(POINTER_SHIFT(pRepo->imem->tData, sizeof(STableData *) * oldMaxTables), 0,
+ sizeof(STableData *) * (maxTables - oldMaxTables));
+ }
+
tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
+ return 0;
}
-#define TSDB_META_FILE_INDEX 10000000
-uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- // STsdbMeta *pMeta = pRepo->tsdbMeta;
- STsdbFileH *pFileH = pRepo->tsdbFileH;
- uint32_t magic = 0;
- char fname[256] = "\0";
+static int keyFGroupCompFunc(const void *key, const void *fgroup) {
+ int fid = *(int *)key;
+ SFileGroup *pFGroup = (SFileGroup *)fgroup;
+ if (fid == pFGroup->fileId) {
+ return 0;
+ } else {
+ return fid > pFGroup->fileId ? 1 : -1;
+ }
+}
- struct stat fState;
+static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
+ int tlen = 0;
+
+ tlen += taosEncodeVariantI32(buf, pCfg->tsdbId);
+ tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize);
+ tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks);
+ tlen += taosEncodeVariantI32(buf, pCfg->maxTables);
+ tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile);
+ tlen += taosEncodeVariantI32(buf, pCfg->keep);
+ tlen += taosEncodeVariantI32(buf, pCfg->keep1);
+ tlen += taosEncodeVariantI32(buf, pCfg->keep2);
+ tlen += taosEncodeVariantI32(buf, pCfg->minRowsPerFileBlock);
+ tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
+ tlen += taosEncodeFixedI8(buf, pCfg->precision);
+ tlen += taosEncodeFixedI8(buf, pCfg->compression);
+
+ return tlen;
+}
- tsdbTrace("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex);
- ASSERT(*index <= eindex);
+static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
+ buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId));
+ buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize));
+ buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks));
+ buf = taosDecodeVariantI32(buf, &(pCfg->maxTables));
+ buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile));
+ buf = taosDecodeVariantI32(buf, &(pCfg->keep));
+ buf = taosDecodeVariantI32(buf, &(pCfg->keep1));
+ buf = taosDecodeVariantI32(buf, &(pCfg->keep2));
+ buf = taosDecodeVariantI32(buf, &(pCfg->minRowsPerFileBlock));
+ buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
+ buf = taosDecodeFixedI8(buf, &(pCfg->precision));
+ buf = taosDecodeFixedI8(buf, &(pCfg->compression));
+
+ return buf;
+}
- char *sdup = strdup(pRepo->rootDir);
- char *prefix = dirname(sdup);
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
+ ASSERT(pTable != NULL);
+
+ STSchema *pSchema = tsdbGetTableSchema(pTable);
+ int sversion = schemaVersion(pSchema);
+
+ if (pBlock->sversion == sversion) return 0;
+ if (pBlock->sversion > sversion) { // need to config
+ tsdbTrace("vgId:%d table %s tid %d has version %d smaller than client version %d, try to config", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), sversion, pBlock->sversion);
+ if (pRepo->appH.configFunc) {
+ void *msg = (*pRepo->appH.configFunc)(REPO_ID(pRepo), TABLE_TID(pTable));
+ if (msg == NULL) {
+ tsdbError("vgId:%d failed to config table %s tid %d since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ TABLE_TID(pTable), tstrerror(terrno));
+ return -1;
+ }
- if (name[0] == 0) { // get the file from index or after, but not larger than eindex
- int fid = (*index) / 3;
+ STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg);
+ if (pTableCfg == NULL) {
+ rpcFreeCont(msg);
+ return -1;
+ }
- if (pFileH->numOfFGroups == 0 || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) {
- if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
- tsdbGetMetaFileName(pRepo->rootDir, fname);
- *index = TSDB_META_FILE_INDEX;
- } else {
- tfree(sdup);
- return 0;
+ if (tsdbUpdateTable(pRepo, (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable, pTableCfg) < 0) {
+ tsdbError("vgId:%d failed to update table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ tsdbClearTableCfg(pTableCfg);
+ rpcFreeCont(msg);
+ return -1;
}
+ tsdbClearTableCfg(pTableCfg);
+ rpcFreeCont(msg);
} else {
- SFileGroup *pFGroup =
- taosbsearch(&fid, pFileH->fGroup, pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE);
- if (pFGroup->fileId == fid) {
- strcpy(fname, pFGroup->files[(*index) % 3].fname);
- } else {
- if (pFGroup->fileId * 3 + 2 < eindex) {
- strcpy(fname, pFGroup->files[0].fname);
- *index = pFGroup->fileId * 3;
- } else {
- tfree(sdup);
- return 0;
- }
- }
+ terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+ return -1;
}
- strcpy(name, fname + strlen(prefix));
- } else { // get the named file at the specified index. If not there, return 0
- if (*index == TSDB_META_FILE_INDEX) { // get meta file
- tsdbGetMetaFileName(pRepo->rootDir, fname);
- } else {
- int fid = (*index) / 3;
- SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid);
- if (pFGroup == NULL) { // not found
- tfree(sdup);
- return 0;
- }
-
- SFile *pFile = &pFGroup->files[(*index) % 3];
- strcpy(fname, pFile->fname);
+ } else {
+ if (tsdbGetTableSchemaByVersion(pTable, pBlock->sversion) == NULL) {
+ tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
+ pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
}
+ terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+ return -1;
}
- if (stat(fname, &fState) < 0) {
- tfree(sdup);
- return 0;
- }
+ return 0;
+ }
- tfree(sdup);
- *size = fState.st_size;
- magic = *size;
+static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
+ // TODO
+ // STsdbCache *pCache = pRepo->tsdbCache;
+ // int oldNumOfBlocks = pCache->totalCacheBlocks;
+
+ // tsdbLockRepo((TsdbRepoT *)pRepo);
+
+ // ASSERT(pCache->totalCacheBlocks != totalBlocks);
+
+ // if (pCache->totalCacheBlocks < totalBlocks) {
+ // ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks);
+ // int blocksToAdd = pCache->totalCacheBlocks - totalBlocks;
+ // pCache->totalCacheBlocks = totalBlocks;
+ // for (int i = 0; i < blocksToAdd; i++) {
+ // if (tsdbAddCacheBlockToPool(pCache) < 0) {
+ // tsdbUnLockRepo((TsdbRepoT *)pRepo);
+ // tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId);
+ // return -1;
+ // }
+ // }
+ // } else {
+ // pCache->totalCacheBlocks = totalBlocks;
+ // tsdbAdjustCacheBlocks(pCache);
+ // }
+ // pRepo->config.totalBlocks = totalBlocks;
+
+ // tsdbUnLockRepo((TsdbRepoT *)pRepo);
+ // tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks,
+ // totalBlocks);
+ return 0;
+}
- return magic;
+#if 0
+
+TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+ STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
+ if (pTable == NULL) return -1;
+
+ return TSDB_GET_TABLE_LAST_KEY(pTable);
}
-void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){
- ASSERT(repo != NULL);
- STsdbRepo * pRepo = repo;
- *totalPoints = pRepo->stat.pointsWritten;
- *totalStorage = pRepo->stat.totalStorage;
- *compStorage = pRepo->stat.compStorage;
-}
\ No newline at end of file
+#endif
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c
new file mode 100644
index 0000000000000000000000000000000000000000..b1de90bc291cc909b51154b3f54e0d59a0278005
--- /dev/null
+++ b/src/tsdb/src/tsdbMemTable.c
@@ -0,0 +1,691 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "tsdb.h"
+#include "tsdbMain.h"
+
+#define TSDB_DATA_SKIPLIST_LEVEL 5
+
+typedef struct {
+ STable * pTable;
+ SSkipListIterator *pIter;
+} SCommitIter;
+
+static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo);
+
+static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes);
+static SMemTable * tsdbNewMemTable(STsdbCfg *pCfg);
+static void tsdbFreeMemTable(SMemTable *pMemTable);
+static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
+static void tsdbFreeTableData(STableData *pTableData);
+static char * tsdbGetTsTupleKey(const void *data);
+static void * tsdbCommitData(void *arg);
+static int tsdbCommitMeta(STsdbRepo *pRepo);
+static void tsdbEndCommit(STsdbRepo *pRepo);
+static TSKEY tsdbNextIterKey(SCommitIter *pIter);
+static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
+static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
+static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo);
+static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables);
+static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey,
+ int maxRowsToRead, SDataCols *pCols);
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
+ STsdbCfg * pCfg = &pRepo->config;
+ int32_t level = 0;
+ int32_t headSize = 0;
+ TSKEY key = dataRowKey(row);
+ SMemTable * pMemTable = pRepo->mem;
+ STableData *pTableData = NULL;
+ SSkipList * pSList = NULL;
+ int bytes = 0;
+
+ if (pMemTable != NULL && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
+ pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
+ pTableData = pMemTable->tData[TABLE_TID(pTable)];
+ pSList = pTableData->pData;
+ }
+
+ tSkipListNewNodeInfo(pSList, &level, &headSize);
+
+ bytes = headSize + dataRowLen(row);
+ SSkipListNode *pNode = tsdbAllocBytes(pRepo, bytes);
+ if (pNode == NULL) {
+ tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
+ REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), bytes, tstrerror(terrno));
+ return -1;
+ }
+ pNode->level = level;
+ dataRowCpy(SL_GET_NODE_DATA(pNode), row);
+
+ // Operations above may change pRepo->mem, retake those values
+ ASSERT(pRepo->mem != NULL);
+ pMemTable = pRepo->mem;
+ pTableData = pMemTable->tData[TABLE_TID(pTable)];
+
+ if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
+ if (pTableData != NULL) { // destroy the table skiplist (may have race condition problem)
+ pMemTable->tData[TABLE_TID(pTable)] = NULL;
+ tsdbFreeTableData(pTableData);
+ }
+ pTableData = tsdbNewTableData(pCfg, pTable);
+ if (pTableData == NULL) {
+ tsdbError("vgId:%d failed to insert row with key %" PRId64
+ " to table %s while create new table data object since %s",
+ REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
+ tsdbFreeBytes(pRepo, (void *)pNode, bytes);
+ return -1;
+ }
+
+ pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
+ }
+
+ ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
+
+ if (tSkipListPut(pTableData->pData, pNode) == NULL) {
+ tsdbFreeBytes(pRepo, (void *)pNode, bytes);
+ } else {
+ if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
+ if (pMemTable->keyLast < key) pMemTable->keyLast = key;
+ pMemTable->numOfRows++;
+
+ if (pTableData->keyFirst > key) pTableData->keyFirst = key;
+ if (pTableData->keyLast < key) pTableData->keyLast = key;
+ pTableData->numOfRows++;
+
+ ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
+ }
+
+ tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
+
+ return 0;
+}
+
+int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
+ if (pMemTable == NULL) return 0;
+ T_REF_INC(pMemTable);
+ return 0;
+}
+
+// Need to lock the repository
+int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
+ if (pMemTable == NULL) return 0;
+
+ if (T_REF_DEC(pMemTable) == 0) {
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbBufPool *pBufPool = pRepo->pPool;
+
+ SListNode *pNode = NULL;
+ if (tsdbLockRepo(pRepo) < 0) return -1;
+ while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) {
+ tdListAppendNode(pBufPool->bufBlockList, pNode);
+ }
+ int code = pthread_cond_signal(&pBufPool->poolNotEmpty);
+ if (code != 0) {
+ tsdbUnlockRepo(pRepo);
+ tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+ for (int i = 0; i < pCfg->maxTables; i++) {
+ if (pMemTable->tData[i] != NULL) {
+ tsdbFreeTableData(pMemTable->tData[i]);
+ }
+ }
+
+ tdListDiscard(pMemTable->actList);
+ tdListDiscard(pMemTable->bufBlockList);
+ tsdbFreeMemTable(pMemTable);
+ }
+ return 0;
+}
+
+int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) {
+ if (tsdbLockRepo(pRepo) < 0) return -1;
+
+ *pMem = pRepo->mem;
+ *pIMem = pRepo->imem;
+ tsdbRefMemTable(pRepo, *pMem);
+ tsdbRefMemTable(pRepo, *pIMem);
+
+ if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+ return 0;
+}
+
+void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
+
+ if (pBufBlock != NULL && pBufBlock->remain < bytes) {
+ if (listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 2) { // need to commit mem
+ if (tsdbAsyncCommit(pRepo) < 0) return NULL;
+ } else {
+ if (tsdbLockRepo(pRepo) < 0) return NULL;
+ SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo);
+ tdListAppendNode(pRepo->mem->bufBlockList, pNode);
+ if (tsdbUnlockRepo(pRepo) < 0) return NULL;
+ }
+ }
+
+ if (pRepo->mem == NULL) {
+ SMemTable *pMemTable = tsdbNewMemTable(&pRepo->config);
+ if (pMemTable == NULL) return NULL;
+
+ if (tsdbLockRepo(pRepo) < 0) {
+ tsdbFreeMemTable(pMemTable);
+ return NULL;
+ }
+
+ SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo);
+ tdListAppendNode(pMemTable->bufBlockList, pNode);
+ pRepo->mem = pMemTable;
+
+ if (tsdbUnlockRepo(pRepo) < 0) return NULL;
+ }
+
+ pBufBlock = tsdbGetCurrBufBlock(pRepo);
+ ASSERT(pBufBlock->remain >= bytes);
+ void *ptr = POINTER_SHIFT(pBufBlock->data, pBufBlock->offset);
+ pBufBlock->offset += bytes;
+ pBufBlock->remain -= bytes;
+
+ return ptr;
+}
+
+int tsdbAsyncCommit(STsdbRepo *pRepo) {
+ SMemTable *pIMem = pRepo->imem;
+ int code = 0;
+
+ if (pIMem != NULL) {
+ ASSERT(pRepo->commit);
+ code = pthread_join(pRepo->commitThread, NULL);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+ pRepo->commit = 0;
+ }
+
+ ASSERT(pRepo->commit == 0);
+ if (pRepo->mem != NULL) {
+ if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
+ if (tsdbLockRepo(pRepo) < 0) return -1;
+ pRepo->imem = pRepo->mem;
+ pRepo->mem = NULL;
+ pRepo->commit = 1;
+ code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ tsdbUnlockRepo(pRepo);
+ return -1;
+ }
+ if (tsdbUnlockRepo(pRepo) < 0) return -1;
+ }
+
+ if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
+
+ return 0;
+}
+
+// ---------------- LOCAL FUNCTIONS ----------------
+static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) {
+ ASSERT(pRepo != NULL);
+ if (pRepo->mem == NULL) return NULL;
+
+ SListNode *pNode = listTail(pRepo->mem->bufBlockList);
+ if (pNode == NULL) return NULL;
+
+ STsdbBufBlock *pBufBlock = NULL;
+ tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void *)(&pBufBlock));
+
+ return pBufBlock;
+}
+
+static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
+ STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
+ ASSERT(pBufBlock != NULL);
+ pBufBlock->offset -= bytes;
+ pBufBlock->remain += bytes;
+ ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
+}
+
+static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) {
+ SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable));
+ if (pMemTable == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pMemTable->keyFirst = INT64_MAX;
+ pMemTable->keyLast = 0;
+ pMemTable->numOfRows = 0;
+
+ pMemTable->tData = (STableData**)calloc(pCfg->maxTables, sizeof(STableData*));
+ if (pMemTable->tData == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pMemTable->actList = tdListNew(0);
+ if (pMemTable->actList == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pMemTable->bufBlockList = tdListNew(sizeof(STsdbBufBlock*));
+ if (pMemTable->bufBlockList == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ T_REF_INC(pMemTable);
+
+ return pMemTable;
+
+_err:
+ tsdbFreeMemTable(pMemTable);
+ return NULL;
+}
+
+static void tsdbFreeMemTable(SMemTable* pMemTable) {
+ if (pMemTable) {
+ ASSERT((pMemTable->bufBlockList == NULL) ? true : (listNEles(pMemTable->bufBlockList) == 0));
+ ASSERT((pMemTable->actList == NULL) ? true : (listNEles(pMemTable->actList) == 0));
+
+ tdListFree(pMemTable->bufBlockList);
+ tdListFree(pMemTable->actList);
+ tfree(pMemTable->tData);
+ free(pMemTable);
+ }
+}
+
+static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
+ STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData));
+ if (pTableData == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ pTableData->uid = TABLE_UID(pTable);
+ pTableData->keyFirst = INT64_MAX;
+ pTableData->keyLast = 0;
+ pTableData->numOfRows = 0;
+
+ pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
+ TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, tsdbGetTsTupleKey);
+ if (pTableData->pData == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ // TODO: operation here should not be here, remove it
+ pTableData->pData->level = 1;
+
+ return pTableData;
+
+_err:
+ tsdbFreeTableData(pTableData);
+ return NULL;
+}
+
+static void tsdbFreeTableData(STableData *pTableData) {
+ if (pTableData) {
+ tSkipListDestroy(pTableData->pData);
+ free(pTableData);
+ }
+}
+
+static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); }
+
+static void *tsdbCommitData(void *arg) {
+ STsdbRepo * pRepo = (STsdbRepo *)arg;
+ SMemTable * pMem = pRepo->imem;
+ STsdbCfg * pCfg = &pRepo->config;
+ SDataCols * pDataCols = NULL;
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ SCommitIter *iters = NULL;
+ SRWHelper whelper = {0};
+ ASSERT(pRepo->commit == 1);
+ ASSERT(pMem != NULL);
+
+ tsdbPrint("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
+ pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
+
+ // Create the iterator to read from cache
+ if (pMem->numOfRows > 0) {
+ iters = tsdbCreateTableIters(pRepo);
+ if (iters == NULL) {
+ tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
+
+ if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
+ tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
+
+ if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
+ REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
+ goto _exit;
+ }
+
+ int sfid = TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision);
+ int efid = TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision);
+
+ // Loop to commit to each file
+ for (int fid = sfid; fid <= efid; fid++) {
+ if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
+ tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _exit;
+ }
+ }
+ }
+
+ // Commit to update meta file
+ if (tsdbCommitMeta(pRepo) < 0) {
+ tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
+
+ tsdbFitRetention(pRepo);
+
+_exit:
+ tdFreeDataCols(pDataCols);
+ tsdbDestroyTableIters(iters, pCfg->maxTables);
+ tsdbDestroyHelper(&whelper);
+ tsdbEndCommit(pRepo);
+ tsdbPrint("vgId:%d commit over", pRepo->config.tsdbId);
+
+ return NULL;
+}
+
+static int tsdbCommitMeta(STsdbRepo *pRepo) {
+ SMemTable *pMem = pRepo->imem;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ SActObj * pAct = NULL;
+ SActCont * pCont = NULL;
+
+ if (listNEles(pMem->actList) > 0) {
+ if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
+ tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ SListNode *pNode = NULL;
+
+ while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
+ pAct = (SActObj *)pNode->data;
+ if (pAct->act == TSDB_UPDATE_META) {
+ pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
+ if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
+ tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+ tstrerror(terrno));
+ goto _err;
+ }
+ } else if (pAct->act == TSDB_DROP_META) {
+ if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
+ tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+ tstrerror(terrno));
+ goto _err;
+ }
+ } else {
+ ASSERT(false);
+ }
+ }
+
+ if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
+ tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+ }
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+static void tsdbEndCommit(STsdbRepo *pRepo) {
+ ASSERT(pRepo->commit == 1);
+ if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
+}
+
+static TSKEY tsdbNextIterKey(SCommitIter *pIter) {
+ if (pIter == NULL) return -1;
+
+ SSkipListNode *node = tSkipListIterGet(pIter->pIter);
+ if (node == NULL) return -1;
+
+ SDataRow row = SL_GET_NODE_DATA(node);
+ return dataRowKey(row);
+}
+
+static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
+ for (int i = 0; i < nIters; i++) {
+ TSKEY nextKey = tsdbNextIterKey(iters + i);
+ if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+ }
+ return 0;
+}
+
+static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
+ *minKey = fileId * daysPerFile * tsMsPerDay[precision];
+ *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
+}
+
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
+ char * dataDir = NULL;
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+ SFileGroup *pGroup = NULL;
+
+ TSKEY minKey = 0, maxKey = 0;
+ tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
+
+ // Check if there are data to commit to this file
+ int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey);
+ if (!hasDataToCommit) {
+ tsdbTrace("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
+ return 0;
+ }
+
+ // Create and open files for commit
+ dataDir = tsdbGetDataDirName(pRepo->rootDir);
+ if (dataDir == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid, pCfg->maxTables)) == NULL) {
+ tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _err;
+ }
+
+ // Open files for write/read
+ if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
+ tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ // Loop to commit data in each table
+ for (int tid = 1; tid < pCfg->maxTables; tid++) {
+ SCommitIter *pIter = iters + tid;
+ if (pIter->pTable == NULL) continue;
+
+ tsdbSetHelperTable(pHelper, pIter->pTable, pRepo);
+
+ if (pIter->pIter != NULL) {
+ tdInitDataCols(pDataCols, tsdbGetTableSchema(pIter->pTable));
+
+ int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5;
+ int nLoop = 0;
+ while (true) {
+ int rowsRead = tsdbReadRowsFromCache(pMeta, pIter->pTable, pIter->pIter, maxKey, maxRowsToRead, pDataCols);
+ ASSERT(rowsRead >= 0);
+ if (pDataCols->numOfRows == 0) break;
+ nLoop++;
+
+ ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
+ ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
+
+ int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
+ ASSERT(rowsWritten != 0);
+ if (rowsWritten < 0) {
+ tsdbError("vgId:%d failed to write data block to table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
+ tstrerror(terrno));
+ goto _err;
+ }
+ ASSERT(rowsWritten <= pDataCols->numOfRows);
+
+ tdPopDataColsPoints(pDataCols, rowsWritten);
+ maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows;
+ }
+
+ ASSERT(pDataCols->numOfRows == 0);
+ }
+
+ // Move the last block to the new .l file if neccessary
+ if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
+ tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ // Write the SCompBlock part
+ if (tsdbWriteCompInfo(pHelper) < 0) {
+ tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+ }
+
+ if (tsdbWriteCompIdx(pHelper) < 0) {
+ tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _err;
+ }
+
+ tsdbCloseHelperFile(pHelper, 0);
+
+ pthread_rwlock_wrlock(&(pFileH->fhlock));
+ pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF;
+ pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF;
+ pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF;
+ pthread_rwlock_unlock(&(pFileH->fhlock));
+
+ return 0;
+
+_err:
+ tfree(dataDir);
+ tsdbCloseHelperFile(pHelper, 1);
+ return -1;
+}
+
+static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo) {
+ STsdbCfg * pCfg = &(pRepo->config);
+ SMemTable *pMem = pRepo->imem;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+ SCommitIter *iters = (SCommitIter *)calloc(pCfg->maxTables, sizeof(SCommitIter));
+ if (iters == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+
+ // reference all tables
+ for (int i = 0; i < pCfg->maxTables; i++) {
+ if (pMeta->tables[i] != NULL) {
+ tsdbRefTable(pMeta->tables[i]);
+ iters[i].pTable = pMeta->tables[i];
+ }
+ }
+
+ if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+
+ for (int i = 0; i < pCfg->maxTables; i++) {
+ if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
+ if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ if (!tSkipListIterNext(iters[i].pIter)) {
+ terrno = TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM;
+ goto _err;
+ }
+ }
+ }
+
+ return iters;
+
+_err:
+ tsdbDestroyTableIters(iters, pCfg->maxTables);
+ return NULL;
+}
+
+static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables) {
+ if (iters == NULL) return;
+
+ for (int i = 1; i < maxTables; i++) {
+ if (iters[i].pTable != NULL) {
+ tsdbUnRefTable(iters[i].pTable);
+ tSkipListDestroyIter(iters[i].pIter);
+ }
+ }
+
+ free(iters);
+}
+
+static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) {
+ ASSERT(maxRowsToRead > 0);
+ if (pIter == NULL) return 0;
+ STSchema *pSchema = NULL;
+
+ int numOfRows = 0;
+
+ do {
+ if (numOfRows >= maxRowsToRead) break;
+
+ SSkipListNode *node = tSkipListIterGet(pIter);
+ if (node == NULL) break;
+
+ SDataRow row = SL_GET_NODE_DATA(node);
+ if (dataRowKey(row) > maxKey) break;
+
+ if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+ pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
+ if (pSchema == NULL) {
+ // TODO: deal with the error here
+ ASSERT(false);
+ }
+ }
+
+ tdAppendDataRowToDataCol(row, pSchema, pCols);
+ numOfRows++;
+ } while (tSkipListIterNext(pIter));
+
+ return numOfRows;
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index 1f31f18ce1cfb0fcda12872eb33963bd8b58d197..83392462abe7c39e2733d13fdd719df3372f0489 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -1,265 +1,462 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
#include
-#include "tskiplist.h"
-#include "tsdb.h"
-#include "taosdef.h"
#include "hash.h"
+#include "taosdef.h"
+#include "tchecksum.h"
+#include "tsdb.h"
#include "tsdbMain.h"
+#include "tskiplist.h"
+
+#define TSDB_SUPER_TABLE_SL_LEVEL 5
+#define DEFAULT_TAG_INDEX_COLUMN 0
+
+static int tsdbCompareSchemaVersion(const void *key1, const void *key2);
+static int tsdbRestoreTable(void *pHandle, void *cont, int contLen);
+static void tsdbOrgMeta(void *pHandle);
+static char * getTagIndexKey(const void *pData);
+static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper);
+static void tsdbFreeTable(STable *pTable);
+static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema);
+static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx);
+static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock);
+static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
+static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
+static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid);
+static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
+static int tsdbTableSetName(STableCfg *config, char *name, bool dup);
+static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
+static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
+static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid);
+static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup);
+static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup);
+static int tsdbEncodeTableName(void **buf, tstr *name);
+static void * tsdbDecodeTableName(void *buf, tstr **name);
+static int tsdbEncodeTable(void **buf, STable *pTable);
+static void * tsdbDecodeTable(void *buf, STable **pRTable);
+static int tsdbGetTableEncodeSize(int8_t act, STable *pTable);
+static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable);
+
+// ------------------ OUTER FUNCTIONS ------------------
+int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ STable * super = NULL;
+ STable * table = NULL;
+ int newSuper = 0;
-#define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
-// #define TSDB_META_FILE_NAME "META"
+ STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
+ if (pTable != NULL) {
+ tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ TABLE_TID(pTable), TABLE_UID(pTable));
+ return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ }
+ if (pCfg->type == TSDB_CHILD_TABLE) {
+ super = tsdbGetTableByUid(pMeta, pCfg->superUid);
+ if (super == NULL) { // super table not exists, try to create it
+ newSuper = 1;
+ super = tsdbNewTable(pCfg, true);
+ if (super == NULL) goto _err;
+ } else {
+ // TODO
+ if (super->type != TSDB_SUPER_TABLE) return -1;
+ if (super->tableId.uid != pCfg->superUid) return -1;
+ tsdbUpdateTable(pRepo, super, pCfg);
+ }
+ }
-static int tsdbFreeTable(STable *pTable);
-static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
-static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx);
-static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx);
+ table = tsdbNewTable(pCfg, false);
+ if (table == NULL) goto _err;
-/**
- * Encode a TSDB table object as a binary content
- * ASSUMPTIONS: VALID PARAMETERS
- *
- * @param pTable table object to encode
- * @param contLen the encoded binary content length
- *
- * @return binary content for success
- * NULL fro failure
- */
-void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) {
- if (pTable == NULL) return;
+ // Register to meta
+ if (newSuper) {
+ if (tsdbAddTableToMeta(pRepo, super, true) < 0) goto _err;
+ }
+ if (tsdbAddTableToMeta(pRepo, table, true) < 0) goto _err;
+
+ // Write to memtable action
+ int tlen1 = (newSuper) ? tsdbGetTableEncodeSize(TSDB_UPDATE_META, super) : 0;
+ int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table);
+ int tlen = tlen1 + tlen2;
+ void *buf = tsdbAllocBytes(pRepo, tlen);
+ ASSERT(buf != NULL);
+ if (newSuper) {
+ void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super);
+ ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1);
+ buf = pBuf;
+ }
+ tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, table);
- void *ptr = buf;
- T_APPEND_MEMBER(ptr, pTable, STable, type);
- // Encode name, todo refactor
- *(int *)ptr = varDataLen(pTable->name);
- ptr = (char *)ptr + sizeof(int);
- memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name));
- ptr = (char *)ptr + varDataLen(pTable->name);
+ return 0;
- T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid);
- T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid);
- T_APPEND_MEMBER(ptr, pTable, STable, superUid);
+_err:
+ tsdbFreeTable(super);
+ tsdbFreeTable(table);
+ return -1;
+}
- if (pTable->type == TSDB_SUPER_TABLE) {
- T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas);
- for (int i = 0; i < pTable->numOfSchemas; i++) {
- ptr = tdEncodeSchema(ptr, pTable->schema[i]);
- }
- ptr = tdEncodeSchema(ptr, pTable->tagSchema);
- } else if (pTable->type == TSDB_CHILD_TABLE) {
- ptr = tdEncodeKVRow(ptr, pTable->tagVal);
- } else {
- T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas);
- for (int i = 0; i < pTable->numOfSchemas; i++) {
- ptr = tdEncodeSchema(ptr, pTable->schema[i]);
+int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ uint64_t uid = tableId.uid;
+ int tid = 0;
+ char * tbname = NULL;
+
+ STable *pTable = tsdbGetTableByUid(pMeta, uid);
+ if (pTable == NULL) {
+ tsdbError("vgId:%d failed to drop table since table not exists! tid:%d uid %" PRId64, REPO_ID(pRepo), tableId.tid,
+ uid);
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
+ }
+
+ tsdbTrace("vgId:%d try to drop table %s type %d", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable));
+
+ tid = TABLE_TID(pTable);
+ tbname = strdup(TABLE_CHAR_NAME(pTable));
+ if (tbname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {
+ if (pTable->cqhandle) pRepo->appH.cqDropFunc(pTable->cqhandle);
+ }
+
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex);
+ while (tSkipListIterNext(pIter)) {
+ STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+ ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
+ int tlen = tsdbGetTableEncodeSize(TSDB_DROP_META, tTable);
+ void *buf = tsdbAllocBytes(pRepo, tlen);
+ ASSERT(buf != NULL);
+ tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable);
+ tsdbRemoveTableFromMeta(pRepo, tTable, false, true);
}
+ tSkipListDestroyIter(pIter);
+ }
+
+ tsdbRemoveTableFromMeta(pRepo, pTable, true, true);
+
+ tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, tbname, tid, uid);
+ free(tbname);
+
+ return 0;
+}
+
+void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes) {
+ // TODO: this function should be changed also
+ STsdbMeta *pMeta = tsdbGetMeta(repo);
+ STable * pTable = tsdbGetTableByUid(pMeta, id->uid);
+
+ STSchema *pSchema = tsdbGetTableTagSchema(pTable);
+ STColumn *pCol = tdGetColOfID(pSchema, colId);
+ if (pCol == NULL) {
+ return NULL; // No matched tag volumn
}
- if (pTable->type == TSDB_STREAM_TABLE) {
- ptr = taosEncodeString(ptr, pTable->sql);
+ char *val = tdGetKVRowValOfCol(pTable->tagVal, colId);
+ assert(type == pCol->type && bytes == pCol->bytes);
+
+ if (val != NULL && IS_VAR_DATA_TYPE(type)) {
+ assert(varDataLen(val) < pCol->bytes);
}
- *contLen = (char *)ptr - buf;
+ return val;
}
-/**
- * Decode from an encoded binary
- * ASSUMPTIONS: valid parameters
- *
- * @param cont binary object
- * @param contLen binary length
- *
- * @return TSDB table object for success
- * NULL for failure
- */
-STable *tsdbDecodeTable(void *cont, int contLen) {
- STable *pTable = (STable *)calloc(1, sizeof(STable));
- if (pTable == NULL) return NULL;
+char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) {
+ // TODO: need to change as thread-safe
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
- void *ptr = cont;
- T_READ_MEMBER(ptr, int8_t, pTable->type);
- if (pTable->type != TSDB_CHILD_TABLE) {
- pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
- if (pTable->schema == NULL) {
- free(pTable);
- return NULL;
- }
+ STable *pTable = tsdbGetTableByUid(pMeta, id->uid);
+
+ if (pTable == NULL) {
+ return NULL;
+ } else {
+ return (char *)pTable->name;
}
- int len = *(int *)ptr;
- ptr = (char *)ptr + sizeof(int);
- pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1);
- if (pTable->name == NULL) return NULL;
+}
- varDataSetLen(pTable->name, len);
- memcpy(pTable->name->data, ptr, len);
+STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
+ if (pMsg == NULL) return NULL;
- ptr = (char *)ptr + len;
- T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid);
- T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid);
- T_READ_MEMBER(ptr, uint64_t, pTable->superUid);
+ SSchema *pSchema = (SSchema *)pMsg->data;
+ int16_t numOfCols = htons(pMsg->numOfColumns);
+ int16_t numOfTags = htons(pMsg->numOfTags);
- if (pTable->type == TSDB_SUPER_TABLE) {
- T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas);
- for (int i = 0; i < pTable->numOfSchemas; i++) {
- pTable->schema[i] = tdDecodeSchema(&ptr);
+ STSchemaBuilder schemaBuilder = {0};
+
+ STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg));
+ if (pCfg == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err;
+ if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ for (int i = 0; i < numOfCols; i++) {
+ if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- pTable->tagSchema = tdDecodeSchema(&ptr);
- } else if (pTable->type == TSDB_CHILD_TABLE) {
- ptr = tdDecodeKVRow(ptr, &pTable->tagVal);
- } else {
- T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas);
- for (int i = 0; i < pTable->numOfSchemas; i++) {
- pTable->schema[i] = tdDecodeSchema(&ptr);
+ }
+ if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err;
+ if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err;
+
+ if (numOfTags > 0) {
+ // Decode tag schema
+ tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion));
+ for (int i = numOfCols; i < numOfCols + numOfTags; i++) {
+ if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+ if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err;
+ if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err;
+ if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err;
+
+ // Decode tag values
+ if (pMsg->tagDataLen) {
+ int accBytes = 0;
+ char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema);
+
+ SKVRowBuilder kvRowBuilder = {0};
+ if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ for (int i = numOfCols; i < numOfCols + numOfTags; i++) {
+ if (tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ accBytes += htons(pSchema[i].bytes);
+ }
+
+ tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false);
+ tdDestroyKVRowBuilder(&kvRowBuilder);
}
}
- if (pTable->type == TSDB_STREAM_TABLE) {
- ptr = taosDecodeString(ptr, &(pTable->sql));
+ if (pMsg->tableType == TSDB_STREAM_TABLE) {
+ char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema);
+ tsdbTableSetStreamSql(pCfg, sql, true);
}
- pTable->lastKey = TSKEY_INITIAL_VAL;
- return pTable;
-}
+ tdDestroyTSchemaBuilder(&schemaBuilder);
-void tsdbFreeEncode(void *cont) {
- if (cont != NULL) free(cont);
-}
+ return pCfg;
-static char* getTagIndexKey(const void* pData) {
- STableIndexElem* elem = (STableIndexElem*) pData;
-
- STSchema* pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable);
- STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
- void * res = tdGetKVRowValOfCol(elem->pTable->tagVal, pCol->colId);
- return res;
+_err:
+ tdDestroyTSchemaBuilder(&schemaBuilder);
+ tsdbClearTableCfg(pCfg);
+ return NULL;
}
-int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
- STsdbMeta *pMeta = (STsdbMeta *)pHandle;
+int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ int16_t tversion = htons(pMsg->tversion);
- STable *pTable = tsdbDecodeTable(cont, contLen);
- if (pTable == NULL) return -1;
-
- if (pTable->type == TSDB_SUPER_TABLE) {
- STColumn* pColSchema = schemaColAt(pTable->tagSchema, 0);
- pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
- 1, 0, 1, getTagIndexKey);
+ STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid));
+ if (pTable == NULL) {
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
+ }
+ if (TABLE_TID(pTable) != htonl(pMsg->tid)) {
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
}
- tsdbAddTableToMeta(pMeta, pTable, false);
+ if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
+ tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable));
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
+ }
- return 0;
-}
+ if (schemaVersion(tsdbGetTableTagSchema(pTable)) < tversion) {
+ tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", REPO_ID(pRepo),
+ schemaVersion(tsdbGetTableTagSchema(pTable)), tversion);
+ void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid));
+ if (msg == NULL) return -1;
+
+ // Deal with error her
+ STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg);
+ STable * super = tsdbGetTableByUid(pMeta, pTableCfg->superUid);
+ ASSERT(super != NULL);
+
+ int32_t code = tsdbUpdateTable(pRepo, super, pTableCfg);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbClearTableCfg(pTableCfg);
+ return code;
+ }
+ tsdbClearTableCfg(pTableCfg);
+ rpcFreeCont(msg);
+ }
-void tsdbOrgMeta(void *pHandle) {
- STsdbMeta *pMeta = (STsdbMeta *)pHandle;
+ STSchema *pTagSchema = tsdbGetTableTagSchema(pTable);
- for (int i = 1; i < pMeta->maxTables; i++) {
- STable *pTable = pMeta->tables[i];
- if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) {
- tsdbAddTableIntoIndex(pMeta, pTable);
- }
+ if (schemaVersion(pTagSchema) > tversion) {
+ tsdbError(
+ "vgId:%d failed to update tag value of table %s since version out of date, client tag version %d server tag "
+ "version %d",
+ REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tversion, schemaVersion(pTable->tagSchema));
+ return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE;
+ }
+ if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) {
+ tsdbRemoveTableFromIndex(pMeta, pTable);
+ }
+ // TODO: remove table from index if it is the first column of tag
+ tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data);
+ if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) {
+ tsdbAddTableIntoIndex(pMeta, pTable);
}
+ return TSDB_CODE_SUCCESS;
}
-/**
- * Initialize the meta handle
- * ASSUMPTIONS: VALID PARAMETER
- */
-STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables, void *pRepo) {
- STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
- if (pMeta == NULL) return NULL;
-
- pMeta->maxTables = maxTables;
- pMeta->nTables = 0;
- pMeta->superList = NULL;
- pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *));
- pMeta->maxRowBytes = 0;
- pMeta->maxCols = 0;
- pMeta->pRepo = pRepo;
+// ------------------ INTERNAL FUNCTIONS ------------------
+STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) {
+ STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta));
+ if (pMeta == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ int code = pthread_rwlock_init(&pMeta->rwLock, NULL);
+ if (code != 0) {
+ tsdbError("vgId:%d failed to init TSDB meta r/w lock since %s", pCfg->tsdbId, strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
+ }
+
+ pMeta->tables = (STable **)calloc(pCfg->maxTables, sizeof(STable *));
if (pMeta->tables == NULL) {
- free(pMeta);
- return NULL;
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- pMeta->map = taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
- if (pMeta->map == NULL) {
- free(pMeta->tables);
- free(pMeta);
- return NULL;
+ pMeta->superList = tdListNew(sizeof(STable *));
+ if (pMeta->superList == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- pMeta->mfh = tsdbInitMetaFile(rootDir, maxTables, tsdbRestoreTable, tsdbOrgMeta, pMeta);
- if (pMeta->mfh == NULL) {
- taosHashCleanup(pMeta->map);
- free(pMeta->tables);
- free(pMeta);
- return NULL;
+ pMeta->uidMap = taosHashInit(pCfg->maxTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+ if (pMeta->uidMap == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
return pMeta;
+
+_err:
+ tsdbFreeMeta(pMeta);
+ return NULL;
}
-int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
- STsdbRepo *pRepo = (STsdbRepo *)pMeta->pRepo;
- if (pMeta == NULL) return 0;
+void tsdbFreeMeta(STsdbMeta *pMeta) {
+ if (pMeta) {
+ taosHashCleanup(pMeta->uidMap);
+ tdListFree(pMeta->superList);
+ tfree(pMeta->tables);
+ pthread_rwlock_destroy(&pMeta->rwLock);
+ free(pMeta);
+ }
+}
- tsdbCloseMetaFile(pMeta->mfh);
+int tsdbOpenMeta(STsdbRepo *pRepo) {
+ char * fname = NULL;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ ASSERT(pMeta != NULL);
- for (int i = 1; i < pMeta->maxTables; i++) {
- if (pMeta->tables[i] != NULL) {
- STable *pTable = pMeta->tables[i];
- if (pTable->type == TSDB_STREAM_TABLE) (*pRepo->appH.cqDropFunc)(pTable->cqhandle);
- tsdbFreeTable(pTable);
- }
+ fname = tsdbGetMetaFileName(pRepo->rootDir);
+ if (fname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
}
- free(pMeta->tables);
-
- STable *pTable = pMeta->superList;
- while (pTable != NULL) {
- STable *pTemp = pTable;
- pTable = pTemp->next;
- tsdbFreeTable(pTemp);
+ pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo);
+ if (pMeta->pStore == NULL) {
+ tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
}
- taosHashCleanup(pMeta->map);
+ tsdbTrace("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
+ tfree(fname);
+ return 0;
+
+_err:
+ tfree(fname);
+ return -1;
+}
+
+int tsdbCloseMeta(STsdbRepo *pRepo) {
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ SListNode *pNode = NULL;
+ STable * pTable = NULL;
+
+ if (pMeta == NULL) return 0;
+ tdCloseKVStore(pMeta->pStore);
+ for (int i = 1; i < pCfg->maxTables; i++) {
+ tsdbFreeTable(pMeta->tables[i]);
+ }
- free(pMeta);
+ while ((pNode = tdListPopHead(pMeta->superList)) != NULL) {
+ tdListNodeGetData(pMeta->superList, pNode, (void *)(&pTable));
+ tsdbFreeTable(pTable);
+ listNodeFree(pNode);
+ }
+ tsdbTrace("vgId:%d TSDB meta is closed", REPO_ID(pRepo));
return 0;
}
-// Get the newest table schema
-STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) {
+STSchema *tsdbGetTableSchema(STable *pTable) {
if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) {
return pTable->schema[pTable->numOfSchemas - 1];
} else if (pTable->type == TSDB_CHILD_TABLE) {
- STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid);
+ STable *pSuper = pTable->pSuper;
if (pSuper == NULL) return NULL;
- return pSuper->schema[pSuper->numOfSchemas-1];
+ return pSuper->schema[pSuper->numOfSchemas - 1];
} else {
return NULL;
}
}
-static int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
- if (*(int16_t *)key1 < (*(STSchema **)key2)->version) {
- return -1;
- } else if (*(int16_t *)key1 > (*(STSchema **)key2)->version) {
- return 1;
- } else {
- return 0;
- }
+STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) {
+ void *ptr = taosHashGet(pMeta->uidMap, (char *)(&uid), sizeof(uid));
+
+ if (ptr == NULL) return NULL;
+
+ return *(STable **)ptr;
}
-STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version) {
- STable *pSearchTable = NULL;
- if (pTable->type == TSDB_CHILD_TABLE) {
- pSearchTable = tsdbGetTableByUid(pMeta, pTable->superUid);
- } else {
- pSearchTable = pTable;
- }
- ASSERT(pSearchTable != NULL);
+STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t version) {
+ STable *pSearchTable = (pTable->type == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
+ if (pSearchTable == NULL) return NULL;
void *ptr = taosbsearch(&version, pSearchTable->schema, pSearchTable->numOfSchemas, sizeof(STSchema *),
tsdbCompareSchemaVersion, TD_EQ);
@@ -268,11 +465,11 @@ STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t
return *(STSchema **)ptr;
}
-STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) {
+STSchema *tsdbGetTableTagSchema(STable *pTable) {
if (pTable->type == TSDB_SUPER_TABLE) {
return pTable->tagSchema;
} else if (pTable->type == TSDB_CHILD_TABLE) {
- STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid);
+ STable *pSuper = pTable->pSuper;
if (pSuper == NULL) return NULL;
return pSuper->tagSchema;
} else {
@@ -280,37 +477,148 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) {
}
}
-void* tsdbGetTableTagVal(TsdbRepoT* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) {
- STsdbMeta* pMeta = tsdbGetMeta(repo);
- STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
+int tsdbUpdateTable(STsdbRepo *pRepo, STable *pTable, STableCfg *pCfg) {
+ // TODO: this function can only be called when there is no query and commit on this table
+ ASSERT(TABLE_TYPE(pTable) != TSDB_CHILD_TABLE);
+ bool changed = false;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
- STSchema *pSchema = tsdbGetTableTagSchema(pMeta, pTable);
- STColumn *pCol = tdGetColOfID(pSchema, colId);
- if (pCol == NULL) {
- return NULL; // No matched tag volumn
+ if (pTable->type == TSDB_SUPER_TABLE) {
+ if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) {
+ if (tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema) < 0) {
+ tsdbError("vgId:%d failed to update table %s tag schema since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ return -1;
+ }
+ }
+ changed = true;
}
-
- char* val = tdGetKVRowValOfCol(pTable->tagVal, colId);
- assert(type == pCol->type && bytes == pCol->bytes);
-
- if (val != NULL && IS_VAR_DATA_TYPE(type)) {
- assert(varDataLen(val) < pCol->bytes);
+
+ STSchema *pTSchema = tsdbGetTableSchema(pTable);
+ if (schemaVersion(pTSchema) < schemaVersion(pCfg->schema)) {
+ if (pTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) {
+ pTable->schema[pTable->numOfSchemas++] = tdDupSchema(pCfg->schema);
+ } else {
+ ASSERT(pTable->numOfSchemas == TSDB_MAX_TABLE_SCHEMAS);
+ STSchema *tSchema = tdDupSchema(pCfg->schema);
+ tdFreeSchema(pTable->schema[0]);
+ memmove(pTable->schema, pTable->schema + 1, sizeof(STSchema *) * (TSDB_MAX_TABLE_SCHEMAS - 1));
+ pTable->schema[pTable->numOfSchemas - 1] = tSchema;
+ }
+
+ pMeta->maxRowBytes = MAX(pMeta->maxRowBytes, dataRowMaxBytesFromSchema(pCfg->schema));
+ pMeta->maxCols = MAX(pMeta->maxCols, schemaNCols(pCfg->schema));
+
+ changed = true;
}
-
- return val;
+
+ if (changed) {
+ int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable);
+ void *buf = tsdbAllocBytes(pRepo, tlen);
+ tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable);
+ }
+
+ return 0;
}
-char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id) {
- STsdbMeta* pMeta = tsdbGetMeta(repo);
- STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
-
- if (pTable == NULL) {
- return NULL;
+int tsdbWLockRepoMeta(STsdbRepo *pRepo) {
+ int code = pthread_rwlock_wrlock(&(pRepo->tsdbMeta->rwLock));
+ if (code != 0) {
+ tsdbError("vgId:%d failed to write lock TSDB meta since %s", REPO_ID(pRepo), strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+
+ return 0;
+}
+
+int tsdbRLockRepoMeta(STsdbRepo *pRepo) {
+ int code = pthread_rwlock_rdlock(&(pRepo->tsdbMeta->rwLock));
+ if (code != 0) {
+ tsdbError("vgId:%d failed to read lock TSDB meta since %s", REPO_ID(pRepo), strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+
+ return 0;
+}
+
+int tsdbUnlockRepoMeta(STsdbRepo *pRepo) {
+ int code = pthread_rwlock_unlock(&(pRepo->tsdbMeta->rwLock));
+ if (code != 0) {
+ tsdbError("vgId:%d failed to unlock TSDB meta since %s", REPO_ID(pRepo), strerror(code));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+
+ return 0;
+}
+
+void tsdbRefTable(STable *pTable) { T_REF_INC(pTable); }
+
+void tsdbUnRefTable(STable *pTable) {
+ if (T_REF_DEC(pTable) == 0) {
+ if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) {
+ tsdbUnRefTable(pTable->pSuper);
+ }
+ tsdbFreeTable(pTable);
+ }
+}
+
+// ------------------ LOCAL FUNCTIONS ------------------
+static int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
+ if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
+ return -1;
+ } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
+ return 1;
} else {
- return (char*) pTable->name;
+ return 0;
}
}
+static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
+ STsdbRepo *pRepo = (STsdbRepo *)pHandle;
+ STable * pTable = NULL;
+
+ if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) {
+ terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+ return -1;
+ }
+
+ tsdbDecodeTable(cont, &pTable);
+
+ if (tsdbAddTableToMeta(pRepo, pTable, false) < 0) {
+ tsdbFreeTable(pTable);
+ return -1;
+ }
+
+ tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ TABLE_TID(pTable), TABLE_UID(pTable));
+ return 0;
+}
+
+static void tsdbOrgMeta(void *pHandle) {
+ STsdbRepo *pRepo = (STsdbRepo *)pHandle;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ STsdbCfg * pCfg = &pRepo->config;
+
+ for (int i = 1; i < pCfg->maxTables; i++) {
+ STable *pTable = pMeta->tables[i];
+ if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) {
+ tsdbAddTableIntoIndex(pMeta, pTable);
+ }
+ }
+}
+
+static char *getTagIndexKey(const void *pData) {
+ STable *pTable = *(STable **)pData;
+
+ STSchema *pSchema = tsdbGetTableTagSchema(pTable);
+ STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
+ void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId);
+ return res;
+}
+
static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
STable *pTable = NULL;
size_t tsize = 0;
@@ -321,19 +629,8 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
goto _err;
}
- pTable->type = pCfg->type;
- pTable->numOfSchemas = 0;
-
if (isSuper) {
pTable->type = TSDB_SUPER_TABLE;
- pTable->tableId.uid = pCfg->superUid;
- pTable->tableId.tid = -1;
- pTable->superUid = TSDB_INVALID_SUPER_TABLE_ID;
- pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
- pTable->numOfSchemas = 1;
- pTable->schema[0] = tdDupSchema(pCfg->schema);
- pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
-
tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) {
@@ -341,20 +638,30 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
goto _err;
}
STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, tsize);
-
- STColumn *pColSchema = schemaColAt(pTable->tagSchema, 0);
- pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 0,
- getTagIndexKey); // Allow duplicate key, no lock
+ TABLE_UID(pTable) = pCfg->superUid;
+ TABLE_TID(pTable) = -1;
+ TABLE_SUID(pTable) = -1;
+ pTable->pSuper = NULL;
+ pTable->numOfSchemas = 1;
+ pTable->schema[0] = tdDupSchema(pCfg->schema);
+ if (pTable->schema[0] == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
+ if (pTable->tagSchema == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ pTable->tagVal = NULL;
+ STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
+ pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
} else {
pTable->type = pCfg->type;
- pTable->tableId.uid = pCfg->tableId.uid;
- pTable->tableId.tid = pCfg->tableId.tid;
- pTable->lastKey = TSKEY_INITIAL_VAL;
-
tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) {
@@ -362,25 +669,39 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
goto _err;
}
STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, tsize);
+ TABLE_UID(pTable) = pCfg->tableId.uid;
+ TABLE_TID(pTable) = pCfg->tableId.tid;
if (pCfg->type == TSDB_CHILD_TABLE) {
- pTable->superUid = pCfg->superUid;
+ TABLE_SUID(pTable) = pCfg->superUid;
pTable->tagVal = tdKVRowDup(pCfg->tagValues);
+ if (pTable->tagVal == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
} else {
- pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
+ TABLE_SUID(pTable) = -1;
pTable->numOfSchemas = 1;
pTable->schema[0] = tdDupSchema(pCfg->schema);
+ if (pTable->schema[0] == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
- if (pCfg->type == TSDB_NORMAL_TABLE) {
- pTable->superUid = -1;
- } else {
- ASSERT(pCfg->type == TSDB_STREAM_TABLE);
- pTable->superUid = -1;
+ if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {
pTable->sql = strdup(pCfg->sql);
+ if (pTable->sql == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
}
}
+
+ pTable->lastKey = TSKEY_INITIAL_VAL;
}
+ T_REF_INC(pTable);
+
return pTable;
_err:
@@ -388,421 +709,475 @@ _err:
return NULL;
}
+static void tsdbFreeTable(STable *pTable) {
+ if (pTable) {
+ tsdbTrace("table %s is destroyed", TABLE_CHAR_NAME(pTable));
+ tfree(TABLE_NAME(pTable));
+ if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
+ for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) {
+ tdFreeSchema(pTable->schema[i]);
+ }
+
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ tdFreeSchema(pTable->tagSchema);
+ }
+ }
+
+ kvRowFree(pTable->tagVal);
+
+ tSkipListDestroy(pTable->pIndex);
+ tfree(pTable->sql);
+ free(pTable);
+ }
+}
+
static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) {
ASSERT(pTable->type == TSDB_SUPER_TABLE);
ASSERT(schemaVersion(pTable->tagSchema) < schemaVersion(newSchema));
STSchema *pOldSchema = pTable->tagSchema;
STSchema *pNewSchema = tdDupSchema(newSchema);
- if (pNewSchema == NULL) return TSDB_CODE_TDB_OUT_OF_MEMORY;
+ if (pNewSchema == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
pTable->tagSchema = pNewSchema;
tdFreeSchema(pOldSchema);
- return TSDB_CODE_SUCCESS;
+ return 0;
}
-int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) {
- ASSERT(pTable->type != TSDB_CHILD_TABLE);
- bool isChanged = false;
+static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) {
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
- if (pTable->type == TSDB_SUPER_TABLE) {
- if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) {
- int32_t code = tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema);
- if (code != TSDB_CODE_SUCCESS) return code;
- }
- isChanged = true;
+ if (addIdx && tsdbWLockRepoMeta(pRepo) < 0) {
+ tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ return -1;
}
- STSchema *pTSchema = tsdbGetTableSchema(pMeta, pTable);
- if (schemaVersion(pTSchema) < schemaVersion(pCfg->schema)) {
- if (pTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) {
- pTable->schema[pTable->numOfSchemas++] = tdDupSchema(pCfg->schema);
- } else {
- ASSERT(pTable->numOfSchemas == TSDB_MAX_TABLE_SCHEMAS);
- STSchema *tSchema = tdDupSchema(pCfg->schema);
- tdFreeSchema(pTable->schema[0]);
- memmove(pTable->schema, pTable->schema+1, sizeof(STSchema *) * (TSDB_MAX_TABLE_SCHEMAS - 1));
- pTable->schema[pTable->numOfSchemas-1] = tSchema;
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ if (tdListAppend(pMeta->superList, (void *)(&pTable)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ goto _err;
}
+ } else {
+ if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index
+ if (tsdbAddTableIntoIndex(pMeta, pTable) < 0) {
+ tsdbTrace("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), tstrerror(terrno));
+ goto _err;
+ }
+ }
+ pMeta->tables[TABLE_TID(pTable)] = pTable;
+ pMeta->nTables++;
+ }
- STSchema *lSchema = pTable->schema[pTable->numOfSchemas - 1];
- if (schemaNCols(lSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(lSchema);
- int bytes = dataRowMaxBytesFromSchema(lSchema);
- if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
-
- isChanged = true;
+ if (taosHashPut(pMeta->uidMap, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable),
+ sizeof(pTable)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), tstrerror(terrno));
+ goto _err;
}
- if (isChanged) {
- char *buf = malloc(1024 * 1024);
- int bufLen = 0;
- tsdbEncodeTable(pTable, buf, &bufLen);
- tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen);
- free(buf);
+ if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
+ STSchema *pSchema = tsdbGetTableSchema(pTable);
+ if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema);
+ if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema);
}
- return TSDB_CODE_SUCCESS;
+ if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1;
+
+ tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ TABLE_TID(pTable), TABLE_UID(pTable));
+ return 0;
+
+_err:
+ tsdbRemoveTableFromMeta(pRepo, pTable, false, false);
+ if (addIdx) tsdbUnlockRepoMeta(pRepo);
+ return -1;
}
-int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
+static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
+ SListIter lIter = {0};
+ SListNode *pNode = NULL;
+ STable * tTable = NULL;
+ STsdbCfg * pCfg = &(pRepo->config);
+
+ STSchema *pSchema = tsdbGetTableSchema(pTable);
+ int maxCols = schemaNCols(pSchema);
+ int maxRowBytes = schemaTLen(pSchema);
+
+ if (lock) tsdbWLockRepoMeta(pRepo);
+
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ tdListInitIter(pMeta->superList, &lIter, TD_LIST_BACKWARD);
+
+ while ((pNode = tdListNext(&lIter)) != NULL) {
+ tdListNodeGetData(pMeta->superList, pNode, (void *)(&tTable));
+ if (pTable == tTable) {
+ tdListPopNode(pMeta->superList, pNode);
+ free(pNode);
+ break;
+ }
+ }
+ } else {
+ pMeta->tables[pTable->tableId.tid] = NULL;
+ if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && rmFromIdx) {
+ tsdbRemoveTableFromIndex(pMeta, pTable);
+ }
- if (tsdbCheckTableCfg(pCfg) < 0) return -1;
-
- STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
- if (pTable != NULL) {
- tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
- pTable->tableId.tid, pTable->tableId.uid);
- return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ pMeta->nTables--;
}
- STable *super = NULL;
- int newSuper = 0;
+ taosHashRemove(pMeta->uidMap, (char *)(&(TABLE_UID(pTable))), sizeof(TABLE_UID(pTable)));
- if (pCfg->type == TSDB_CHILD_TABLE) {
- super = tsdbGetTableByUid(pMeta, pCfg->superUid);
- if (super == NULL) { // super table not exists, try to create it
- newSuper = 1;
- super = tsdbNewTable(pCfg, true);
- if (super == NULL) return -1;
- } else {
- if (super->type != TSDB_SUPER_TABLE) return -1;
- if (super->tableId.uid != pCfg->superUid) return -1;
- tsdbUpdateTable(pMeta, super, pCfg);
+ if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) {
+ maxCols = 0;
+ maxRowBytes = 0;
+ for (int i = 0; i < pCfg->maxTables; i++) {
+ STable *pTable = pMeta->tables[i];
+ if (pTable != NULL) {
+ pSchema = tsdbGetTableSchema(pTable);
+ maxCols = MAX(maxCols, schemaNCols(pSchema));
+ maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema));
+ }
}
}
- STable *table = tsdbNewTable(pCfg, false);
- if (table == NULL) {
- if (newSuper) {
- tsdbFreeTable(super);
- return -1;
- }
- }
-
- table->lastKey = TSKEY_INITIAL_VAL;
-
- // Register to meta
- if (newSuper) {
- tsdbAddTableToMeta(pMeta, super, true);
- tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name),
- super->tableId.uid);
- }
- tsdbAddTableToMeta(pMeta, table, true);
- tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name),
- table->tableId.tid, table->tableId.uid);
+ if (lock) tsdbUnlockRepoMeta(pRepo);
+ tsdbTrace("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+ tsdbUnRefTable(pTable);
+}
- // Write to meta file
- int bufLen = 0;
- char *buf = malloc(1024*1024);
- if (newSuper) {
- tsdbEncodeTable(super, buf, &bufLen);
- tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen);
- }
+static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
+ ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
+ STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable));
+ ASSERT(pSTable != NULL);
- tsdbEncodeTable(table, buf, &bufLen);
- tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen);
- tfree(buf);
+ pTable->pSuper = pSTable;
- return 0;
-}
+ int32_t level = 0;
+ int32_t headSize = 0;
-/**
- * Check if a table is valid to insert.
- * @return NULL for invalid and the pointer to the table if valid
- */
-STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId) {
- STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid);
- if (pTable == NULL) {
- return NULL;
+ tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
+
+ // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
+ // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
+ SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
+ if (pNode == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
}
+ pNode->level = level;
- if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) return NULL;
- if (pTable->tableId.tid != tableId.tid) return NULL;
+ memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
- return pTable;
+ tSkipListPut(pSTable->pIndex, pNode);
+ T_REF_INC(pSTable);
+ return 0;
}
-STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
- if (pMsg == NULL) return NULL;
- SSchema * pSchema = (SSchema *)pMsg->data;
- int16_t numOfCols = htons(pMsg->numOfColumns);
- int16_t numOfTags = htons(pMsg->numOfTags);
- STSchemaBuilder schemaBuilder = {0};
-
- STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg));
- if (pCfg == NULL) return NULL;
+static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
+ ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
- if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err;
- if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) goto _err;
+ STable *pSTable = pTable->pSuper;
+ ASSERT(pSTable != NULL);
- for (int i = 0; i < numOfCols; i++) {
- tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
- }
- if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err;
- if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err;
+ STSchema *pSchema = tsdbGetTableTagSchema(pTable);
+ STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
- if (numOfTags > 0) {
- // Decode tag schema
- tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion));
- for (int i = numOfCols; i < numOfCols + numOfTags; i++) {
- tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
- }
- if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err;
- if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err;
- if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err;
+ char * key = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId);
+ SArray *res = tSkipListGet(pSTable->pIndex, key);
- // Decode tag values
- if (pMsg->tagDataLen) {
- int accBytes = 0;
- char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema);
+ size_t size = taosArrayGetSize(res);
+ ASSERT(size > 0);
- SKVRowBuilder kvRowBuilder = {0};
- if (tdInitKVRowBuilder(&kvRowBuilder) < 0) goto _err;
- for (int i = numOfCols; i < numOfCols + numOfTags; i++) {
- tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes);
- accBytes += htons(pSchema[i].bytes);
- }
+ for (int32_t i = 0; i < size; ++i) {
+ SSkipListNode *pNode = taosArrayGetP(res, i);
- tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false);
- tdDestroyKVRowBuilder(&kvRowBuilder);
+ // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
+ if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
+ tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
- if (pMsg->tableType == TSDB_STREAM_TABLE) {
- char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema);
- tsdbTableSetStreamSql(pCfg, sql, true);
+ taosArrayDestroy(res);
+ return 0;
+}
+
+static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) {
+ if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) {
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_TYPE;
+ return -1;
}
- tdDestroyTSchemaBuilder(&schemaBuilder);
+ memset((void *)config, 0, sizeof(*config));
- return pCfg;
+ config->type = type;
+ config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
+ config->tableId.uid = uid;
+ config->tableId.tid = tid;
+ return 0;
+}
-_err:
- tdDestroyTSchemaBuilder(&schemaBuilder);
- tsdbClearTableCfg(pCfg);
- tfree(pCfg);
- return NULL;
+static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+ if (dup) {
+ config->schema = tdDupSchema(pSchema);
+ if (config->schema == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ } else {
+ config->schema = pSchema;
+ }
+ return 0;
}
-// int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
-int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
- STsdbRepo *pRepo = (STsdbRepo *)repo;
- if (pRepo == NULL) return -1;
+static int tsdbTableSetName(STableCfg *config, char *name, bool dup) {
+ if (dup) {
+ config->name = strdup(name);
+ if (config->name == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ } else {
+ config->name = name;
+ }
- STsdbMeta *pMeta = pRepo->tsdbMeta;
- if (pMeta == NULL) return -1;
+ return 0;
+}
- STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid);
- if (pTable == NULL) {
- tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId,
- tableId.tid, tableId.uid);
+static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+ if (config->type != TSDB_CHILD_TABLE) {
+ terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
return -1;
}
- if (pTable->cqhandle != NULL) {
- pRepo->appH.cqDropFunc(pTable->cqhandle);
+ if (dup) {
+ config->tagSchema = tdDupSchema(pSchema);
+ if (config->tagSchema == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ } else {
+ config->tagSchema = pSchema;
}
+ return 0;
+}
- tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
- tableId.tid, tableId.uid);
- if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1;
+static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) {
+ if (config->type != TSDB_CHILD_TABLE) {
+ terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
+ return -1;
+ }
+ if (dup) {
+ config->sname = strdup(sname);
+ if (config->sname == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ } else {
+ config->sname = sname;
+ }
return 0;
-
}
-// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) {
-// tSkipListPut(pTable->mem->pData, pNode);
-// return 0;
-// }
-
-static void tsdbFreeMemTable(SMemTable *pMemTable) {
- if (pMemTable) {
- tSkipListDestroy(pMemTable->pData);
+static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) {
+ if (config->type != TSDB_CHILD_TABLE || uid == TSDB_INVALID_SUPER_TABLE_ID) {
+ terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
+ return -1;
}
- free(pMemTable);
+ config->superUid = uid;
+ return 0;
}
-static int tsdbFreeTable(STable *pTable) {
- if (pTable == NULL) return 0;
+static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) {
+ if (config->type != TSDB_CHILD_TABLE) {
+ terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
+ return -1;
+ }
- if (pTable->type == TSDB_CHILD_TABLE) {
- kvRowFree(pTable->tagVal);
- } else {
- if (pTable->schema) {
- for (int i = 0; i < pTable->numOfSchemas; i++) tdFreeSchema(pTable->schema[i]);
- free(pTable->schema);
+ if (dup) {
+ config->tagValues = tdKVRowDup(row);
+ if (config->tagValues == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
}
+ } else {
+ config->tagValues = row;
}
- if (pTable->type == TSDB_STREAM_TABLE) {
- tfree(pTable->sql);
- }
+ return 0;
+}
- // Free content
- if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) {
- tdFreeSchema(pTable->tagSchema);
- tSkipListDestroy(pTable->pIndex);
+static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) {
+ if (config->type != TSDB_STREAM_TABLE) {
+ terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
+ return -1;
}
- tsdbFreeMemTable(pTable->mem);
- tsdbFreeMemTable(pTable->imem);
+ if (dup) {
+ config->sql = strdup(sql);
+ if (config->sql == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ } else {
+ config->sql = sql;
+ }
- tfree(pTable->name);
- free(pTable);
return 0;
}
-static int32_t tsdbCheckTableCfg(STableCfg *pCfg) {
- // TODO
- return 0;
+void tsdbClearTableCfg(STableCfg *config) {
+ if (config) {
+ if (config->schema) tdFreeSchema(config->schema);
+ if (config->tagSchema) tdFreeSchema(config->tagSchema);
+ if (config->tagValues) kvRowFree(config->tagValues);
+ tfree(config->name);
+ tfree(config->sname);
+ tfree(config->sql);
+ free(config);
+ }
}
-STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) {
- void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid));
+static int tsdbEncodeTableName(void **buf, tstr *name) {
+ int tlen = 0;
- if (ptr == NULL) return NULL;
+ tlen += taosEncodeFixedI16(buf, name->len);
+ if (buf != NULL) {
+ memcpy(*buf, name->data, name->len);
+ *buf = POINTER_SHIFT(*buf, name->len);
+ }
+ tlen += name->len;
- return *(STable **)ptr;
+ return tlen;
}
-static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
- STsdbRepo *pRepo = (STsdbRepo *)pMeta->pRepo;
- if (pTable->type == TSDB_SUPER_TABLE) {
- // add super table to the linked list
- if (pMeta->superList == NULL) {
- pMeta->superList = pTable;
- pTable->next = NULL;
- pTable->prev = NULL;
- } else {
- pTable->next = pMeta->superList;
- pTable->prev = NULL;
- pTable->next->prev = pTable;
- pMeta->superList = pTable;
- }
- } else {
- // add non-super table to the array
- pMeta->tables[pTable->tableId.tid] = pTable;
- if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index
- tsdbAddTableIntoIndex(pMeta, pTable);
- }
- if (pTable->type == TSDB_STREAM_TABLE && addIdx) {
- pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable));
- }
-
- pMeta->nTables++;
- }
+static void *tsdbDecodeTableName(void *buf, tstr **name) {
+ VarDataLenT len = 0;
- // Update the pMeta->maxCols and pMeta->maxRowBytes
- if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE) {
- if (schemaNCols(pTable->schema[pTable->numOfSchemas - 1]) > pMeta->maxCols)
- pMeta->maxCols = schemaNCols(pTable->schema[pTable->numOfSchemas - 1]);
- int bytes = dataRowMaxBytesFromSchema(pTable->schema[pTable->numOfSchemas - 1]);
- if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
+ buf = taosDecodeFixedI16(buf, &len);
+ *name = calloc(1, sizeof(tstr) + len + 1);
+ if (*name == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
}
+ (*name)->len = len;
+ memcpy((*name)->data, buf, len);
- if (taosHashPut(pMeta->map, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), sizeof(pTable)) < 0) {
- return -1;
- }
- return 0;
+ buf = POINTER_SHIFT(buf, len);
+ return buf;
}
-static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx) {
- if (pTable->type == TSDB_SUPER_TABLE) {
- SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex);
- while (tSkipListIterNext(pIter)) {
- STableIndexElem *pEle = (STableIndexElem *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
- STable *tTable = pEle->pTable;
+static int tsdbEncodeTable(void **buf, STable *pTable) {
+ ASSERT(pTable != NULL);
+ int tlen = 0;
- ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
+ tlen += taosEncodeFixedU8(buf, pTable->type);
+ tlen += tsdbEncodeTableName(buf, pTable->name);
+ tlen += taosEncodeFixedU64(buf, TABLE_UID(pTable));
+ tlen += taosEncodeFixedI32(buf, TABLE_TID(pTable));
- tsdbRemoveTableFromMeta(pMeta, tTable, false);
+ if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) {
+ tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable));
+ tlen += tdEncodeKVRow(buf, pTable->tagVal);
+ } else {
+ tlen += taosEncodeFixedU8(buf, pTable->numOfSchemas);
+ for (int i = 0; i < pTable->numOfSchemas; i++) {
+ tlen += tdEncodeSchema(buf, pTable->schema[i]);
}
- tSkipListDestroyIter(pIter);
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ tlen += tdEncodeSchema(buf, pTable->tagSchema);
+ }
- if (pTable->prev != NULL) {
- pTable->prev->next = pTable->next;
- if (pTable->next != NULL) {
- pTable->next->prev = pTable->prev;
- }
- } else {
- pMeta->superList = pTable->next;
+ if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {
+ tlen += taosEncodeString(buf, pTable->sql);
}
+ }
+
+ return tlen;
+}
+
+static void *tsdbDecodeTable(void *buf, STable **pRTable) {
+ STable *pTable = (STable *)calloc(1, sizeof(STable));
+ if (pTable == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
+ }
+ uint8_t type = 0;
+
+ buf = taosDecodeFixedU8(buf, &type);
+ pTable->type = type;
+ buf = tsdbDecodeTableName(buf, &(pTable->name));
+ buf = taosDecodeFixedU64(buf, &TABLE_UID(pTable));
+ buf = taosDecodeFixedI32(buf, &TABLE_TID(pTable));
+
+ if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) {
+ buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable));
+ buf = tdDecodeKVRow(buf, &(pTable->tagVal));
} else {
- pMeta->tables[pTable->tableId.tid] = NULL;
- if (pTable->type == TSDB_CHILD_TABLE && rmFromIdx) {
- tsdbRemoveTableFromIndex(pMeta, pTable);
+ buf = taosDecodeFixedU8(buf, &(pTable->numOfSchemas));
+ for (int i = 0; i < pTable->numOfSchemas; i++) {
+ buf = tdDecodeSchema(buf, &(pTable->schema[i]));
}
- if (pTable->type == TSDB_STREAM_TABLE && rmFromIdx) {
- // TODO
+
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ buf = tdDecodeSchema(buf, &(pTable->tagSchema));
+ STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
+ pTable->pIndex =
+ tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey);
+ if (pTable->pIndex == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbFreeTable(pTable);
+ return NULL;
+ }
}
- pMeta->nTables--;
+ if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {
+ buf = taosDecodeString(buf, &(pTable->sql));
+ }
}
- taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid));
- tsdbFreeTable(pTable);
- return 0;
+ T_REF_INC(pTable);
+
+ *pRTable = pTable;
+
+ return buf;
}
-int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
- assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
- STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
- assert(pSTable != NULL);
-
- int32_t level = 0;
- int32_t headSize = 0;
-
- tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
-
- // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
- // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
- SSkipListNode* pNode = calloc(1, headSize + sizeof(STableIndexElem));
- pNode->level = level;
-
- SSkipList* list = pSTable->pIndex;
- STableIndexElem* elem = (STableIndexElem*) (SL_GET_NODE_DATA(pNode));
-
- elem->pTable = pTable;
- elem->pMeta = pMeta;
-
- tSkipListPut(list, pNode);
- return 0;
+static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
+ int tlen = sizeof(SListNode) + sizeof(SActObj);
+ if (act == TSDB_UPDATE_META) tlen += (sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM));
+
+ return tlen;
}
-int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
- assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
-
- STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
- assert(pSTable != NULL);
-
- STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable);
- STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
-
- char* key = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId);
- SArray* res = tSkipListGet(pSTable->pIndex, key);
-
- size_t size = taosArrayGetSize(res);
- assert(size > 0);
-
- for(int32_t i = 0; i < size; ++i) {
- SSkipListNode* pNode = taosArrayGetP(res, i);
-
- STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
- if (pElem->pTable == pTable) { // this is the exact what we need
- tSkipListRemoveNode(pSTable->pIndex, pNode);
- }
+static void *tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable) {
+ SListNode *pNode = (SListNode *)buf;
+ SActObj * pAct = (SActObj *)(pNode->data);
+ SActCont * pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(*pAct));
+ void * pBuf = (void *)pCont;
+
+ pNode->prev = pNode->next = NULL;
+ pAct->act = act;
+ pAct->uid = TABLE_UID(pTable);
+
+ if (act == TSDB_UPDATE_META) {
+ pBuf = (void *)(pCont->cont);
+ pCont->len = tsdbEncodeTable(&pBuf, pTable) + sizeof(TSCKSUM);
+ taosCalcChecksumAppend(0, (uint8_t *)pCont->cont, pCont->len);
+ pBuf = POINTER_SHIFT(pBuf, sizeof(TSCKSUM));
}
-
- taosArrayDestroy(res);
- return 0;
-}
+ tdListAppendNode(pRepo->mem->actList, pNode);
-char *getTSTupleKey(const void * data) {
- SDataRow row = (SDataRow)data;
- return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE);
+ return pBuf;
}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c
deleted file mode 100644
index 921db8674aff5dd4463c122e277adb76d09fe9b1..0000000000000000000000000000000000000000
--- a/src/tsdb/src/tsdbMetaFile.c
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-#include "os.h"
-
-#include "taosdef.h"
-#include "hash.h"
-#include "tsdbMain.h"
-
-#define TSDB_META_FILE_VERSION_MAJOR 1
-#define TSDB_META_FILE_VERSION_MINOR 0
-#define TSDB_META_FILE_HEADER_SIZE 512
-
-typedef struct {
- int32_t offset;
- int32_t size;
- uint64_t uid;
-} SRecordInfo;
-
-// static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
-// static int32_t tsdbCheckMetaHeader(int fd);
-static int32_t tsdbWriteMetaHeader(int fd);
-static int tsdbCreateMetaFile(char *fname);
-static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh);
-
-SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH) {
- char fname[128] = "\0";
- if (tsdbGetMetaFileName(rootDir, fname) < 0) return NULL;
-
- SMetaFile *mfh = (SMetaFile *)calloc(1, sizeof(SMetaFile));
- if (mfh == NULL) return NULL;
-
- mfh->iFunc = iFunc;
- mfh->aFunc = aFunc;
- mfh->appH = appH;
- mfh->nDel = 0;
- mfh->tombSize = 0;
- mfh->size = 0;
-
- // OPEN MAP
- mfh->map =
- taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
- if (mfh->map == NULL) {
- free(mfh);
- return NULL;
- }
-
- // OPEN FILE
- if (access(fname, F_OK) < 0) { // file not exists
- mfh->fd = tsdbCreateMetaFile(fname);
- if (mfh->fd < 0) {
- taosHashCleanup(mfh->map);
- free(mfh);
- return NULL;
- }
- mfh->size += TSDB_META_FILE_HEADER_SIZE;
- } else { // file exists, recover from file
- if (tsdbRestoreFromMetaFile(fname, mfh) < 0) {
- taosHashCleanup(mfh->map);
- free(mfh);
- return NULL;
- }
- }
-
- return mfh;
-}
-
-int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) {
- if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
- return -1;
- }
-
- SRecordInfo info;
- info.offset = mfh->size;
- info.size = contLen;
- info.uid = uid;
-
- mfh->size += (contLen + sizeof(SRecordInfo));
-
- if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
- return -1;
- }
-
- // TODO: make below a function to implement
- if (lseek(mfh->fd, info.offset, SEEK_SET) < 0) {
- return -1;
- }
-
- if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
- return -1;
- }
-
- if (write(mfh->fd, cont, contLen) < 0) {
- return -1;
- }
-
- // fsync(mfh->fd);
-
- mfh->tombSize++;
-
- return 0;
-}
-
-int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid) {
- char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
- if (ptr == NULL) return -1;
-
- SRecordInfo info = *(SRecordInfo *)ptr;
-
- // Remove record from hash table
- taosHashRemove(mfh->map, (char *)(&uid), sizeof(uid));
-
- // Remove record from file
-
- info.offset = -info.offset;
- if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
- return -1;
- }
-
- if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
- return -1;
- }
-
- // fsync(mfh->fd);
-
- mfh->nDel++;
-
- return 0;
-}
-
-int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) {
- char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
- if (ptr == NULL) return -1;
-
- SRecordInfo info = *(SRecordInfo *)ptr;
- // Update the hash table
- if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
- return -1;
- }
-
- // Update record in file
- if (info.size >= contLen) { // Just update it in place
- info.size = contLen;
-
- } else { // Just append to the end of file
- info.offset = mfh->size;
- info.size = contLen;
-
- mfh->size += contLen;
- }
- if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
- return -1;
- }
-
- if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
- return -1;
- }
-
- // fsync(mfh->fd);
-
- return 0;
-}
-
-void tsdbCloseMetaFile(SMetaFile *mfh) {
- if (mfh == NULL) return;
- close(mfh->fd);
-
- taosHashCleanup(mfh->map);
- tfree(mfh);
-}
-
-int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {
- if (rootDir == NULL) return -1;
- sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME);
- return 0;
-}
-
-// static int32_t tsdbCheckMetaHeader(int fd) {
-// // TODO: write the meta file header check function
-// return 0;
-// }
-
-static int32_t tsdbWriteMetaHeader(int fd) {
- // TODO: write the meta file header to file
- char head[TSDB_META_FILE_HEADER_SIZE] = "\0";
- sprintf(head, "version: %d.%d", TSDB_META_FILE_VERSION_MAJOR, TSDB_META_FILE_VERSION_MINOR);
-
- write(fd, (void *)head, TSDB_META_FILE_HEADER_SIZE);
- return 0;
-}
-
-// static int32_t tsdbReadMetaHeader(int fd) {
-// lseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET);
-// return 0;
-// }
-
-static int tsdbCreateMetaFile(char *fname) {
- int fd = open(fname, O_RDWR | O_CREAT, 0755);
- if (fd < 0) return -1;
-
- if (tsdbWriteMetaHeader(fd) < 0) {
- close(fd);
- return -1;
- }
-
- return fd;
-}
-
-static int tsdbCheckMetaFileIntegrety(int fd) {
- // TODO
- return 0;
-}
-
-static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) {
- int fd = open(fname, O_RDWR);
- if (fd < 0) return -1;
-
- if (tsdbCheckMetaFileIntegrety(fd) < 0) {
- // TODO: decide if to auto-recover the file
- close(fd);
- return -1;
- }
-
- if (lseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) {
- // TODO: deal with the error
- close(fd);
- return -1;
- }
-
- mfh->size += TSDB_META_FILE_HEADER_SIZE;
-
- mfh->fd = fd;
-
- void *buf = NULL;
- // int buf_size = 0;
-
- SRecordInfo info;
- while (1) {
- if (read(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) == 0) break;
- if (info.offset < 0) {
- mfh->size += (info.size + sizeof(SRecordInfo));
- mfh->tombSize += (info.size + sizeof(SRecordInfo));
- lseek(mfh->fd, info.size, SEEK_CUR);
- mfh->size = mfh->size + sizeof(SRecordInfo) + info.size;
- mfh->tombSize = mfh->tombSize + sizeof(SRecordInfo) + info.size;
- } else {
- if (taosHashPut(mfh->map, (char *)(&info.uid), sizeof(info.uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
- if (buf) free(buf);
- return -1;
- }
-
- buf = realloc(buf, info.size);
- if (buf == NULL) return -1;
-
- if (read(mfh->fd, buf, info.size) < 0) {
- if (buf) free(buf);
- return -1;
- }
- (*mfh->iFunc)(mfh->appH, buf, info.size);
-
- mfh->size = mfh->size + sizeof(SRecordInfo) + info.size;
- }
-
- }
- (*mfh->aFunc)(mfh->appH);
-
- if (buf) free(buf);
-
- return 0;
-}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
index add484b5f4136b2776e47e0baa4547fd606f1705..c1bfa2939b4fe1c1990c13a58f6466d75080ba74 100644
--- a/src/tsdb/src/tsdbRWHelper.c
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -14,137 +14,47 @@
*/
#include "os.h"
-#include "tsdbMain.h"
-#include "tchecksum.h"
-#include "tscompression.h"
#include "talgo.h"
+#include "tchecksum.h"
#include "tcoding.h"
+#include "tscompression.h"
+#include "tsdbMain.h"
-// Local function definitions
-// static int tsdbCheckHelperCfg(SHelperCfg *pCfg);
-static int tsdbInitHelperFile(SRWHelper *pHelper);
-// static void tsdbClearHelperFile(SHelperFile *pHFile);
-static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
-static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
- SCompBlock *pCompBlock, bool isLast, bool isSuperBlock);
-static int compareKeyBlock(const void *arg1, const void *arg2);
-static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols);
-static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
-static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey);
-static void tsdbResetHelperBlock(SRWHelper *pHelper);
-
-// ---------- Operations on Helper File part
-static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
- memset((void *)&pHelper->files, 0, sizeof(pHelper->files));
- pHelper->files.fid = -1;
- pHelper->files.headF.fd = -1;
- pHelper->files.dataF.fd = -1;
- pHelper->files.lastF.fd = -1;
- pHelper->files.nHeadF.fd = -1;
- pHelper->files.nLastF.fd = -1;
-}
-
-static int tsdbInitHelperFile(SRWHelper *pHelper) {
- // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
- size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
- pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize);
- if (pHelper->pCompIdx == NULL) return -1;
-
- tsdbResetHelperFileImpl(pHelper);
- return 0;
-}
-
-static void tsdbDestroyHelperFile(SRWHelper *pHelper) {
- tsdbCloseHelperFile(pHelper, false);
- tzfree(pHelper->pCompIdx);
-}
-
-// ---------- Operations on Helper Table part
-static void tsdbResetHelperTableImpl(SRWHelper *pHelper) {
- memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable));
- pHelper->hasOldLastBlock = false;
-}
-
-static void tsdbResetHelperTable(SRWHelper *pHelper) {
- tsdbResetHelperBlock(pHelper);
- tsdbResetHelperTableImpl(pHelper);
- helperClearState(pHelper, (TSDB_HELPER_TABLE_SET|TSDB_HELPER_INFO_LOAD));
-}
-
-static void tsdbInitHelperTable(SRWHelper *pHelper) {
- tsdbResetHelperTableImpl(pHelper);
-}
-
-static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); }
-
-// ---------- Operations on Helper Block part
-static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) {
- tdResetDataCols(pHelper->pDataCols[0]);
- tdResetDataCols(pHelper->pDataCols[1]);
-}
-
-static void tsdbResetHelperBlock(SRWHelper *pHelper) {
- tsdbResetHelperBlockImpl(pHelper);
- // helperClearState(pHelper, TSDB_HELPER_)
-}
-
-static int tsdbInitHelperBlock(SRWHelper *pHelper) {
- pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows);
- pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows);
- if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1;
-
- tsdbResetHelperBlockImpl(pHelper);
-
- return 0;
-}
-
-static void tsdbDestroyHelperBlock(SRWHelper *pHelper) {
- tzfree(pHelper->pCompData);
- tdFreeDataCols(pHelper->pDataCols[0]);
- tdFreeDataCols(pHelper->pDataCols[1]);
-}
-
-static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) {
- if (pHelper == NULL || pRepo == NULL) return -1;
-
- memset((void *)pHelper, 0, sizeof(*pHelper));
-
- // Init global configuration
- pHelper->config.type = type;
- pHelper->config.maxTables = pRepo->config.maxTables;
- pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes;
- pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock;
- pHelper->config.maxCols = pRepo->tsdbMeta->maxCols;
- pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock;
- pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock;
- pHelper->config.compress = pRepo->config.compression;
-
- pHelper->state = TSDB_HELPER_CLEAR_STATE;
-
- // Init file part
- if (tsdbInitHelperFile(pHelper) < 0) goto _err;
-
- // Init table part
- tsdbInitHelperTable(pHelper);
-
- // Init block part
- if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
-
- pHelper->pBuffer =
- tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols +
- pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM));
- if (pHelper->pBuffer == NULL) goto _err;
-
- return 0;
-
-_err:
- tsdbDestroyHelper(pHelper);
- return -1;
-}
-
-// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------
+static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
+static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
+ SCompBlock *pCompBlock, bool isLast, bool isSuperBlock);
+static int compareKeyBlock(const void *arg1, const void *arg2);
+static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols);
+static int compTSKEY(const void *key1, const void *key2);
+static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
+static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
+static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
+static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey);
+static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
+static int tsdbInitHelperFile(SRWHelper *pHelper);
+static void tsdbDestroyHelperFile(SRWHelper *pHelper);
+static void tsdbResetHelperTableImpl(SRWHelper *pHelper);
+static void tsdbResetHelperTable(SRWHelper *pHelper);
+static void tsdbInitHelperTable(SRWHelper *pHelper);
+static void tsdbDestroyHelperTable(SRWHelper *pHelper);
+static void tsdbResetHelperBlockImpl(SRWHelper *pHelper);
+static void tsdbResetHelperBlock(SRWHelper *pHelper);
+static int tsdbInitHelperBlock(SRWHelper *pHelper);
+static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type);
+static int comparColIdCompCol(const void *arg1, const void *arg2);
+static int comparColIdDataCol(const void *arg1, const void *arg2);
+static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf);
+static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds,
+ SDataCols *pDataCols);
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
+ int maxPoints, char *buffer, int bufferSize);
+static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols);
+static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx);
+static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx);
+static void tsdbDestroyHelperBlock(SRWHelper *pHelper);
+
+// ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER);
}
@@ -180,7 +90,6 @@ void tsdbResetHelper(SRWHelper *pHelper) {
}
}
-// ------------ Operations for read/write purpose
int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
ASSERT(pHelper != NULL && pGroup != NULL);
@@ -194,33 +103,35 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD];
pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA];
pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST];
- if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) {
- char *fnameDup = strdup(pHelper->files.headF.fname);
- if (fnameDup == NULL) return -1;
- char *dataDir = dirname(fnameDup);
-
- tsdbGetFileName(dataDir, pHelper->files.fid, ".h", pHelper->files.nHeadF.fname);
- tsdbGetFileName(dataDir, pHelper->files.fid, ".l", pHelper->files.nLastF.fname);
- free((void *)fnameDup);
+ if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+ tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, pHelper->files.nHeadF.fname);
+ tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, pHelper->files.nLastF.fname);
}
// Open the files
if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err;
- if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) {
+ if (helperType(pHelper) == TSDB_WRITE_HELPER) {
if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err;
if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err;
// Create and open .h
if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1;
- // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
- if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE)
+ // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM);
+ if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo),
+ TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno));
+ errno = TAOS_SYSTEM_ERROR(errno);
goto _err;
+ }
// Create and open .l file if should
if (tsdbShouldCreateNewLast(pHelper)) {
if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err;
if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE)
- goto _err;
+ tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo),
+ TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
} else {
if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err;
@@ -231,50 +142,55 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
return tsdbLoadCompIdx(pHelper, NULL);
- _err:
+_err:
return -1;
}
int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
if (pHelper->files.headF.fd > 0) {
- fsync(pHelper->files.headF.fd);
close(pHelper->files.headF.fd);
pHelper->files.headF.fd = -1;
}
if (pHelper->files.dataF.fd > 0) {
- if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.dataF), 0);
- fsync(pHelper->files.dataF.fd);
+ if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+ tsdbUpdateFileHeader(&(pHelper->files.dataF), 0);
+ fsync(pHelper->files.dataF.fd);
+ }
close(pHelper->files.dataF.fd);
pHelper->files.dataF.fd = -1;
}
if (pHelper->files.lastF.fd > 0) {
- fsync(pHelper->files.lastF.fd);
- close(pHelper->files.lastF.fd);
+ if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+ fsync(pHelper->files.lastF.fd);
+ close(pHelper->files.lastF.fd);
+ }
pHelper->files.lastF.fd = -1;
}
- if (pHelper->files.nHeadF.fd > 0) {
- if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0);
- fsync(pHelper->files.nHeadF.fd);
- close(pHelper->files.nHeadF.fd);
- pHelper->files.nHeadF.fd = -1;
- if (hasError) {
- remove(pHelper->files.nHeadF.fname);
- } else {
- rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname);
- pHelper->files.headF.info = pHelper->files.nHeadF.info;
+ if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+ if (pHelper->files.nHeadF.fd > 0) {
+ if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0);
+ fsync(pHelper->files.nHeadF.fd);
+ close(pHelper->files.nHeadF.fd);
+ pHelper->files.nHeadF.fd = -1;
+ if (hasError) {
+ remove(pHelper->files.nHeadF.fname);
+ } else {
+ rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname);
+ pHelper->files.headF.info = pHelper->files.nHeadF.info;
+ }
}
- }
-
- if (pHelper->files.nLastF.fd > 0) {
- if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0);
- fsync(pHelper->files.nLastF.fd);
- close(pHelper->files.nLastF.fd);
- pHelper->files.nLastF.fd = -1;
- if (hasError) {
- remove(pHelper->files.nLastF.fname);
- } else {
- rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname);
- pHelper->files.lastF.info = pHelper->files.nLastF.info;
+
+ if (pHelper->files.nLastF.fd > 0) {
+ if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0);
+ fsync(pHelper->files.nLastF.fd);
+ close(pHelper->files.nLastF.fd);
+ pHelper->files.nLastF.fd = -1;
+ if (hasError) {
+ remove(pHelper->files.nLastF.fname);
+ } else {
+ rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname);
+ pHelper->files.lastF.info = pHelper->files.nLastF.info;
+ }
}
}
return 0;
@@ -289,7 +205,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
pHelper->tableInfo.tid = pTable->tableId.tid;
pHelper->tableInfo.uid = pTable->tableId.uid;
- STSchema *pSchema = tsdbGetTableSchema(pRepo->tsdbMeta, pTable);
+ STSchema *pSchema = tsdbGetTableSchema(pTable);
pHelper->tableInfo.sversion = schemaVersion(pSchema);
tdInitDataCols(pHelper->pDataCols[0], pSchema);
@@ -306,18 +222,20 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
/**
* Write part of of points from pDataCols to file
- *
+ *
* @return: number of points written to file successfully
* -1 for failure
*/
int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
- ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
+ ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
ASSERT(pDataCols->numOfRows > 0);
SCompBlock compBlock;
int rowsToWrite = 0;
TSKEY keyFirst = dataColsKeyFirst(pDataCols);
+ STsdbCfg *pCfg = &pHelper->pRepo->config;
+
ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD));
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose
@@ -331,7 +249,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
SFile *pWFile = NULL;
bool isLast = false;
- if (rowsToWrite >= pHelper->config.minRowsPerFileBlock) {
+ if (rowsToWrite >= pCfg->minRowsPerFileBlock) {
pWFile = &(pHelper->files.dataF);
} else {
isLast = true;
@@ -342,8 +260,8 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) goto _err;
} else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block
- SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks),
- pIdx->numOfBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE);
+ SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), pIdx->numOfBlocks,
+ sizeof(SCompBlock), compareKeyBlock, TD_GE);
int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks);
@@ -358,10 +276,11 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
if (rowsToWrite < 0) goto _err;
- } else { // Save as a super block in the middle
- rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1);
+ } else { // Save as a super block in the middle
+ rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst - 1);
ASSERT(rowsToWrite > 0);
- if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err;
+ if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0)
+ goto _err;
if (tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err;
}
}
@@ -374,8 +293,10 @@ _err:
}
int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
- ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
- SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+ STsdbCfg *pCfg = &pHelper->pRepo->config;
+
+ ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
+ SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
SCompBlock compBlock;
if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) {
if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1;
@@ -386,7 +307,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
if (pCompBlock->numOfSubBlocks > 1) {
if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows > 0 &&
- pHelper->pDataCols[0]->numOfRows < pHelper->config.minRowsPerFileBlock);
+ pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0)
return -1;
@@ -436,25 +357,26 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
}
int tsdbWriteCompIdx(SRWHelper *pHelper) {
- ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
+ STsdbCfg *pCfg = &pHelper->pRepo->config;
+
+ ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
if (offset < 0) return -1;
SFile *pFile = &(pHelper->files.nHeadF);
pFile->info.offset = offset;
- // TODO: change the implementation of pHelper->pBuffer
void *buf = pHelper->pBuffer;
- for (uint32_t i = 0; i < pHelper->config.maxTables; i++) {
+ for (uint32_t i = 0; i < pCfg->maxTables; i++) {
SCompIdx *pCompIdx = pHelper->pCompIdx + i;
if (pCompIdx->offset > 0) {
int drift = POINTER_DISTANCE(buf, pHelper->pBuffer);
if (tsizeof(pHelper->pBuffer) - drift < 128) {
- pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer)*2);
+ pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2);
}
buf = POINTER_SHIFT(pHelper->pBuffer, drift);
- buf = taosEncodeVariantU32(buf, i);
- buf = tsdbEncodeSCompIdx(buf, pCompIdx);
+ taosEncodeVariantU32(&buf, i);
+ tsdbEncodeSCompIdx(&buf, pCompIdx);
}
}
@@ -467,42 +389,59 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
}
int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
+ STsdbCfg *pCfg = &(pHelper->pRepo->config);
+
ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN);
if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) {
// If not load from file, just load it in object
SFile *pFile = &(pHelper->files.headF);
- int fd = pFile->fd;
+ int fd = pFile->fd;
memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx));
if (pFile->info.offset > 0) {
ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE);
- if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1;
- if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) return -1;
- if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len)
+ if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) {
+ tsdbError("vgId:%d failed to lseek file %s to %u since %s", REPO_ID(pHelper->pRepo), pFile->fname,
+ pFile->info.offset, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+ if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
+ }
+ if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
+ tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
+ pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) {
- // TODO: File is broken, try to deal with it
+ tsdbError("vgId:%d file %s SCompIdx part is corrupted. offset %u len %u", REPO_ID(pHelper->pRepo), pFile->fname,
+ pFile->info.offset, pFile->info.len);
+ terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
// Decode it
void *ptr = pHelper->pBuffer;
- while (((char *)ptr - (char *)pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) {
+ while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) {
uint32_t tid = 0;
if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1;
- ASSERT(tid > 0 && tid < pHelper->config.maxTables);
+ ASSERT(tid > 0 && tid < pCfg->maxTables);
if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1;
- ASSERT((char *)ptr - (char *)pHelper->pBuffer <= pFile->info.len - sizeof(TSCKSUM));
+ ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM));
}
- ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM)));
- if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+ if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
}
-
}
helperSetState(pHelper, TSDB_HELPER_IDX_LOAD);
@@ -582,53 +521,12 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols)
}
}
-static int comparColIdCompCol(const void *arg1, const void *arg2) {
- return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
-}
-
-static int comparColIdDataCol(const void *arg1, const void *arg2) {
- return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId;
-}
-
-static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) {
- size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols;
- if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1;
- if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1;
-
- return 0;
-}
-
-static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds,
- SDataCols *pDataCols) {
- if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1;
- int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
-
- void *ptr = NULL;
- for (int i = 0; i < numOfColIds; i++) {
- int16_t colId = colIds[i];
-
- ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), comparColIdCompCol);
- if (ptr == NULL) continue;
- SCompCol *pCompCol = (SCompCol *)ptr;
-
- ptr = bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol);
- ASSERT(ptr != NULL);
- SDataCol *pDataCol = (SDataCol *)ptr;
-
- pDataCol->len = pCompCol->len;
- if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1;
- }
-
- return 0;
-}
-
-// Load specific column data from file
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) {
SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
- ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block
+ ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block
- int numOfSubBlocks = pCompBlock->numOfSubBlocks;
+ int numOfSubBlocks = pCompBlock->numOfSubBlocks;
SCompBlock *pStartBlock =
(numOfSubBlocks == 1) ? pCompBlock : (SCompBlock *)((char *)pHelper->pCompInfo->blocks + pCompBlock->offset);
@@ -642,95 +540,6 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx,
return 0;
}
-static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
- int maxPoints, char *buffer, int bufferSize) {
- // Verify by checksum
- if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
-
- // Decode the data
- if (comp) {
- // // Need to decompress
- pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
- content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
- if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
- dataColSetOffset(pDataCol, numOfRows);
- }
- } else {
- // No need to decompress, just memcpy it
- pDataCol->len = len - sizeof(TSCKSUM);
- memcpy(pDataCol->pData, content, pDataCol->len);
- if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
- dataColSetOffset(pDataCol, numOfRows);
- }
- }
- return 0;
-}
-
-/**
- * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1)
- */
-static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
- ASSERT(pCompBlock->numOfSubBlocks <= 1);
-
- ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len);
-
- SCompData *pCompData = (SCompData *)pHelper->pBuffer;
-
- int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
- if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
- if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err;
- ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
-
- int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
- if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
-
- pDataCols->numOfRows = pCompBlock->numOfRows;
-
- // Recover the data
- int ccol = 0;
- int dcol = 0;
- while (dcol < pDataCols->numOfCols) {
- SDataCol *pDataCol = &(pDataCols->cols[dcol]);
- if (ccol >= pCompData->numOfCols) {
- // Set current column as NULL and forward
- dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
- dcol++;
- continue;
- }
-
- SCompCol *pCompCol = &(pCompData->cols[ccol]);
-
- if (pCompCol->colId == pDataCol->colId) {
- if (pCompBlock->algorithm == TWO_STAGE_COMP) {
- int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES;
- if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) {
- zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows);
- }
- pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize);
- if (pHelper->compBuffer == NULL) goto _err;
- }
- if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
- pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints,
- pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
- goto _err;
- dcol++;
- ccol++;
- } else if (pCompCol->colId < pDataCol->colId) {
- ccol++;
- } else {
- // Set current column as NULL and forward
- dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
- dcol++;
- }
- }
-
- return 0;
-
-_err:
- return -1;
-}
-
-// Load the whole block data
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target) {
// SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
@@ -754,6 +563,7 @@ _err:
return -1;
}
+// ---------------------- INTERNAL FUNCTIONS ----------------------
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
ASSERT(pHelper->files.lastF.fd > 0);
struct stat st;
@@ -762,24 +572,29 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
return false;
}
-static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock,
- bool isLast, bool isSuperBlock) {
- ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
- ASSERT(isLast ? rowsToWrite < pHelper->config.minRowsPerFileBlock : true);
-
+static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
+ SCompBlock *pCompBlock, bool isLast, bool isSuperBlock) {
+ STsdbCfg *pCfg = &(pHelper->pRepo->config);
SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
- int64_t offset = 0;
+ int64_t offset = 0;
+
+ ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pCfg->maxRowsPerFileBlock);
+ ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true);
+
offset = lseek(pFile->fd, 0, SEEK_END);
- if (offset < 0) goto _err;
+ if (offset < 0) {
+ tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
int nColsNotAllNull = 0;
for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
SDataCol *pDataCol = pDataCols->cols + ncol;
SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
- if (isNEleNull(pDataCol, rowsToWrite)) {
- // all data to commit are NULL, just ignore it
+ if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it
continue;
}
@@ -815,15 +630,18 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
- if (pHelper->config.compress) {
- if (pHelper->config.compress == TWO_STAGE_COMP) {
+ if (pCfg->compression) {
+ if (pCfg->compression == TWO_STAGE_COMP) {
pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES);
- if (pHelper->compBuffer == NULL) goto _err;
+ if (pHelper->compBuffer == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
}
pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
- (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize,
- pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer));
+ (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pCfg->compression,
+ pHelper->compBuffer, tsizeof(pHelper->compBuffer));
} else {
pCompCol->len = tlen;
memcpy(tptr, pDataCol->pData, pCompCol->len);
@@ -845,12 +663,17 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
// Write the whole block to file
- if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) goto _err;
+ if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+ tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
+ strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
// Update pCompBlock membership vairables
pCompBlock->last = isLast;
pCompBlock->offset = offset;
- pCompBlock->algorithm = pHelper->config.compress;
+ pCompBlock->algorithm = pCfg->compression;
pCompBlock->numOfRows = rowsToWrite;
pCompBlock->sversion = pHelper->tableInfo.sversion;
pCompBlock->len = (int32_t)lsize;
@@ -859,9 +682,15 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
+ tsdbTrace("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64
+ " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
+ REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset),
+ (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst,
+ pCompBlock->keyLast);
+
return 0;
- _err:
+_err:
return -1;
}
@@ -878,15 +707,11 @@ static int compareKeyBlock(const void *arg1, const void *arg2) {
return 0;
}
-// static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) {
-// return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2));
-// }
-
-// Merge the data with a block in file
static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) {
// TODO: set pHelper->hasOldBlock
int rowsWritten = 0;
SCompBlock compBlock = {0};
+ STsdbCfg * pCfg = &pHelper->pRepo->config;
ASSERT(pDataCols->numOfRows > 0);
TSKEY keyFirst = dataColsKeyFirst(pDataCols);
@@ -899,13 +724,14 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
ASSERT(keyFirst >= blockAtIdx(pHelper, blkIdx)->keyFirst);
// ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
- if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
- ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
- int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface
+ if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
+ ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock &&
+ blkIdx == pIdx->numOfBlocks - 1);
+ int defaultRowsToWrite = pCfg->maxRowsPerFileBlock * 4 / 5; // TODO: make a interface
rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows);
if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
- (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pHelper->config.minRowsPerFileBlock) &&
+ (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pCfg->minRowsPerFileBlock) &&
(pHelper->files.nLastF.fd) < 0) {
if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0)
goto _err;
@@ -918,15 +744,15 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err;
// Write
SFile *pWFile = NULL;
- bool isLast = false;
- if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.minRowsPerFileBlock) {
+ bool isLast = false;
+ if (pHelper->pDataCols[0]->numOfRows >= pCfg->minRowsPerFileBlock) {
pWFile = &(pHelper->files.dataF);
} else {
isLast = true;
pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF);
}
- if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0],
- pHelper->pDataCols[0]->numOfRows, &compBlock, isLast, true) < 0)
+ if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfRows, &compBlock,
+ isLast, true) < 0)
goto _err;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err;
}
@@ -940,9 +766,10 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
TSKEY keyLimit = (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : blockAtIdx(pHelper, blkIdx + 1)->keyFirst - 1;
// rows1: number of rows must merge in this block
- int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast);
+ int rows1 =
+ tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast);
// rows2: max number of rows the block can have more
- int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows;
+ int rows2 = pCfg->maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows;
// rows3: number of rows between this block and the next block
int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit);
@@ -950,7 +777,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
if ((rows2 >= rows1) && (blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
((!blockAtIdx(pHelper, blkIdx)->last) ||
- ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock) &&
+ ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock) &&
(pHelper->files.nLastF.fd < 0)))) {
rowsWritten = rows1;
bool isLast = false;
@@ -972,14 +799,14 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
rowsWritten = rows3;
- int iter1 = 0; // iter over pHelper->pDataCols[0]
- int iter2 = 0; // iter over pDataCols
+ int iter1 = 0; // iter over pHelper->pDataCols[0]
+ int iter2 = 0; // iter over pDataCols
int round = 0;
// tdResetDataCols(pHelper->pDataCols[1]);
while (true) {
if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) break;
tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pHelper->pDataCols[0]->numOfRows,
- pDataCols, &iter2, rowsWritten, pHelper->config.maxRowsPerFileBlock * 4 / 5);
+ pDataCols, &iter2, rowsWritten, pCfg->maxRowsPerFileBlock * 4 / 5);
ASSERT(pHelper->pDataCols[1]->numOfRows > 0);
if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1],
pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0)
@@ -997,7 +824,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
return rowsWritten;
- _err:
+_err:
return -1;
}
@@ -1012,7 +839,6 @@ static int compTSKEY(const void *key1, const void *key2) {
}
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
-
if (tsizeof((void *)pHelper->pCompInfo) <= esize) {
size_t tsize = esize + sizeof(SCompBlock) * 16;
pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize);
@@ -1058,6 +884,9 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst);
}
+ tsdbTrace("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
+ blkIdx);
+
return 0;
_err:
@@ -1075,7 +904,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
size_t spaceNeeded =
(pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock);
- if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err;
+ if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err;
pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
@@ -1092,7 +921,6 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
}
}
-
*(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock;
pSCompBlock->numOfSubBlocks++;
@@ -1112,7 +940,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
}
}
- if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len-sizeof(TSCKSUM));
+ if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM));
size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo));
if (tsize > 0) {
@@ -1141,6 +969,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast;
pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last;
+ tsdbTrace("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx);
+
return 0;
_err:
@@ -1179,6 +1009,9 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast;
pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last;
+ tsdbTrace("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
+ blkIdx);
+
return 0;
}
@@ -1206,71 +1039,295 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey)
return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1;
}
-void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) {
- buf = taosEncodeVariantU32(buf, pIdx->len);
- buf = taosEncodeVariantU32(buf, pIdx->offset);
- buf = taosEncodeFixedU8(buf, pIdx->hasLast);
- buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks);
- buf = taosEncodeFixedU64(buf, pIdx->uid);
- buf = taosEncodeFixedU64(buf, pIdx->maxKey);
+static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
+ memset((void *)&pHelper->files, 0, sizeof(pHelper->files));
+ pHelper->files.fid = -1;
+ pHelper->files.headF.fd = -1;
+ pHelper->files.dataF.fd = -1;
+ pHelper->files.lastF.fd = -1;
+ pHelper->files.nHeadF.fd = -1;
+ pHelper->files.nLastF.fd = -1;
+}
- return buf;
+static int tsdbInitHelperFile(SRWHelper *pHelper) {
+ STsdbCfg *pCfg = &pHelper->pRepo->config;
+ size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM);
+ pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize);
+ if (pHelper->pCompIdx == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ tsdbResetHelperFileImpl(pHelper);
+ return 0;
}
-void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
- uint8_t hasLast = 0;
- uint32_t numOfBlocks = 0;
- uint64_t value = 0;
+static void tsdbDestroyHelperFile(SRWHelper *pHelper) {
+ tsdbCloseHelperFile(pHelper, false);
+ tsdbResetHelperFileImpl(pHelper);
+ tzfree(pHelper->pCompIdx);
+}
- if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
- if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
- if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
- pIdx->hasLast = hasLast;
- if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
- pIdx->numOfBlocks = numOfBlocks;
- if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
- pIdx->uid = (int64_t)value;
- if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
- pIdx->maxKey = (TSKEY)value;
+// ---------- Operations on Helper Table part
+static void tsdbResetHelperTableImpl(SRWHelper *pHelper) {
+ memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable));
+ pHelper->hasOldLastBlock = false;
+}
- return buf;
+static void tsdbResetHelperTable(SRWHelper *pHelper) {
+ tsdbResetHelperBlock(pHelper);
+ tsdbResetHelperTableImpl(pHelper);
+ helperClearState(pHelper, (TSDB_HELPER_TABLE_SET | TSDB_HELPER_INFO_LOAD));
+}
+
+static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); }
+
+static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); }
+
+// ---------- Operations on Helper Block part
+static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) {
+ tdResetDataCols(pHelper->pDataCols[0]);
+ tdResetDataCols(pHelper->pDataCols[1]);
}
-int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
- char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+static void tsdbResetHelperBlock(SRWHelper *pHelper) {
+ tsdbResetHelperBlockImpl(pHelper);
+ // helperClearState(pHelper, TSDB_HELPER_)
+}
- void *pBuf = (void *)buf;
- pBuf = taosEncodeFixedU32(pBuf, version);
- pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info));
+static int tsdbInitHelperBlock(SRWHelper *pHelper) {
+ STsdbRepo *pRepo = helperRepo(pHelper);
+ STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta;
+
+ pHelper->pDataCols[0] =
+ tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
+ pHelper->pDataCols[1] =
+ tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
+ if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
- taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+ tsdbResetHelperBlockImpl(pHelper);
- if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1;
- if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1;
+ return 0;
+}
+
+static void tsdbDestroyHelperBlock(SRWHelper *pHelper) {
+ tzfree(pHelper->pCompData);
+ tdFreeDataCols(pHelper->pDataCols[0]);
+ tdFreeDataCols(pHelper->pDataCols[1]);
+}
+
+static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) {
+ STsdbCfg *pCfg = &pRepo->config;
+ memset((void *)pHelper, 0, sizeof(*pHelper));
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+ helperType(pHelper) = type;
+ helperRepo(pHelper) = pRepo;
+ helperState(pHelper) = TSDB_HELPER_CLEAR_STATE;
+
+ // Init file part
+ if (tsdbInitHelperFile(pHelper) < 0) goto _err;
+
+ // Init table part
+ tsdbInitHelperTable(pHelper);
+
+ // Init block part
+ if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
+
+ // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write
+ pHelper->pBuffer =
+ tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols +
+ pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM));
+ if (pHelper->pBuffer == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
return 0;
+
+_err:
+ tsdbDestroyHelper(pHelper);
+ return -1;
}
+static int comparColIdCompCol(const void *arg1, const void *arg2) {
+ return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
+}
+static int comparColIdDataCol(const void *arg1, const void *arg2) {
+ return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId;
+}
-void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) {
- buf = taosEncodeFixedU32(buf, pInfo->offset);
- buf = taosEncodeFixedU32(buf, pInfo->len);
- buf = taosEncodeFixedU64(buf, pInfo->size);
- buf = taosEncodeFixedU64(buf, pInfo->tombSize);
- buf = taosEncodeFixedU32(buf, pInfo->totalBlocks);
- buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
+static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) {
+ size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols;
+ if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1;
+ if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1;
- return buf;
+ return 0;
}
-void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
- buf = taosDecodeFixedU32(buf, &(pInfo->offset));
- buf = taosDecodeFixedU32(buf, &(pInfo->len));
- buf = taosDecodeFixedU64(buf, &(pInfo->size));
- buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
- buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
- buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
+static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds,
+ SDataCols *pDataCols) {
+ if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1;
+ int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
+
+ void *ptr = NULL;
+ for (int i = 0; i < numOfColIds; i++) {
+ int16_t colId = colIds[i];
+
+ ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol),
+ comparColIdCompCol);
+ if (ptr == NULL) continue;
+ SCompCol *pCompCol = (SCompCol *)ptr;
+
+ ptr =
+ bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol);
+ ASSERT(ptr != NULL);
+ SDataCol *pDataCol = (SDataCol *)ptr;
+
+ pDataCol->len = pCompCol->len;
+ if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1;
+ }
+
+ return 0;
+}
+
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
+ int maxPoints, char *buffer, int bufferSize) {
+ // Verify by checksum
+ if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
+
+ // Decode the data
+ if (comp) {
+ // // Need to decompress
+ pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
+ content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
+ if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+ dataColSetOffset(pDataCol, numOfRows);
+ }
+ } else {
+ // No need to decompress, just memcpy it
+ pDataCol->len = len - sizeof(TSCKSUM);
+ memcpy(pDataCol->pData, content, pDataCol->len);
+ if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+ dataColSetOffset(pDataCol, numOfRows);
+ }
+ }
+ return 0;
+}
+
+static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
+ ASSERT(pCompBlock->numOfSubBlocks <= 1);
+
+ ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len);
+
+ SCompData *pCompData = (SCompData *)pHelper->pBuffer;
+
+ SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF);
+
+ int fd = pFile->fd;
+ if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) {
+ tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
+ pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+ tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
+ pFile->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
+
+ int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
+ if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) {
+ tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo),
+ pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len);
+ terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+ goto _err;
+ }
+
+ pDataCols->numOfRows = pCompBlock->numOfRows;
+
+ // Recover the data
+ int ccol = 0;
+ int dcol = 0;
+ while (dcol < pDataCols->numOfCols) {
+ SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+ if (ccol >= pCompData->numOfCols) {
+ // Set current column as NULL and forward
+ dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
+ dcol++;
+ continue;
+ }
+
+ SCompCol *pCompCol = &(pCompData->cols[ccol]);
+
+ if (pCompCol->colId == pDataCol->colId) {
+ if (pCompBlock->algorithm == TWO_STAGE_COMP) {
+ int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES;
+ if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) {
+ zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows);
+ }
+ pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize);
+ if (pHelper->compBuffer == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+ if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
+ pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints,
+ pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
+ goto _err;
+ dcol++;
+ ccol++;
+ } else if (pCompCol->colId < pDataCol->colId) {
+ ccol++;
+ } else {
+ // Set current column as NULL and forward
+ dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
+ dcol++;
+ }
+ }
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) {
+ int tlen = 0;
+
+ tlen += taosEncodeVariantU32(buf, pIdx->len);
+ tlen += taosEncodeVariantU32(buf, pIdx->offset);
+ tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
+ tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
+ tlen += taosEncodeFixedU64(buf, pIdx->uid);
+ tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
+
+ return tlen;
+}
+
+static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
+ uint8_t hasLast = 0;
+ uint32_t numOfBlocks = 0;
+ uint64_t value = 0;
+
+ if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
+ if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
+ if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
+ pIdx->hasLast = hasLast;
+ if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
+ pIdx->numOfBlocks = numOfBlocks;
+ if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+ pIdx->uid = (int64_t)value;
+ if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+ pIdx->maxKey = (TSKEY)value;
return buf;
-}
\ No newline at end of file
+}
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index a62ad5bbd3bc112673a487ffd2451a152b55c47f..d1981943e6359334cd53c0d17176cc6785e3a617 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -72,9 +72,11 @@ typedef struct STableCheckInfo {
int32_t compSize;
int32_t numOfBlocks; // number of qualified data blocks not the original blocks
SDataCols* pDataCols;
-
int32_t chosen; // indicate which iterator should move forward
bool initBuf; // whether to initialize the in-memory skip list iterator or not
+ SMemTable* mem; // in-mem buffer, hold the ref count
+ SMemTable* imem; // imem buffer, hold the ref count to avoid release
+
SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator
} STableCheckInfo;
@@ -135,7 +137,7 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
pCompBlockLoadInfo->fileId = -1;
}
-TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
+TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
// todo 1. filter not exist table
// todo 2. add the reference count for each table that is involved in query
@@ -195,7 +197,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
}
}
- tsdbTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
+ tsdbTrace("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
@@ -203,7 +205,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
return (TsdbQueryHandleT) pQueryHandle;
}
-TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
pQueryHandle->type = TSDB_QUERY_TYPE_LAST;
@@ -229,7 +231,7 @@ SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle) {
return res;
}
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
@@ -247,22 +249,24 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
pCheckInfo->initBuf = true;
int32_t order = pHandle->order;
-
+
+ tsdbTakeMemSnapshot(pHandle->pTsdb, &pCheckInfo->mem, &pCheckInfo->imem);
+
// no data in buffer, abort
- if (pTable->mem == NULL && pTable->imem == NULL) {
+ if (pCheckInfo->mem == NULL && pCheckInfo->imem == NULL) {
return false;
}
assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
- if (pTable->mem) {
- pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey,
- TSDB_DATA_TYPE_TIMESTAMP, order);
+ if (pCheckInfo->mem && pCheckInfo->mem->tData[pCheckInfo->tableId.tid] != NULL) {
+ pCheckInfo->iter = tSkipListCreateIterFromVal(pCheckInfo->mem->tData[pCheckInfo->tableId.tid]->pData,
+ (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
}
- if (pTable->imem) {
- pCheckInfo->iiter = tSkipListCreateIterFromVal(pTable->imem->pData, (const char*) &pCheckInfo->lastKey,
- TSDB_DATA_TYPE_TIMESTAMP, order);
+ if (pCheckInfo->imem && pCheckInfo->imem->tData[pCheckInfo->tableId.tid] != NULL) {
+ pCheckInfo->iiter = tSkipListCreateIterFromVal(pCheckInfo->imem->tData[pCheckInfo->tableId.tid]->pData,
+ (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
}
// both iterators are NULL, no data in buffer right now
@@ -586,10 +590,12 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
if (pCheckInfo->pDataCols == NULL) {
STsdbMeta* pMeta = tsdbGetMeta(pRepo);
- pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
+ // TODO
+ pCheckInfo->pDataCols =
+ tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
}
- tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj));
+ tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj));
if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) {
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo;
@@ -792,38 +798,77 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
//data in buffer has greater timestamp, copy data in file block
- for (int32_t i = 0; i < requiredNumOfCols; ++i) {
+ int32_t i = 0, j = 0;
+ while(i < requiredNumOfCols && j < pCols->numOfCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
- int32_t bytes = pColInfo->info.bytes;
-
+
+ SDataCol* src = &pCols->cols[j];
+ if (src->colId < pColInfo->info.colId) {
+ j++;
+ continue;
+ }
+
+ int32_t bytes = pColInfo->info.bytes;
+
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
}
-
- for (int32_t j = 0; j < pCols->numOfCols; ++j) { // todo opt performance
- SDataCol* src = &pCols->cols[j];
-
- if (pColInfo->info.colId == src->colId) {
-
- if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
- memmove(pData, src->pData + bytes * start, bytes * num);
- } else { // handle the var-string
- char* dst = pData;
-
- // todo refactor, only copy one-by-one
- for (int32_t k = start; k < num + start; ++k) {
- char* p = tdGetColDataOfRow(src, k);
- memcpy(dst, p, varDataTLen(p));
- dst += bytes;
- }
+
+ if (pColInfo->info.colId == src->colId) {
+
+ if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
+ memmove(pData, src->pData + bytes * start, bytes * num);
+ } else { // handle the var-string
+ char* dst = pData;
+
+ // todo refactor, only copy one-by-one
+ for (int32_t k = start; k < num + start; ++k) {
+ char* p = tdGetColDataOfRow(src, k);
+ memcpy(dst, p, varDataTLen(p));
+ dst += bytes;
}
-
- break;
}
+
+ j++;
+ i++;
+ } else { // pColInfo->info.colId < src->colId, it is a NULL data
+ if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+ char* dst = pData;
+
+ for(int32_t k = start; k < num + start; ++k) {
+ setVardataNull(dst, pColInfo->info.type);
+ dst += bytes;
+ }
+ } else {
+ setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num);
+ }
+ i++;
}
}
+
+ while (i < requiredNumOfCols) { // the remain columns are all null data
+ SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+ if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+ pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
+ } else {
+ pData = pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
+ }
+
+ if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+ char* dst = pData;
+
+ for(int32_t k = start; k < num + start; ++k) {
+ setVardataNull(dst, pColInfo->info.type);
+ dst += pColInfo->info.bytes;
+ }
+ } else {
+ setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num);
+ }
+
+ i++;
+ }
pQueryHandle->cur.win.ekey = tsArray[end];
pQueryHandle->cur.lastKey = tsArray[end] + step;
@@ -836,7 +881,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
char* pData = NULL;
// the schema version info is embeded in SDataRow
- STSchema* pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row));
+ STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
int32_t numOfRowCols = schemaNCols(pSchema);
int32_t i = 0, j = 0;
@@ -1095,7 +1140,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
cur->rows = numOfRows;
cur->pos = pos;
- tsdbTrace("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" %p", pQueryHandle, cur->win.skey,
+ tsdbTrace("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey,
cur->win.ekey, cur->rows, pQueryHandle->qinfo);
}
@@ -1195,7 +1240,7 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset &&
pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
// todo add more information
- tsdbError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->compBlock->offset);
+ tsdbError("error in header file, two block with same offset:%" PRId64, (int64_t)pLeftBlockInfoEx->compBlock->offset);
}
return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1;
@@ -1822,8 +1867,8 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
- STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*) pNode));
- taosArrayPush(list, &elem->pTable->tableId);
+ STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
+ taosArrayPush(list, &(*pTable)->tableId);
}
tSkipListDestroyIter(iter);
@@ -1842,8 +1887,8 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) {
size_t size = taosArrayGetSize(pTableList);
for (int32_t i = 0; i < size; ++i) { // todo speedup by using reserve space.
- STableIndexElem* elem = taosArrayGet(pTableList, i);
- taosArrayPush(pRes, &elem->pTable->tableId);
+ STable* pTable = taosArrayGetP(pTableList, i);
+ taosArrayPush(pRes, &pTable->tableId);
}
}
@@ -1930,8 +1975,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
int32_t colIndex = pColIndex->colIndex;
- assert((colIndex >= 0 && colIndex < schemaNCols(pTableGroupSupp->pTagSchema)) ||
- (colIndex == TSDB_TBNAME_COLUMN_INDEX));
+ assert(colIndex >= TSDB_TBNAME_COLUMN_INDEX);
char * f1 = NULL;
char * f2 = NULL;
@@ -1950,7 +1994,20 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
f1 = tdGetKVRowValOfCol(pTable1->tagVal, pCol->colId);
f2 = tdGetKVRowValOfCol(pTable2->tagVal, pCol->colId);
}
-
+
+ // this tags value may be NULL
+ if (f1 == NULL && f2 == NULL) {
+ continue;
+ }
+
+ if (f1 == NULL) {
+ return -1;
+ }
+
+ if (f2 == NULL) {
+ return 1;
+ }
+
int32_t ret = doCompare(f1, f2, type, bytes);
if (ret == 0) {
continue;
@@ -1990,7 +2047,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTab
}
SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols,
- TsdbRepoT* tsdb) {
+ TSDB_REPO_T* tsdb) {
assert(pTableList != NULL);
SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
@@ -2008,7 +2065,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
}
taosArrayPush(pTableGroup, &sa);
- tsdbTrace("all %d tables belong to one group", size);
+ tsdbTrace("all %zu tables belong to one group", size);
} else {
STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter));
pSupp->tsdbMeta = tsdbGetMeta(tsdb);
@@ -2027,16 +2084,16 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param;
- STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+ STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL;
int8_t type = pInfo->sch.type;
if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
- val = (char*) elem->pTable->name;
+ val = (char*) pTable->name;
type = TSDB_DATA_TYPE_BINARY;
} else {
- val = tdGetKVRowValOfCol(elem->pTable->tagVal, pInfo->sch.colId);
+ val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
}
//todo :the val is possible to be null, so check it out carefully
@@ -2092,7 +2149,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
.pExtInfo = pSTable->tagSchema,
};
- SArray* pTableList = taosArrayInit(8, sizeof(STableIndexElem));
+ SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp);
tExprTreeDestroy(&pExpr, destroyHelper);
@@ -2102,7 +2159,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
return TSDB_CODE_SUCCESS;
}
-int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@@ -2112,14 +2169,14 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag
}
if (pTable->type != TSDB_SUPER_TABLE) {
- tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s",
- tsdb, uid, pTable->tableId.tid, pTable->name);
+ tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
+ pTable->name->data);
return TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
}
SArray* res = taosArrayInit(8, sizeof(STableId));
- STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pTable);
+ STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
// no tags and tbname condition, all child tables of this stable are involved
if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
@@ -2128,7 +2185,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag
pGroupInfo->numOfTables = taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
- tsdbTrace("no tbname condition or tagcond, all tables belongs to one group, numOfTables:%d", pGroupInfo->numOfTables);
+ tsdbTrace("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
} else {
// todo add error
}
@@ -2172,11 +2229,14 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag
pGroupInfo->numOfTables = taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
+ tsdbTrace("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid,
+ pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
+
taosArrayDestroy(res);
return ret;
}
-int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
if (pTable == NULL) {
return TSDB_CODE_TDB_INVALID_TABLE_ID;
@@ -2203,6 +2263,10 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
+
+ tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->mem);
+ tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->imem);
+
tSkipListDestroyIter(pTableCheckInfo->iter);
if (pTableCheckInfo->pDataCols != NULL) {
diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp
index e26caa4ae7bd28841bdcb5fd27bee1dd4cc53744..0c42ca1ee373e7ce02584ad613464cadf423f999 100644
--- a/src/tsdb/tests/tsdbTests.cpp
+++ b/src/tsdb/tests/tsdbTests.cpp
@@ -13,7 +13,7 @@ static double getCurTime() {
}
typedef struct {
- TsdbRepoT *pRepo;
+ TSDB_REPO_T *pRepo;
bool isAscend;
int tid;
uint64_t uid;
@@ -136,7 +136,7 @@ TEST(TsdbTest, createRepo) {
tsdbSetDefaultCfg(&config);
ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
- TsdbRepoT *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+ TSDB_REPO_T *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
ASSERT_NE(pRepo, nullptr);
// 2. Create a normal table
diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h
index e22c959a568a9008cf74f0c75fafda3daea35c49..907c69bf016f863e0e1a785209b30ce32c9c89b6 100644
--- a/src/util/inc/tcoding.h
+++ b/src/util/inc/tcoding.h
@@ -33,9 +33,13 @@ static const int32_t TNUMBER = 1;
#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
// ---- Fixed U8
-static FORCE_INLINE void *taosEncodeFixedU8(void *buf, uint8_t value) {
- ((uint8_t *)buf)[0] = value;
- return POINTER_SHIFT(buf, sizeof(value));
+static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) {
+ if (buf != NULL) {
+ ((uint8_t *)(*buf))[0] = value;
+ *buf = POINTER_SHIFT(*buf, sizeof(value));
+ }
+
+ return (int)sizeof(value);
}
static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) {
@@ -44,9 +48,12 @@ static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) {
}
// ---- Fixed I8
-static FORCE_INLINE void *taosEncodeFixedI8(void *buf, int8_t value) {
- ((int8_t *)buf)[0] = value;
- return POINTER_SHIFT(buf, sizeof(value));
+static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) {
+ if (buf != NULL) {
+ ((int8_t *)(*buf))[0] = value;
+ *buf = POINTER_SHIFT(*buf, sizeof(value));
+ }
+ return (int)sizeof(value);
}
static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) {
@@ -55,15 +62,18 @@ static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) {
}
// ---- Fixed U16
-static FORCE_INLINE void *taosEncodeFixedU16(void *buf, uint16_t value) {
- if (IS_LITTLE_ENDIAN()) {
- memcpy(buf, &value, sizeof(value));
- } else {
- ((uint8_t *)buf)[0] = value & 0xff;
- ((uint8_t *)buf)[1] = (value >> 8) & 0xff;
+static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) {
+ if (buf != NULL) {
+ if (IS_LITTLE_ENDIAN()) {
+ memcpy(*buf, &value, sizeof(value));
+ } else {
+ ((uint8_t *)(*buf))[0] = value & 0xff;
+ ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff;
+ }
+ *buf = POINTER_SHIFT(*buf, sizeof(value));
}
- return POINTER_SHIFT(buf, sizeof(value));
+ return (int)sizeof(value);
}
static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) {
@@ -78,7 +88,7 @@ static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) {
}
// ---- Fixed I16
-static FORCE_INLINE void *taosEncodeFixedI16(void *buf, int16_t value) {
+static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) {
return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value));
}
@@ -90,17 +100,20 @@ static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) {
}
// ---- Fixed U32
-static FORCE_INLINE void *taosEncodeFixedU32(void *buf, uint32_t value) {
- if (IS_LITTLE_ENDIAN()) {
- memcpy(buf, &value, sizeof(value));
- } else {
- ((uint8_t *)buf)[0] = value & 0xff;
- ((uint8_t *)buf)[1] = (value >> 8) & 0xff;
- ((uint8_t *)buf)[2] = (value >> 16) & 0xff;
- ((uint8_t *)buf)[3] = (value >> 24) & 0xff;
+static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) {
+ if (buf != NULL) {
+ if (IS_LITTLE_ENDIAN()) {
+ memcpy(*buf, &value, sizeof(value));
+ } else {
+ ((uint8_t *)(*buf))[0] = value & 0xff;
+ ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff;
+ ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff;
+ ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff;
+ }
+ *buf = POINTER_SHIFT(*buf, sizeof(value));
}
- return POINTER_SHIFT(buf, sizeof(value));
+ return (int)sizeof(value);
}
static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) {
@@ -117,7 +130,7 @@ static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) {
}
// ---- Fixed I32
-static FORCE_INLINE void *taosEncodeFixedI32(void *buf, int32_t value) {
+static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) {
return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value));
}
@@ -129,21 +142,25 @@ static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) {
}
// ---- Fixed U64
-static FORCE_INLINE void *taosEncodeFixedU64(void *buf, uint64_t value) {
- if (IS_LITTLE_ENDIAN()) {
- memcpy(buf, &value, sizeof(value));
- } else {
- ((uint8_t *)buf)[0] = value & 0xff;
- ((uint8_t *)buf)[1] = (value >> 8) & 0xff;
- ((uint8_t *)buf)[2] = (value >> 16) & 0xff;
- ((uint8_t *)buf)[3] = (value >> 24) & 0xff;
- ((uint8_t *)buf)[4] = (value >> 32) & 0xff;
- ((uint8_t *)buf)[5] = (value >> 40) & 0xff;
- ((uint8_t *)buf)[6] = (value >> 48) & 0xff;
- ((uint8_t *)buf)[7] = (value >> 56) & 0xff;
+static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) {
+ if (buf != NULL) {
+ if (IS_LITTLE_ENDIAN()) {
+ memcpy(*buf, &value, sizeof(value));
+ } else {
+ ((uint8_t *)(*buf))[0] = value & 0xff;
+ ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff;
+ ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff;
+ ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff;
+ ((uint8_t *)(*buf))[4] = (value >> 32) & 0xff;
+ ((uint8_t *)(*buf))[5] = (value >> 40) & 0xff;
+ ((uint8_t *)(*buf))[6] = (value >> 48) & 0xff;
+ ((uint8_t *)(*buf))[7] = (value >> 56) & 0xff;
+ }
+
+ *buf = POINTER_SHIFT(*buf, sizeof(value));
}
- return POINTER_SHIFT(buf, sizeof(value));
+ return (int)sizeof(value);
}
static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) {
@@ -164,7 +181,7 @@ static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) {
}
// ---- Fixed I64
-static FORCE_INLINE void *taosEncodeFixedI64(void *buf, int64_t value) {
+static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) {
return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value));
}
@@ -176,18 +193,21 @@ static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) {
}
// ---- Variant U16
-static FORCE_INLINE void *taosEncodeVariantU16(void *buf, uint16_t value) {
+static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) {
int i = 0;
while (value >= ENCODE_LIMIT) {
- ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT);
+ if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 3);
}
- ((uint8_t *)buf)[i] = value;
+ if (buf != NULL) {
+ ((uint8_t *)(*buf))[i] = value;
+ *buf = POINTER_SHIFT(*buf, i + 1);
+ }
- return POINTER_SHIFT(buf, i + 1);
+ return i + 1;
}
static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) {
@@ -209,7 +229,7 @@ static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) {
}
// ---- Variant I16
-static FORCE_INLINE void *taosEncodeVariantI16(void *buf, int16_t value) {
+static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) {
return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value));
}
@@ -221,18 +241,21 @@ static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) {
}
// ---- Variant U32
-static FORCE_INLINE void *taosEncodeVariantU32(void *buf, uint32_t value) {
+static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) {
int i = 0;
while (value >= ENCODE_LIMIT) {
- ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT);
+ if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 5);
}
- ((uint8_t *)buf)[i] = value;
+ if (buf != NULL) {
+ ((uint8_t *)(*buf))[i] = value;
+ *buf = POINTER_SHIFT(*buf, i + 1);
+ }
- return POINTER_SHIFT(buf, i + 1);
+ return i + 1;
}
static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) {
@@ -254,7 +277,7 @@ static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) {
}
// ---- Variant I32
-static FORCE_INLINE void *taosEncodeVariantI32(void *buf, int32_t value) {
+static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) {
return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value));
}
@@ -266,18 +289,21 @@ static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) {
}
// ---- Variant U64
-static FORCE_INLINE void *taosEncodeVariantU64(void *buf, uint64_t value) {
+static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) {
int i = 0;
while (value >= ENCODE_LIMIT) {
- ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT);
+ if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 10);
}
- ((uint8_t *)buf)[i] = value;
+ if (buf != NULL) {
+ ((uint8_t *)(*buf))[i] = value;
+ *buf = POINTER_SHIFT(*buf, i + 1);
+ }
- return POINTER_SHIFT(buf, i + 1);
+ return i + 1;
}
static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) {
@@ -299,7 +325,7 @@ static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) {
}
// ---- Variant I64
-static FORCE_INLINE void *taosEncodeVariantI64(void *buf, int64_t value) {
+static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) {
return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value));
}
@@ -311,13 +337,18 @@ static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) {
}
// ---- string
-static FORCE_INLINE void *taosEncodeString(void *buf, char *value) {
+static FORCE_INLINE int taosEncodeString(void **buf, char *value) {
+ int tlen = 0;
size_t size = strlen(value);
- buf = taosEncodeVariantU64(buf, size);
- memcpy(buf, value, size);
+ tlen += taosEncodeVariantU64(buf, size);
+ if (buf != NULL) {
+ memcpy(*buf, value, size);
+ *buf = POINTER_SHIFT(*buf, size);
+ }
+ tlen += size;
- return POINTER_SHIFT(buf, size);
+ return tlen;
}
static FORCE_INLINE void *taosDecodeString(void *buf, char **value) {
diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h
index a57d0e95cfe65048a90356677a021e698ee2daaa..346e567c41e0a209b7dde96de55faa74a704f9b3 100644
--- a/src/util/inc/tkvstore.h
+++ b/src/util/inc/tkvstore.h
@@ -25,7 +25,7 @@ typedef int (*iterFunc)(void *, void *cont, int contLen);
typedef void (*afterFunc)(void *);
typedef struct {
- int64_t size;
+ int64_t size; // including 512 bytes of header size
int64_t tombSize;
int64_t nRecords;
int64_t nDels;
diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h
index 9e4dfe45801ed448e6c55a6e69c87b22e9e77b88..a4ed9311e24bf5098f9cca8c05edd0998b8d6253 100644
--- a/src/util/inc/tlist.h
+++ b/src/util/inc/tlist.h
@@ -57,6 +57,7 @@ SListNode *tdListPopHead(SList *list);
SListNode *tdListPopTail(SList *list);
SListNode *tdListPopNode(SList *list, SListNode *node);
void tdListMove(SList *src, SList *dst);
+void tdListDiscard(SList *list);
void tdListNodeGetData(SList *list, SListNode *node, void *target);
void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction);
diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h
index 284ea924b087e992fec3eb79b52455c8b8bea8c1..4e063c17bbc0df857071d39841f4824d1cfe3883 100644
--- a/src/util/inc/tlog.h
+++ b/src/util/inc/tlog.h
@@ -32,8 +32,18 @@ int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles);
void taosCloseLog();
void taosResetLog();
-void taosPrintLog(const char *const flags, int32_t dflag, const char *const format, ...);
-void taosPrintLongString(const char *const flags, int32_t dflag, const char *const format, ...);
+void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
+#ifdef __GNUC__
+ __attribute__((format(printf, 3, 4)))
+#endif
+;
+
+void taosPrintLongString(const char * flags, int32_t dflag, const char *format, ...)
+#ifdef __GNUC__
+ __attribute__((format(printf, 3, 4)))
+#endif
+;
+
void taosDumpData(unsigned char *msg, int32_t len);
#ifdef __cplusplus
diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h
index 9483c1cc35e6d01c7b49c993d34f0c03ad950fdd..fa2517e19a16942fd874cab1fff00262c1b90722 100644
--- a/src/util/inc/tref.h
+++ b/src/util/inc/tref.h
@@ -31,7 +31,7 @@ typedef void (*_ref_fn_t)(const void* pObj);
_ref_fn_t end; \
} _ref_func = {.begin = (s), .end = (e)};
-#define T_REF_INC(x) (atomic_add_fetch_16(&((x)->_ref.val), 1));
+#define T_REF_INC(x) (atomic_add_fetch_16(&((x)->_ref.val), 1))
#define T_REF_INC_WITH_CB(x, p) \
do { \
@@ -41,7 +41,7 @@ typedef void (*_ref_fn_t)(const void* pObj);
} \
} while (0)
-#define T_REF_DEC(x) (atomic_sub_fetch_16(&((x)->_ref.val), 1));
+#define T_REF_DEC(x) (atomic_sub_fetch_16(&((x)->_ref.val), 1))
#define T_REF_DEC_WITH_CB(x, p) \
do { \
diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h
index 60a79f69afe8b7521e1869eea8ae0e96f4660487..7b0e498d766f2564b417291febb559b613fd3f1d 100644
--- a/src/util/inc/tstoken.h
+++ b/src/util/inc/tstoken.h
@@ -24,14 +24,7 @@ extern "C" {
#include "tutil.h"
#include "ttokendef.h"
-#define TK_SPACE 200
-#define TK_COMMENT 201
-#define TK_ILLEGAL 202
-#define TK_HEX 203 // hex number 0x123
-#define TK_OCT 204 // oct number
-#define TK_BIN 205 // bin format data 0b111
-#define TK_FILE 206
-#define TK_QUESTION 207 // denoting the placeholder of "?",when invoking statement bind query
+
#define TSQL_TBNAME "TBNAME"
#define TSQL_TBNAME_L "tbname"
@@ -120,6 +113,7 @@ static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) {
type = TK_FLOAT;
goto _end;
+ break;
}
case '0': {
diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h
index d38f983718cabf97f8954eb7787783055e411094..a314f0e31ddde3e1e58c9c8ba7bcfd89efe4b1d2 100644
--- a/src/util/inc/tutil.h
+++ b/src/util/inc/tutil.h
@@ -34,18 +34,19 @@ extern "C" {
#define WCHAR wchar_t
-#define tfree(x) \
- { \
- if (x) { \
- free((void*)(x)); \
- x = 0; \
- } \
+#define tfree(x) \
+ { \
+ if (x) { \
+ free((void *)(x)); \
+ x = 0; \
+ } \
}
-#define tstrncpy(dst, src, size) do { \
+#define tstrncpy(dst, src, size) \
+ do { \
strncpy((dst), (src), (size)); \
- (dst)[(size) - 1] = 0; \
-} while (0);
+ (dst)[(size)-1] = 0; \
+ } while (0);
#define tclose(x) taosCloseSocket(x)
@@ -116,6 +117,10 @@ extern "C" {
#define POW2(x) ((x) * (x))
+uint32_t taosRand(void);
+
+size_t twcslen(const wchar_t *wcs);
+
int32_t strdequote(char *src);
size_t strtrim(char *src);
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 95d0ce70817bc3a9ac8b731734ea30144c7e72ce..1de013c416ca25080bb6edaf00a93cb324bf3668 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -309,9 +309,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
}
void taosHashCleanup(SHashObj *pHashObj) {
- if (pHashObj == NULL || pHashObj->capacity <= 0) {
- return;
- }
+ if (pHashObj == NULL) return;
SHashNode *pNode, *pNext;
diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c
index b99e722262288014b4eb4cae8252235a2205c099..19225e998a1b2a5fead58b781b448ccf4b604415 100755
--- a/src/util/src/tarray.c
+++ b/src/util/src/tarray.c
@@ -95,9 +95,6 @@ void* taosArrayGetP(const SArray* pArray, size_t index) {
assert(index < pArray->size);
void* d = TARRAY_GET_ELEM(pArray, index);
- if (d == NULL) {
- return NULL;
- }
return *(void**)d;
}
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 2b6083a91ccef8ef2a08902117e8f67340ba6a37..48603a014e1479a5dfdd319f7b23f49669ee3f80 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -119,7 +119,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo
int32_t size = pNode->size;
taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize);
- uTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size);
+ uTrace("key:%s is removed from cache,total:%" PRId64 ",size:%dbytes", pNode->key, pCacheObj->totalSize, size);
if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data);
free(pNode);
}
@@ -288,7 +288,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz
if (NULL != pNode) {
pCacheObj->totalSize += pNode->size;
- uTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes",
+ uTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes",
key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize);
} else {
uError("key:%s failed to added into cache, out of memory", key);
@@ -433,7 +433,6 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
if (pCacheObj->deleting == 1) {
- taosHashDestroyIter(pIter);
break;
}
@@ -507,7 +506,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != (uint64_t)pElem->pData) {
- uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
+ uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData);
return;
}
@@ -561,7 +560,6 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
}
}
- assert(pCacheObj->numOfElemsInTrash >= 0);
__cache_unlock(pCacheObj);
}
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index f9d306e6255d9a0ad769d7f67f44b8d7ab5414c1..889d38ff207315ed489f7877e3ee9f566a5cde41 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -191,9 +191,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
break;
}
- str++;
-
- int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo);
+ int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
@@ -241,9 +239,11 @@ static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
-
+
wchar_t pattern[128] = {0};
- memcpy(pattern, varDataVal(pRight), varDataLen(pRight)/TSDB_NCHAR_SIZE);
+ assert(TSDB_PATTERN_STRING_MAX_LEN < 128);
+
+ memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
assert(varDataLen(pRight) < 128);
int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c
index e3b3d65052e4d34c9c424b565d084ae123efcac0..b085e26e07177189d2b7d350a82ef779576d3cf2 100644
--- a/src/util/src/tcompression.c
+++ b/src/util/src/tcompression.c
@@ -551,7 +551,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char
delta_of_delta = 0;
} else {
if (is_bigendian()) {
- memcpy(&dd1 + LONG_BYTES - nbytes, input + ipos, nbytes);
+ memcpy(((char *)(&dd1)) + LONG_BYTES - nbytes, input + ipos, nbytes);
} else {
memcpy(&dd1, input + ipos, nbytes);
}
@@ -576,7 +576,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char
delta_of_delta = 0;
} else {
if (is_bigendian()) {
- memcpy(&dd2 + LONG_BYTES - nbytes, input + ipos, nbytes);
+ memcpy(((char *)(&dd2)) + LONG_BYTES - nbytes, input + ipos, nbytes);
} else {
memcpy(&dd2, input + ipos, nbytes);
}
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index 543a84dc44cf3105aee4583bc31882df37702fad..bcea8d16543cb26e6b2c951c83382794f7cf291d 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -74,7 +74,7 @@ static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) {
*option = value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
- uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value,
+ uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
@@ -106,7 +106,11 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
wordexp_t full_path;
- wordexp(input_value, &full_path, 0);
+ if (0 != wordexp(input_value, &full_path, 0)) {
+ printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno));
+ wordfree(&full_path);
+ return;
+ }
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
strcpy(option, full_path.we_wordv[0]);
}
@@ -115,8 +119,13 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
struct stat dirstat;
if (stat(option, &dirstat) < 0) {
int code = mkdir(option, 0755);
- uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
- cfg->option, input_value, code);
+ if (code < 0) {
+ uError("config option:%s, input value:%s, directory not exist, create fail with return code:%d",
+ cfg->option, input_value, code);
+ } else {
+ uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
+ cfg->option, input_value, code);
+ }
}
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
@@ -240,7 +249,12 @@ void taosReadGlobalLogCfg() {
sdbDebugFlag = 135;
wordexp_t full_path;
- wordexp(configDir, &full_path, 0);
+ if ( 0 != wordexp(configDir, &full_path, 0)) {
+ printf("\nconfig file: %s wordexp fail! reason:%s\n", configDir, strerror(errno));
+ wordfree(&full_path);
+ return;
+ }
+
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) {
printf("\nconfig file: %s path overflow max len %d, all variables are set to default\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1);
diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c
index 546693c4bbc82750b7bbc1a117469fb816fd0362..3ef10b1b0dd91c52b235bcd55e388cf0b4bc434e 100644
--- a/src/util/src/tcrc32c.c
+++ b/src/util/src/tcrc32c.c
@@ -736,7 +736,7 @@ static uint32_t table[16][256] = {
0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa}
};
-
+#ifndef _TD_ARM_
static uint32_t long_shifts[4][256] = {
{0x00000000, 0xe040e0ac, 0xc56db7a9, 0x252d5705, 0x8f3719a3, 0x6f77f90f,
0x4a5aae0a, 0xaa1a4ea6, 0x1b8245b7, 0xfbc2a51b, 0xdeeff21e, 0x3eaf12b2,
@@ -1090,7 +1090,7 @@ static uint32_t short_shifts[4][256] = {
0x3c3f083d, 0x1984fde6, 0x7748e38b, 0x52f31650, 0xaad0df51, 0x8f6b2a8a,
0xe1a734e7, 0xc41cc13c, 0x140cd014, 0x31b725cf, 0x5f7b3ba2, 0x7ac0ce79,
0x82e30778, 0xa758f2a3, 0xc994ecce, 0xec2f1915}};
-
+#endif
#if 0
static uint32_t append_trivial(uint32_t crc, crc_stream input, size_t length) {
for (size_t i = 0; i < length; ++i) {
@@ -1187,13 +1187,13 @@ uint32_t crc32c_sf(uint32_t crci, crc_stream input, size_t length) {
}
return (uint32_t)crc ^ 0xffffffff;
}
-
+#ifndef _TD_ARM_
/* Apply the zeros operator table to crc. */
static uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) {
return shift_table[0][crc & 0xff] ^ shift_table[1][(crc >> 8) & 0xff] ^
shift_table[2][(crc >> 16) & 0xff] ^ shift_table[3][crc >> 24];
}
-
+#endif
/* Compute a CRC-32C. If the crc32 instruction is available, use the hardware
version. Otherwise, use the software version. */
uint32_t (*crc32c)(uint32_t crci, crc_stream bytes, size_t len) = crc32c_sf;
diff --git a/src/util/src/tdes.c b/src/util/src/tdes.c
index 3112fb411148925bf064b356df6e8dfc708ce86e..c76938d3aad47a416ee0a8e510d5ea3f7a6481c6 100644
--- a/src/util/src/tdes.c
+++ b/src/util/src/tdes.c
@@ -140,7 +140,7 @@ void print_char_as_binary(char input) {
void generate_key(unsigned char* key) {
int i;
for (i = 0; i < 8; i++) {
- key[i] = rand() % 255;
+ key[i] = taosRand() % 255;
}
}
diff --git a/src/util/src/terror.c b/src/util/src/terror.c
index c040a11362a01175a126a7d954af4a945285061d..aa6925464e12a1863b8dc510d7389ed97267d142 100644
--- a/src/util/src/terror.c
+++ b/src/util/src/terror.c
@@ -15,7 +15,6 @@
#include
#include
-#include
#include
#include
diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c
index 88cd446349a351da7855d7fcdfcd2291c830381f..ee34ca68fd47198d2006f4bf7cd7ff47ea11c760 100644
--- a/src/util/src/tkvstore.c
+++ b/src/util/src/tkvstore.c
@@ -41,7 +41,7 @@ typedef struct {
} SKVRecord;
static int tdInitKVStoreHeader(int fd, char *fname);
-static void * tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo);
+static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo);
static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
static char * tdGetKVStoreSnapshotFname(char *fdata);
@@ -49,7 +49,7 @@ static char * tdGetKVStoreNewFname(char *fdata);
static void tdFreeKVStore(SKVStore *pStore);
static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
-static void * tdEncodeKVRecord(void *buf, SKVRecord *pRecord);
+static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord);
static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord);
static int tdRestoreKVStore(SKVStore *pStore);
@@ -61,25 +61,26 @@ int tdCreateKVStore(char *fname) {
return -1;
}
- if (tdInitKVStoreHeader(fd, fname) < 0) {
- close(fd);
- return -1;
- }
+ if (tdInitKVStoreHeader(fd, fname) < 0) goto _err;
if (fsync(fd) < 0) {
uError("failed to fsync file %s since %s", fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
- close(fd);
- return -1;
+ goto _err;
}
if (close(fd) < 0) {
uError("failed to close file %s since %s", fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
+ goto _err;
}
return 0;
+
+_err:
+ if (fd > 0) close(fd);
+ remove(fname);
+ return -1;
}
int tdDestroyKVStore(char *fname) {
@@ -105,7 +106,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
goto _err;
}
- if (access(pStore->fsnap, F_OK) == 0) {
+ if (access(pStore->fsnap, F_OK) == 0) { // .snap file exists
uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
pStore->sfd = open(pStore->fsnap, O_RDONLY);
if (pStore->sfd < 0) {
@@ -114,16 +115,22 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
goto _err;
}
- if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) goto _err;
+ if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) {
+ if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err;
+ } else {
+ if (ftruncate(pStore->fd, info.size) < 0) {
+ uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
- if (ftruncate(pStore->fd, info.size) < 0) {
- uError("failed to truncate %s to " PRId64 " size since %s", pStore->fname, info.size, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
+ if (fsync(pStore->fd) < 0) {
+ uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
+ goto _err;
+ }
}
- if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
-
close(pStore->sfd);
pStore->sfd = -1;
remove(pStore->fsnap);
@@ -131,22 +138,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
- struct stat tfstat;
- if (fstat(pStore->fd, &tfstat) < 0) {
- uError("failed to fstat file %s since %s", pStore->fname, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- ASSERT(info.size == tfstat.st_size);
-
- if (lseek(pStore->fd, TD_KVSTORE_HEADER_SIZE, SEEK_SET) < 0) {
- uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- pStore->info.size += TD_KVSTORE_HEADER_SIZE;
+ pStore->info.size = TD_KVSTORE_HEADER_SIZE;
if (tdRestoreKVStore(pStore) < 0) goto _err;
@@ -230,12 +222,44 @@ _err:
}
int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) {
+ SKVRecord rInfo = {0};
+ char buf[64] = "\0";
+ char * pBuf = buf;
+
+ rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
+ if (rInfo.offset < 0) {
+ uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
+ return -1;
+ }
+
+ rInfo.uid = uid;
+ rInfo.size = contLen;
+
+ int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo);
+ ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
+ ASSERT(tlen == sizeof(SKVRecord));
+
+ if (twrite(pStore->fd, buf, tlen) < tlen) {
+ uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (twrite(pStore->fd, cont, contLen) < contLen) {
+ uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
+ return -1;
+ }
+
+ pStore->info.size += (sizeof(SKVRecord) + contLen);
SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid));
- if (pRecord != NULL) {
- pStore->info.tombSize += (pRecord->size + sizeof(SKVRecord));
+ if (pRecord != NULL) { // just to insert
+ pStore->info.tombSize += pRecord->size;
+ } else {
+ pStore->info.nRecords++;
}
- // TODO
+ taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
+
return 0;
}
@@ -243,9 +267,9 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
SKVRecord rInfo = {0};
char buf[128] = "\0";
- SKVRecord *pRecord = taosHashGet(pStore->map, &uid, sizeof(uid));
+ SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid));
if (pRecord == NULL) {
- uError("failed to drop KV store record with key " PRIu64 " since not find", uid);
+ uError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
return -1;
}
@@ -253,10 +277,11 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
rInfo.uid = pRecord->uid;
rInfo.size = pRecord->size;
- void *pBuf = tdEncodeKVRecord(buf, &rInfo);
+ void *pBuf = buf;
+ tdEncodeKVRecord(&pBuf, &rInfo);
if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
- uError("failed to write %d bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
+ uError("failed to write %" PRId64 " bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -266,6 +291,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
pStore->info.nRecords--;
pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
+ taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid));
+
return 0;
}
@@ -326,7 +353,10 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
return -1;
}
- tdEncodeStoreInfo(buf, pInfo);
+ void *pBuf = buf;
+ tdEncodeStoreInfo(&pBuf, pInfo);
+ ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
+
taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
@@ -343,13 +373,14 @@ static int tdInitKVStoreHeader(int fd, char *fname) {
return tdUpdateKVStoreHeader(fd, fname, &info);
}
-static void *tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo) {
- buf = taosEncodeVariantI64(buf, pInfo->size);
- buf = taosEncodeVariantI64(buf, pInfo->tombSize);
- buf = taosEncodeVariantI64(buf, pInfo->nRecords);
- buf = taosEncodeVariantI64(buf, pInfo->nDels);
+static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) {
+ int tlen = 0;
+ tlen += taosEncodeVariantI64(buf, pInfo->size);
+ tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
+ tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
+ tlen += taosEncodeVariantI64(buf, pInfo->nDels);
- return buf;
+ return tlen;
}
static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
@@ -362,17 +393,19 @@ static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
}
static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
- SKVStore *pStore = (SKVStore *)malloc(sizeof(SKVStore));
+ SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore));
if (pStore == NULL) goto _err;
pStore->fname = strdup(fname);
- if (pStore->map == NULL) {
+ if (pStore->fname == NULL) {
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
goto _err;
}
pStore->fsnap = tdGetKVStoreSnapshotFname(fname);
- if (pStore->fsnap == NULL) goto _err;
+ if (pStore->fsnap == NULL) {
+ goto _err;
+ }
pStore->fnew = tdGetKVStoreNewFname(fname);
if (pStore->fnew == NULL) goto _err;
@@ -428,12 +461,13 @@ static char *tdGetKVStoreNewFname(char *fdata) {
return fname;
}
-static void *tdEncodeKVRecord(void *buf, SKVRecord *pRecord) {
- buf = taosEncodeFixedU64(buf, pRecord->uid);
- buf = taosEncodeFixedI64(buf, pRecord->offset);
- buf = taosEncodeFixedI64(buf, pRecord->size);
+static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) {
+ int tlen = 0;
+ tlen += taosEncodeFixedU64(buf, pRecord->uid);
+ tlen += taosEncodeFixedI64(buf, pRecord->offset);
+ tlen += taosEncodeFixedI64(buf, pRecord->size);
- return buf;
+ return tlen;
}
static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) {
@@ -445,34 +479,36 @@ static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) {
}
static int tdRestoreKVStore(SKVStore *pStore) {
- char tbuf[128] = "\0";
- void * buf = NULL;
- int maxBufSize = 0;
- SKVRecord rInfo = {0};
+ char tbuf[128] = "\0";
+ void * buf = NULL;
+ int maxBufSize = 0;
+ SKVRecord rInfo = {0};
+ SHashMutableIterator *pIter = NULL;
ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR));
+ ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
while (true) {
ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord));
if (tsize == 0) break;
if (tsize < sizeof(SKVRecord)) {
- uError("failed to read %d bytes from file %s since %s", sizeof(SKVRecord), pStore->fname, strerror(errno));
+ uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
+ pStore->info.size, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
char *pBuf = tdDecodeKVRecord(tbuf, &rInfo);
ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord));
- ASSERT(rInfo.offset > 0 ? pStore->info.size == rInfo.offset : true);
+ ASSERT(pStore->info.size == rInfo.offset);
if (rInfo.offset < 0) {
taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid));
pStore->info.size += sizeof(SKVRecord);
pStore->info.nRecords--;
pStore->info.nDels++;
- pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) + sizeof(SKVRecord));
+ pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
} else {
- // TODO: add statistics
ASSERT(rInfo.offset > 0 && rInfo.size > 0);
if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
uError("failed to put record in KV store %s", pStore->fname);
@@ -487,6 +523,9 @@ static int tdRestoreKVStore(SKVStore *pStore) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
+
+ pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
+ pStore->info.nRecords++;
}
}
@@ -497,7 +536,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
goto _err;
}
- SHashMutableIterator *pIter = taosHashCreateIter(pStore->map);
+ pIter = taosHashCreateIter(pStore->map);
if (pIter == NULL) {
uError("failed to create hash iter while opening KV store %s", pStore->fname);
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
@@ -507,35 +546,37 @@ static int tdRestoreKVStore(SKVStore *pStore) {
while (taosHashIterNext(pIter)) {
SKVRecord *pRecord = taosHashIterGet(pIter);
- if (lseek(pStore->fd, pRecord->offset, SEEK_SET) < 0) {
- uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
+ if (lseek(pStore->fd, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
+ uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) {
- uError("failed to read %d bytes from file %s since %s", pRecord->size, pStore->fname, strerror(errno));
+ uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
+ strerror(errno), pRecord->offset);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (!taosCheckChecksumWhole((uint8_t *)buf, pRecord->size)) {
- uError("file %s has checksum error, offset " PRId64 " size %d", pStore->fname, pRecord->offset, pRecord->size);
- terrno = TSDB_CODE_COM_FILE_CORRUPTED;
- goto _err;
+ if (pStore->iFunc) {
+ if ((*pStore->iFunc)(pStore->appH, buf, pRecord->size) < 0) {
+ uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64
+ " since %s",
+ pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno));
+ goto _err;
+ }
}
-
- if (pStore->iFunc) (*pStore->iFunc)(pStore->appH, buf, pRecord->size);
}
- taosHashDestroyIter(pIter);
-
if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
+ taosHashDestroyIter(pIter);
tfree(buf);
return 0;
_err:
+ taosHashDestroyIter(pIter);
tfree(buf);
return -1;
}
\ No newline at end of file
diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c
index bdb12a59f9776eaf2924c6c08c35c1b204e2065a..f402c2307e68cee6100a28e53dce1935af44fb3f 100644
--- a/src/util/src/tlist.c
+++ b/src/util/src/tlist.c
@@ -39,8 +39,10 @@ void tdListEmpty(SList *list) {
}
void tdListFree(SList *list) {
- tdListEmpty(list);
- free(list);
+ if (list) {
+ tdListEmpty(list);
+ free(list);
+ }
}
void tdListPrependNode(SList *list, SListNode *node) {
@@ -81,7 +83,7 @@ int tdListPrepend(SList *list, void *data) {
}
int tdListAppend(SList *list, void *data) {
- SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
+ SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize);
if (node == NULL) return -1;
memcpy((void *)(node->data), data, list->eleSize);
@@ -148,6 +150,13 @@ void tdListMove(SList *src, SList *dst) {
}
}
+void tdListDiscard(SList *list) {
+ if (list) {
+ list->head = list->tail = NULL;
+ list->numOfEles = 0;
+ }
+}
+
void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); }
void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction) {
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index 01a0eabbca39258951f2e0d6c4e098c7bf009be8..599cf3262b61359b05a71087a3b6ad2dde4139cb 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -142,6 +142,10 @@ static void *taosThreadToOpenNewFile(void *param) {
umask(0);
int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+ if (fd < 0) {
+ uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno));
+ return NULL;
+ }
taosLockFile(fd);
lseek(fd, 0, SEEK_SET);
@@ -184,7 +188,7 @@ void taosResetLog() {
tsLogObj.lines = tsLogObj.maxLines + 10;
taosOpenNewLogFile();
- remove(lastName);
+ (void)remove(lastName);
uPrint("==================================");
uPrint(" reset log file ");
@@ -229,7 +233,9 @@ static void taosGetLogFileName(char *fn) {
}
}
- strcpy(tsLogObj.logName, fn);
+ if (strlen(fn) < LOG_FILE_NAME_LEN) {
+ strcpy(tsLogObj.logName, fn);
+ }
}
static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
@@ -249,15 +255,20 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
tsLogObj.fileNum = maxFileNum;
taosGetLogFileName(fn);
- strcpy(name, fn);
- strcat(name, ".0");
+ if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
+ strcpy(name, fn);
+ strcat(name, ".0");
+ }
// if none of the log files exist, open 0, if both exists, open the old one
if (stat(name, &logstat0) < 0) {
tsLogObj.flag = 0;
} else {
- strcpy(name, fn);
- strcat(name, ".1");
+ if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
+ strcpy(name, fn);
+ strcat(name, ".1");
+ }
+
if (stat(name, &logstat1) < 0) {
tsLogObj.flag = 1;
} else {
@@ -279,7 +290,10 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
// only an estimate for number of lines
struct stat filestat;
- fstat(tsLogObj.logHandle->fd, &filestat);
+ if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) {
+ printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
+ return -1;
+ }
size = (int32_t)filestat.st_size;
tsLogObj.lines = size / 60;
@@ -295,7 +309,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
return 0;
}
-void taosPrintLog(const char *const flags, int32_t dflag, const char *const format, ...) {
+void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
@@ -313,7 +327,7 @@ void taosPrintLog(const char *const flags, int32_t dflag, const char *const form
curTime = timeSecs.tv_sec;
ptm = localtime_r(&curTime, &Tm);
- len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
+ len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId());
len += sprintf(buffer + len, "%s", flags);
@@ -382,7 +396,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
return;
}
-void taosPrintLongString(const char *const flags, int32_t dflag, const char *const format, ...) {
+void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
@@ -400,7 +414,7 @@ void taosPrintLongString(const char *const flags, int32_t dflag, const char *con
curTime = timeSecs.tv_sec;
ptm = localtime_r(&curTime, &Tm);
- len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
+ len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId());
len += sprintf(buffer + len, "%s", flags);
@@ -414,8 +428,12 @@ void taosPrintLongString(const char *const flags, int32_t dflag, const char *con
buffer[len] = 0;
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
- taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
-
+ if (tsAsyncLog) {
+ taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
+ } else {
+ twrite(tsLogObj.logHandle->fd, buffer, len);
+ }
+
if (tsLogObj.maxLines > 0) {
atomic_add_fetch_32(&tsLogObj.lines, 1);
diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c
index 38d930ac4e89456aa706c87880ceb52107bb8849..ec5f90990b4d84a92f898656f835c71a976676fc 100644
--- a/src/util/src/tmem.c
+++ b/src/util/src/tmem.c
@@ -29,7 +29,7 @@ static FILE* fpAllocLog = NULL;
// memory allocator which fails randomly
extern int32_t taosGetTimestampSec();
-static int32_t startTime = INT32_MAX;;
+static int32_t startTime = INT32_MAX;
static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
if (taosGetTimestampSec() < startTime) {
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 31ed6e2f7bded579b75066e590c68b8bc8580a3e..a8d9e8d416ce9c9ee2653a665ecb080930e2c7ab 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -169,7 +169,9 @@ void taosGetNoteName(char *fn, taosNoteInfo * pNote)
}
}
- strcpy(pNote->taosNoteName, fn);
+ if (strlen(fn) < NOTE_FILE_NAME_LEN) {
+ strcpy(pNote->taosNoteName, fn);
+ }
}
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote)
@@ -182,14 +184,18 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
pNote->taosNoteFileNum = maxNoteNum;
taosGetNoteName(fn, pNote);
+ if (strlen(fn) > NOTE_FILE_NAME_LEN * 2 - 2) {
+ fprintf(stderr, "the len of file name overflow:%s\n", fn);
+ return -1;
+ }
+
strcpy(name, fn);
strcat(name, ".0");
// if none of the note files exist, open 0, if both exists, open the old one
if (stat(name, ¬estat0) < 0) {
pNote->taosNoteFlag = 0;
- }
- else {
+ } else {
strcpy(name, fn);
strcat(name, ".1");
if (stat(name, ¬estat1) < 0) {
@@ -214,7 +220,10 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
// only an estimate for number of lines
struct stat filestat;
- fstat(pNote->taosNoteFd, &filestat);
+ if (fstat(pNote->taosNoteFd, &filestat) < 0) {
+ fprintf(stderr, "failed to fstat note file:%s reason:%s\n", name, strerror(errno));
+ return -1;
+ }
size = (int)filestat.st_size;
pNote->taosNoteLines = size / 60;
@@ -226,7 +235,7 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
{
va_list argpointer;
- char buffer[MAX_NOTE_LINE_SIZE];
+ char buffer[MAX_NOTE_LINE_SIZE+2];
int len;
struct tm Tm, *ptm;
struct timeval timeSecs;
diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c
index f1f34818655bd0ca70d62f1dfe81a5fb14238a68..3a474fe6c15629953dda23fc525362037cf4221a 100644
--- a/src/util/src/tskiplist.c
+++ b/src/util/src/tskiplist.c
@@ -38,7 +38,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
const uint32_t factor = 4;
int32_t n = 1;
- while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
+ while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) {
n++;
}
@@ -219,6 +219,8 @@ void *tSkipListDestroy(SSkipList *pSkipList) {
void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) {
if (pSkipList == NULL) {
+ *level = 1;
+ *headSize = SL_NODE_HEADER_SIZE(*level);
return;
}
@@ -277,7 +279,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
pthread_rwlock_unlock(pSkipList->lock);
}
- return forward[0];
+ return NULL;
}
tSkipListDoInsert(pSkipList, forward, pNode);
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 1f4d57115b2758b9144863a44f0fc49da2c14142..e19d91837fdede534a61244e48cb4101ec6c4aca 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -222,9 +222,7 @@ int taosReadn(int fd, char *ptr, int nbytes) {
int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
int sockFd;
- int ttl = 128;
- int reuse, nocheck;
- int bufSize = 8192000;
+ int bufSize = 1024000;
uTrace("open udp socket:0x%x:%hu", ip, port);
@@ -238,31 +236,6 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
return -1;
}
- reuse = 1;
- if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
- uError("setsockopt SO_REUSEADDR failed): %d (%s)", errno, strerror(errno));
- close(sockFd);
- return -1;
- };
-
- nocheck = 1;
- if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_NO_CHECK, (void *)&nocheck, sizeof(nocheck)) < 0) {
- if (!taosSkipSocketCheck()) {
- uError("setsockopt SO_NO_CHECK failed: %d (%s)", errno, strerror(errno));
- close(sockFd);
- return -1;
- } else {
- uPrint("Skipping setsockopt SO_NO_CHECK error: %d (%s)", errno, strerror(errno));
- }
- }
-
- ttl = 128;
- if (taosSetSockOpt(sockFd, IPPROTO_IP, IP_TTL, (void *)&ttl, sizeof(ttl)) < 0) {
- uError("setsockopt IP_TTL failed: %d (%s)", errno, strerror(errno));
- close(sockFd);
- return -1;
- }
-
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the send buffer size for UDP socket\n");
close(sockFd);
@@ -278,7 +251,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
/* bind socket to local address */
if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
uError("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port);
- taosCloseSocket(sockFd);
+ close(sockFd);
return -1;
}
@@ -321,7 +294,7 @@ int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientI
if (ret != 0) {
//uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
- taosCloseSocket(sockFd);
+ close(sockFd);
sockFd = -1;
}
diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c
index 4dd6360752f4fa67d5269c67589b2b35cbd32b4a..02d72dd1f4539a012bf50b46f94b4a7647d73675 100644
--- a/src/util/src/ttime.c
+++ b/src/util/src/ttime.c
@@ -72,49 +72,6 @@ void deltaToUtcInitOnce() {
return;
}
-int64_t user_mktime(struct tm * tm)
-{
-#define TAOS_MINUTE 60
-#define TAOS_HOUR (60*TAOS_MINUTE)
-#define TAOS_DAY (24*TAOS_HOUR)
-#define TAOS_YEAR (365*TAOS_DAY)
-
-static int month[12] = {
- 0,
- TAOS_DAY*(31),
- TAOS_DAY*(31+29),
- TAOS_DAY*(31+29+31),
- TAOS_DAY*(31+29+31+30),
- TAOS_DAY*(31+29+31+30+31),
- TAOS_DAY*(31+29+31+30+31+30),
- TAOS_DAY*(31+29+31+30+31+30+31),
- TAOS_DAY*(31+29+31+30+31+30+31+31),
- TAOS_DAY*(31+29+31+30+31+30+31+31+30),
- TAOS_DAY*(31+29+31+30+31+30+31+31+30+31),
- TAOS_DAY*(31+29+31+30+31+30+31+31+30+31+30)
-};
-
- int64_t res;
- int year;
-
- year= tm->tm_year - 70;
- res= TAOS_YEAR*year + TAOS_DAY*((year+1)/4);
- res+= month[tm->tm_mon];
-
- if(tm->tm_mon > 1 && ((year+2)%4)) {
- res-= TAOS_DAY;
- }
-
- res+= TAOS_DAY*(tm->tm_mday-1);
- res+= TAOS_HOUR*tm->tm_hour;
- res+= TAOS_MINUTE*tm->tm_min;
- res+= tm->tm_sec;
-
- return res + m_deltaUtc;
-
-}
-
-
static int64_t parseFraction(char* str, char** end, int32_t timePrec);
static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec);
static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c
index cdb2c6c0beb5ea0223fb077dde20c08a5966a294..f7acc1132de674e7a33cc4e1c3de4c786c3c1612 100644
--- a/src/util/src/ttimer.c
+++ b/src/util/src/ttimer.c
@@ -139,7 +139,7 @@ static void unlockTimerList(timer_list_t* list) {
int64_t tid = taosGetPthreadId();
if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) {
assert(false);
- tmrError("%d trying to unlock a timer list not locked by current thread.", tid);
+ tmrError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid);
}
}
@@ -290,6 +290,7 @@ static void addToExpired(tmr_obj_t* head) {
SSchedMsg schedMsg;
schedMsg.fp = NULL;
schedMsg.tfp = processExpiredTimer;
+ schedMsg.msg = NULL;
schedMsg.ahandle = head;
schedMsg.thandle = NULL;
taosScheduleTask(tmrQhandle, &schedMsg);
@@ -513,14 +514,17 @@ static void taosTmrModuleInit(void) {
tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno));
return;
}
+ pthread_mutex_lock(&wheel->mutex);
wheel->nextScanAt = now + wheel->resolution;
wheel->index = 0;
wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*));
if (wheel->slots == NULL) {
tmrError("failed to allocate wheel slots");
+ pthread_mutex_unlock(&wheel->mutex);
return;
}
timerMap.size += wheel->size;
+ pthread_mutex_unlock(&wheel->mutex);
}
timerMap.count = 0;
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 92091174153d6645474f906d3cdb09243db4455f..aa5bfe322a9e0912ae13eecdaa75783505986648 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -27,6 +27,57 @@
#include "tulog.h"
#include "taoserror.h"
+
+#ifdef WINDOWS
+uint32_t taosRand(void)
+{
+ return rand();
+}
+#else
+uint32_t taosRand(void)
+{
+/*
+ int fd;
+ int seed;
+
+ fd = open("/dev/urandom", 0);
+ if (fd < 0) {
+ seed = time(0);
+ } else {
+ int len = read(fd, &seed, sizeof(seed));
+ if (len < 0) {
+ seed = time(0);
+ }
+ close(fd);
+ }
+
+ return (uint32_t)seed;
+*/
+ return rand();
+}
+#endif
+
+size_t twcslen(const wchar_t *wcs) {
+#ifdef WINDOWS
+ int *wstr = (int *)wcs;
+ if (NULL == wstr) {
+ return 0;
+ }
+
+ size_t n = 0;
+ while (1) {
+ if (0 == *wstr++) {
+ break;
+ }
+ n++;
+ }
+
+ return n;
+#else
+ return wcslen(wcs);
+#endif
+}
+
int32_t strdequote(char *z) {
if (z == NULL) {
return 0;
@@ -434,8 +485,10 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
strcpy(tmpPath, tmpDir);
strcat(tmpPath, tdengineTmpFileNamePrefix);
- strcat(tmpPath, fileNamePrefix);
- strcat(tmpPath, "-%d-%s");
+ if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+ strcat(tmpPath, fileNamePrefix);
+ strcat(tmpPath, "-%d-%s");
+ }
char rand[8] = {0};
taosRandStr(rand, tListLen(rand) - 1);
@@ -445,9 +498,9 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
void taosRandStr(char* str, int32_t size) {
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
int32_t len = 39;
-
- for(int32_t i = 0; i < size; ++i) {
- str[i] = set[rand()%len];
+
+ for (int32_t i = 0; i < size; ++i) {
+ str[i] = set[taosRand() % len];
}
}
@@ -557,7 +610,7 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
return false;
}
- int versionNumberPos[4] = {0};
+ int versionNumberPos[5] = {0};
int len = strlen(versionStr);
int dot = 0;
for (int pos = 0; pos < len && dot < 4; ++pos) {
@@ -709,7 +762,7 @@ void taosRemoveDir(char *rootDir) {
if (de->d_type & DT_DIR) {
taosRemoveDir(filename);
} else {
- remove(filename);
+ (void)remove(filename);
uPrint("file:%s is removed", filename);
}
}
@@ -718,4 +771,4 @@ void taosRemoveDir(char *rootDir) {
rmdir(rootDir);
uPrint("dir:%s is removed", rootDir);
-}
\ No newline at end of file
+}
diff --git a/src/util/tests/codingTests.cpp b/src/util/tests/codingTests.cpp
index 57e21a828c21d340ea998c3a4689ede772322f80..0cd9524646f0bae67c8e24234d1a204302f90350 100644
--- a/src/util/tests/codingTests.cpp
+++ b/src/util/tests/codingTests.cpp
@@ -9,120 +9,144 @@ static bool test_fixed_uint16(uint16_t value) {
char buf[20] = "\0";
uint16_t value_check = 0;
- void *ptr1 = taosEncodeFixedU16(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedU16(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ int tlen = taosEncodeFixedU16(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedU16(static_cast(buf), &value_check);
+
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_fixed_int16(int16_t value) {
char buf[20] = "\0";
int16_t value_check = 0;
- void *ptr1 = taosEncodeFixedI16(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedI16(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeFixedI16(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedI16(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_fixed_uint32(uint32_t value) {
char buf[20] = "\0";
uint32_t value_check = 0;
- void *ptr1 = taosEncodeFixedU32(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedU32(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeFixedU32(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedU32(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_fixed_int32(int32_t value) {
char buf[20] = "\0";
int32_t value_check = 0;
- void *ptr1 = taosEncodeFixedI32(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedI32(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ int tlen = taosEncodeFixedI32(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedI32(static_cast(buf), &value_check);
+
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_fixed_uint64(uint64_t value) {
char buf[20] = "\0";
uint64_t value_check = 0;
- void *ptr1 = taosEncodeFixedU64(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedU64(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeFixedU64(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedU64(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_fixed_int64(int64_t value) {
char buf[20] = "\0";
int64_t value_check = 0;
- void *ptr1 = taosEncodeFixedI64(static_cast(buf), value);
- void *ptr2 = taosDecodeFixedI64(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeFixedI64(static_cast(&pBuf), value);
+ void *ptr = taosDecodeFixedI64(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_uint16(uint16_t value) {
char buf[20] = "\0";
uint16_t value_check = 0;
- void *ptr1 = taosEncodeVariantU16(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantU16(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ int tlen = taosEncodeVariantU16(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantU16(static_cast(buf), &value_check);
+
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_int16(int16_t value) {
char buf[20] = "\0";
int16_t value_check = 0;
- void *ptr1 = taosEncodeVariantI16(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantI16(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeVariantI16(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantI16(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_uint32(uint32_t value) {
char buf[20] = "\0";
uint32_t value_check = 0;
- void *ptr1 = taosEncodeVariantU32(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantU32(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeVariantU32(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantU32(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_int32(int32_t value) {
char buf[20] = "\0";
int32_t value_check = 0;
- void *ptr1 = taosEncodeVariantI32(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantI32(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ int tlen = taosEncodeVariantI32(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantI32(static_cast(buf), &value_check);
+
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_uint64(uint64_t value) {
char buf[20] = "\0";
uint64_t value_check = 0;
- void *ptr1 = taosEncodeVariantU64(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantU64(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeVariantU64(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantU64(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
static bool test_variant_int64(int64_t value) {
char buf[20] = "\0";
int64_t value_check = 0;
- void *ptr1 = taosEncodeVariantI64(static_cast(buf), value);
- void *ptr2 = taosDecodeVariantI64(static_cast(buf), &value_check);
+ void *pBuf = (void *)buf;
+
+ int tlen = taosEncodeVariantI64(static_cast(&pBuf), value);
+ void *ptr = taosDecodeVariantI64(static_cast(buf), &value_check);
- return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2));
+ return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen);
}
TEST(codingTest, fixed_encode_decode) {
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index ab74e329e6dd2585cdf0634e5112b087c0aa6ffc..d5a7dbfd2e970269f5bcf2ad57015c068d96a44e 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -37,6 +37,7 @@ typedef struct {
int32_t refCount; // reference count
int status;
int8_t role;
+ int8_t accessState;
int64_t version; // current version
int64_t fversion; // version on saved data file
void *wqueue;
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index fef0b071a632d08e9f9068539ec9ff138d48b80a..6c8f23b424983ebeca2f688961b6e4811bd96a23 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -119,11 +119,11 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
- tsdbCfg.compression = pVnodeCfg->cfg.compression;;
+ tsdbCfg.compression = pVnodeCfg->cfg.compression;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
- code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL);
+ code = tsdbCreateRepo(tsdbDir, &tsdbCfg);
if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code));
return TSDB_CODE_VND_INIT_FAILED;
@@ -195,6 +195,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->version = 0;
pVnode->tsdbCfg.tsdbId = pVnode->vgId;
pVnode->rootDir = strdup(rootDir);
+ pVnode->accessState = TSDB_VN_ALL_ACCCESS;
int32_t code = vnodeReadCfg(pVnode);
if (code != TSDB_CODE_SUCCESS) {
@@ -431,6 +432,20 @@ void vnodeBuildStatusMsg(void *param) {
taosHashDestroyIter(pIter);
}
+void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) {
+ for (int32_t i = 0; i < numOfVnodes; ++i) {
+ pAccess[i].vgId = htonl(pAccess[i].vgId);
+ SVnodeObj *pVnode = vnodeAccquireVnode(pAccess[i].vgId);
+ if (pVnode != NULL) {
+ pVnode->accessState = pAccess[i].accessState;
+ if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) {
+ vTrace("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState)
+ }
+ vnodeRelease(pVnode);
+ }
+ }
+}
+
static void vnodeCleanUp(SVnodeObj *pVnode) {
// remove from hash, so new messages wont be consumed
taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
@@ -719,7 +734,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
if (!quorum || quorum->type != cJSON_Number) {
- vError("failed to read vnode cfg, quorum not found", pVnode->vgId);
+ vError("vgId: %d, failed to read vnode cfg, quorum not found", pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.quorum = (int8_t)quorum->valueint;
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 1e770d8d277db74194b134d2cab35a59ef7eff22..91c6d28e22e1a918bfee95c5ee384cfef2fd54a2 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -86,10 +86,13 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
- vWarn("QInfo:%p connection %p broken, kill query", killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+ vWarn("QInfo:%p connection %p broken, kill query", (void*)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
- qKillQuery((qinfo_t) killQueryMsg->qhandle);
+ // this message arrived here by means of the query message, so release the vnode is necessary
+ qKillQuery((qinfo_t) killQueryMsg->qhandle, vnodeRelease, pVnode);
+ vnodeRelease(pVnode);
+
return TSDB_CODE_TSC_QUERY_CANCELLED; // todo change the error code
}
@@ -112,8 +115,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
//NOTE: there two refcount, needs to kill twice, todo refactor
- qKillQuery(pQInfo);
- qKillQuery(pQInfo);
+ qKillQuery(pQInfo, vnodeRelease, pVnode);
+ qKillQuery(pQInfo, vnodeRelease, pVnode);
return pRsp->code;
}
@@ -128,7 +131,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (pQInfo != NULL) {
vTrace("vgId:%d, QInfo:%p, do qTableQuery", pVnode->vgId, pQInfo);
- qTableQuery(pQInfo); // do execute query
+ qTableQuery(pQInfo, vnodeRelease, pVnode); // do execute query
}
return code;
@@ -146,7 +149,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (pRetrieve->free == 1) {
vTrace("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pQInfo);
- int32_t ret = qKillQuery(pQInfo);
+ int32_t ret = qKillQuery(pQInfo, vnodeRelease, pVnode);
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
pRet->len = sizeof(SRetrieveTableRsp);
@@ -175,8 +178,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
pRet->qhandle = pQInfo;
code = TSDB_CODE_VND_ACTION_NEED_REPROCESSED;
} else { // no further execution invoked, release the ref to vnode
- qDestroyQueryInfo(pQInfo);
- vnodeRelease(pVnode);
+ qDestroyQueryInfo(pQInfo, vnodeRelease, pVnode);
}
}
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index fa322757e243dd87c090467e02211ed61461aeef..769ef03bf6b0806b5699d51e26b8eb1a71e83ff0 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -53,6 +53,10 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL)
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
+ if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
+ return TSDB_CODE_VND_NO_WRITE_AUTH;
+ }
+
if (pHead->version == 0) { // from client or CQ
if (pVnode->status != TAOS_VN_STATUS_READY)
return TSDB_CODE_VND_INVALID_VGROUP_ID; // it may be in deleting or closing state
@@ -87,7 +91,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
}
static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
- int32_t code = 0;
+ int32_t code = TSDB_CODE_SUCCESS;
// save insert result into item
@@ -96,7 +100,7 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
pRet->len = sizeof(SShellSubmitRspMsg);
pRet->rsp = rpcMallocCont(pRet->len);
SShellSubmitRspMsg *pRsp = pRet->rsp;
- code = tsdbInsertData(pVnode->tsdb, pCont, pRsp);
+ if (tsdbInsertData(pVnode->tsdb, pCont, pRsp) < 0) code = terrno;
pRsp->numOfFailedBlocks = 0; //TODO
//pRet->len += pRsp->numOfFailedBlocks * sizeof(SShellSubmitRspBlock); //TODO
pRsp->code = 0;
@@ -106,10 +110,11 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
}
static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
+ int code = TSDB_CODE_SUCCESS;
STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
if (pCfg == NULL) return terrno;
- int32_t code = tsdbCreateTable(pVnode->tsdb, pCfg);
+ if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
tsdbClearTableCfg(pCfg);
return code;
@@ -117,47 +122,47 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
SMDDropTableMsg *pTable = pCont;
- int32_t code = 0;
+ int32_t code = TSDB_CODE_SUCCESS;
vTrace("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableId);
- STableId tableId = {
- .uid = htobe64(pTable->uid),
- .tid = htonl(pTable->sid)
- };
+ STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->sid)};
- code = tsdbDropTable(pVnode->tsdb, tableId);
+ if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno;
return code;
}
static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
- STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
- if (pCfg == NULL) return terrno;
- int32_t code = tsdbAlterTable(pVnode->tsdb, pCfg);
- tsdbClearTableCfg(pCfg);
- return code;
+ // TODO: disposed in tsdb
+ // STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
+ // if (pCfg == NULL) return terrno;
+ // if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
+
+ // tsdbClearTableCfg(pCfg);
+ vTrace("vgId:%d, alter table msg is received", pVnode->vgId);
+ return TSDB_CODE_SUCCESS;
}
static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
SMDDropSTableMsg *pTable = pCont;
- int32_t code = 0;
+ int32_t code = TSDB_CODE_SUCCESS;
vTrace("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId);
-
- STableId stableId = {
- .uid = htobe64(pTable->uid),
- .tid = -1
- };
- code = tsdbDropTable(pVnode->tsdb, stableId);
-
+ STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1};
+
+ if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno;
+
vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code));
-
+
return code;
}
static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
- return tsdbUpdateTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont);
+ if (tsdbUpdateTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont) < 0) {
+ return terrno;
+ }
+ return TSDB_CODE_SUCCESS;
}
int vnodeWriteToQueue(void *param, void *data, int type) {
diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
index 6c37746d24dedbfe835fb29eb95d73d7782349cb..f6728359e5e58303d48b21c71325d9c0d5c6fe89 100644
--- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
+++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
@@ -118,7 +118,7 @@
com.fasterxml.jackson.core
jackson-databind
- 2.9.10.4
+ 2.10.0.pr1
diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c
index 34c785a271c4217f89803539b3f1cb6aa3ca1111..5030af4ad61c11dff0ae786cf18d507ffac1a3cf 100644
--- a/tests/examples/c/demo.c
+++ b/tests/examples/c/demo.c
@@ -13,145 +13,55 @@
* along with this program. If not, see .
*/
-// TAOS standard API example. The same syntax as MySQL, but only a subet
+// TAOS standard API example. The same syntax as MySQL, but only a subset
// to compile: gcc -o demo demo.c -ltaos
-#include
#include
#include
#include
#include // TAOS header file
-#include
-#include
-
-static int32_t doQuery(TAOS* taos, const char* sql) {
- struct timeval t1 = {0};
- gettimeofday(&t1, NULL);
-
- TAOS_RES* res = taos_query(taos, sql);
- if (taos_errno(res) != 0) {
- printf("failed to execute query, reason:%s\n", taos_errstr(taos));
- return -1;
- }
-
- TAOS_ROW row = NULL;
- char buf[512] = {0};
-
- int32_t numOfFields = taos_num_fields(res);
- TAOS_FIELD* pFields = taos_fetch_fields(res);
-
- int32_t i = 0;
- while((row = taos_fetch_row(res)) != NULL) {
- taos_print_row(buf, row, pFields, numOfFields);
- printf("%d:%s\n", ++i, buf);
- memset(buf, 0, 512);
- }
-
- taos_free_result(res);
-
- struct timeval t2 = {0};
- gettimeofday(&t2, NULL);
-
- printf("elapsed time:%"PRId64 " ms\n", ((t2.tv_sec*1000000 + t2.tv_usec) - (t1.tv_sec*1000000 + t1.tv_usec))/1000);
- return 0;
-}
-
-void* oneLoader(void* param) {
- TAOS* conn = (TAOS*) param;
-
- for(int32_t i = 0; i < 20000; ++i) {
-// doQuery(conn, "show databases");
- doQuery(conn, "use test");
-// doQuery(conn, "describe t12");
-// doQuery(conn, "show tables");
-// doQuery(conn, "create table if not exists abc (ts timestamp, k int)");
-// doQuery(conn, "select * from t12");
- }
-
- return 0;
-}
-
-
-static __attribute__((unused)) void multiThreadTest(int32_t numOfThreads, void* conn) {
- pthread_attr_t thattr;
- pthread_attr_init(&thattr);
- pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
-
- pthread_t* threadId = (pthread_t*)malloc(sizeof(pthread_t)*(uint32_t)numOfThreads);
-
- for (int i = 0; i < numOfThreads; ++i) {
- pthread_create(&threadId[i], NULL, oneLoader, conn);
- }
-
- for (int32_t i = 0; i < numOfThreads; ++i) {
- pthread_join(threadId[i], NULL);
- }
-
- free(threadId);
- pthread_attr_destroy(&thattr);
-}
int main(int argc, char *argv[]) {
TAOS * taos;
char qstr[1024];
TAOS_RES *result;
-
+
// connect to server
if (argc < 2) {
printf("please input server-ip \n");
return 0;
}
-
- taos_options(TSDB_OPTION_CONFIGDIR, "~/sec/cfg");
-
+
// init TAOS
taos_init();
-
+
taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
if (taos == NULL) {
- printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
+ printf("failed to connect to server, reason:%s\n", taos_errstr(taos));
exit(1);
}
-
printf("success to connect to server\n");
-// doQuery(taos, "select c1,count(*) from group_db0.group_mt0 where c1<8 group by c1");
-// doQuery(taos, "select * from test.m1");
-
-// multiThreadTest(1, taos);
-// doQuery(taos, "select tbname from test.m1");
-// doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0') interval(1s) group by t1");
-// doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0', 'lm2_tb1', 'lm2_tb2') interval(1s)");
- for(int32_t i = 0; i < 200; ++i) {
- doQuery(taos, "select * from lm2_db0.lm2_stb0");
- }
-// doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))");
-
- taos_close(taos);
- return 0;
-
+
+
taos_query(taos, "drop database demo");
if (taos_query(taos, "create database demo") != 0) {
printf("failed to create database, reason:%s\n", taos_errstr(taos));
exit(1);
}
printf("success to create database\n");
-
-
+
taos_query(taos, "use demo");
-
-
+
// create table
if (taos_query(taos, "create table m1 (ts timestamp, speed int)") != 0) {
printf("failed to create table, reason:%s\n", taos_errstr(taos));
exit(1);
}
printf("success to create table\n");
-
-
+
// sleep for one second to make sure table is created on data node
// taosMsleep(1000);
-
-
+
// insert 10 records
int i = 0;
for (i = 0; i < 10; ++i) {
@@ -162,29 +72,21 @@ int main(int argc, char *argv[]) {
//sleep(1);
}
printf("success to insert rows, total %d rows\n", i);
-
-
+
// query the records
sprintf(qstr, "SELECT * FROM m1");
- if (taos_query(taos, qstr) != 0) {
- printf("failed to select, reason:%s\n", taos_errstr(taos));
- exit(1);
- }
-
- if (result == NULL) {
- printf("failed to get result, reason:%s\n", taos_errstr(taos));
+ result = taos_query(taos, qstr);
+ if (result == NULL || taos_errno(result) != 0) {
+ printf("failed to select, reason:%s\n", taos_errstr(result));
exit(1);
}
-// TAOS_ROW row;
-
TAOS_ROW row;
int rows = 0;
int num_fields = taos_field_count(taos);
TAOS_FIELD *fields = taos_fetch_fields(result);
char temp[256];
-
-
+
printf("select * from table, result:\n");
// fetch the records row by row
while ((row = taos_fetch_row(result))) {
@@ -192,7 +94,7 @@ int main(int argc, char *argv[]) {
taos_print_row(temp, row, fields, num_fields);
printf("%s\n", temp);
}
-
+
taos_free_result(result);
printf("====demo end====\n\n");
return getchar();
diff --git a/tests/examples/c/stream.c b/tests/examples/c/stream.c
old mode 100755
new mode 100644
diff --git a/tests/pytest/alter/alter_table_crash.py b/tests/pytest/alter/alter_table_crash.py
new file mode 100644
index 0000000000000000000000000000000000000000..903bb60e6a4af36970bbc8f6e38f9274fe434c70
--- /dev/null
+++ b/tests/pytest/alter/alter_table_crash.py
@@ -0,0 +1,83 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============Case 1: add column, restart taosd, drop the same colum then add it back")
+ tdSql.execute(
+ "create table st(ts timestamp, speed int) tags(loc nchar(20))")
+ tdSql.execute(
+ "insert into t1 using st tags('beijing') values(now, 1)")
+ tdSql.execute(
+ "alter table st add column tbcol binary(20)")
+
+ # restart taosd
+ tdDnodes.forcestop(1)
+ tdDnodes.start(1)
+
+ tdSql.execute(
+ "alter table st drop column tbcol")
+ tdSql.execute(
+ "alter table st add column tbcol binary(20)")
+
+ tdSql.query("select * from st")
+ tdSql.checkRows(1)
+
+
+ print("==============Case 2: keep adding columns, restart taosd")
+ tdSql.execute(
+ "create table dt(ts timestamp, tbcol1 tinyint) tags(tgcol1 tinyint)")
+ tdSql.execute(
+ "alter table dt add column tbcol2 int")
+ tdSql.execute(
+ "alter table dt add column tbcol3 smallint")
+ tdSql.execute(
+ "alter table dt add column tbcol4 bigint")
+ tdSql.execute(
+ "alter table dt add column tbcol5 float")
+ tdSql.execute(
+ "alter table dt add column tbcol6 double")
+ tdSql.execute(
+ "alter table dt add column tbcol7 bool")
+ tdSql.execute(
+ "alter table dt add column tbcol8 nchar(20)")
+ tdSql.execute(
+ "alter table dt add column tbcol9 binary(20)")
+
+ # restart taosd
+ tdDnodes.forcestop(1)
+ tdDnodes.start(1)
+
+ tdSql.query("select * from st")
+ tdSql.checkRows(0)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index dfa3b5504832ce6495286b4f7ea1f1748b56c77a..0bacd63ec14b69fbc0f03fab97651943cae9d9d4 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -126,6 +126,9 @@ python3 ./test.py -f import_merge/importInsertThenImport.py
python3 ./test.py -f user/user_create.py
python3 ./test.py -f user/pass_len.py
+# stable
+python3 ./test.py -f stable/query_after_reset.py
+
# table
python3 ./test.py -f table/del_stable.py
@@ -138,7 +141,11 @@ python3 ./test.py -f query/filterAllIntTypes.py
python3 ./test.py -f query/filterFloatAndDouble.py
python3 ./test.py -f query/filterOtherTypes.py
python3 ./test.py -f query/querySort.py
+python3 ./test.py -f query/queryJoin.py
#stream
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py
+
+#alter table
+python3 ./test.py -f alter/alter_table_crash.py
\ No newline at end of file
diff --git a/tests/pytest/insert/nchar-unicode.py b/tests/pytest/insert/nchar-unicode.py
index 152a09723ed262cfa6ceaad15ba89829996182b2..c417a6bca20a47a2035fbc1c75854e43147e34e3 100644
--- a/tests/pytest/insert/nchar-unicode.py
+++ b/tests/pytest/insert/nchar-unicode.py
@@ -25,7 +25,6 @@ class TDTestCase:
def run(self):
tdSql.prepare()
- tdSql.error('create table tb (ts timestamp, col nchar(1022))')
tdSql.execute('create table tb (ts timestamp, col nchar(1021))')
tdSql.execute("insert into tb values (now, 'taosdata')")
tdSql.query("select * from tb")
diff --git a/tests/pytest/query/filterAllIntTypes.py b/tests/pytest/query/filterAllIntTypes.py
index cf65d2b1021c22200b89a94a8f6093bc45937872..a2bab63c886558d87eb63e3b26b222c83afdebc0 100644
--- a/tests/pytest/query/filterAllIntTypes.py
+++ b/tests/pytest/query/filterAllIntTypes.py
@@ -88,6 +88,19 @@ class TDTestCase:
tdSql.query("select * from st%s where num != 50" % curType)
tdSql.checkRows(101)
+ # range for int type on column
+ tdSql.query("select * from st%s where num > 50 and num < 100" % curType)
+ tdSql.checkRows(49)
+
+ tdSql.query("select * from st%s where num >= 50 and num < 100" % curType)
+ tdSql.checkRows(50)
+
+ tdSql.query("select * from st%s where num > 50 and num <= 100" % curType)
+ tdSql.checkRows(50)
+
+ tdSql.query("select * from st%s where num >= 50 and num <= 100" % curType)
+ tdSql.checkRows(51)
+
# > for int type on tag
tdSql.query("select * from st%s where id > 5" % curType)
tdSql.checkRows(52)
@@ -116,6 +129,23 @@ class TDTestCase:
tdSql.query("select * from st%s where id != 5" % curType)
tdSql.checkRows(92)
+ # != for int type on tag
+ tdSql.query("select * from st%s where id != 5" % curType)
+ tdSql.checkRows(92)
+
+ # range for int type on tag
+ tdSql.query("select * from st%s where id > 5 and id < 7" % curType)
+ tdSql.checkRows(10)
+
+ tdSql.query("select * from st%s where id >= 5 and id < 7" % curType)
+ tdSql.checkRows(20)
+
+ tdSql.query("select * from st%s where id > 5 and id <= 7" % curType)
+ tdSql.checkRows(20)
+
+ tdSql.query("select * from st%s where id >= 5 and id <= 7" % curType)
+ tdSql.checkRows(30)
+
print(
"======= Verify filter for %s type finished =========" %
curType)
diff --git a/tests/pytest/query/filterFloatAndDouble.py b/tests/pytest/query/filterFloatAndDouble.py
index bd349a2a25c556723775e558d890cbca19803156..8d0e5baeca6775b129c5fc2438bb1a6bc8957b87 100644
--- a/tests/pytest/query/filterFloatAndDouble.py
+++ b/tests/pytest/query/filterFloatAndDouble.py
@@ -67,6 +67,19 @@ class TDTestCase:
tdSql.query("select * from st where num < 5.5")
tdSql.checkRows(4)
+ # range for float type on column
+ tdSql.query("select * from st where num > 5.5 and num < 11.0")
+ tdSql.checkRows(4)
+
+ tdSql.query("select * from st where num >= 5.5 and num < 11.0")
+ tdSql.checkRows(5)
+
+ tdSql.query("select * from st where num > 5.5 and num <= 11.0")
+ tdSql.checkRows(5)
+
+ tdSql.query("select * from st where num >= 5.5 and num <= 11.0")
+ tdSql.checkRows(6)
+
# > for float type on tag
tdSql.query("select * from st where tagcol1 > 1.1")
tdSql.checkRows(0)
@@ -123,6 +136,19 @@ class TDTestCase:
tdSql.query("select * from st where speed < 11.5")
tdSql.checkRows(4)
+ # range for double type on column
+ tdSql.query("select * from st where speed > 11.5 and speed < 20.7")
+ tdSql.checkRows(3)
+
+ tdSql.query("select * from st where speed >= 11.5 and speed < 20.7")
+ tdSql.checkRows(4)
+
+ tdSql.query("select * from st where speed > 11.5 and speed <= 20.7")
+ tdSql.checkRows(4)
+
+ tdSql.query("select * from st where speed >= 11.5 and speed <= 20.7")
+ tdSql.checkRows(5)
+
# > for double type on tag
tdSql.query("select * from st where tagcol2 > 2.3")
tdSql.checkRows(0)
diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py
index bc7df18c8da9dafce62f20b13800b266815c2cbf..1db5497604923cecd8111015dd0ba497b914b453 100644
--- a/tests/pytest/query/filterOtherTypes.py
+++ b/tests/pytest/query/filterOtherTypes.py
@@ -30,9 +30,9 @@ class TDTestCase:
print("======= Verify filter for bool, nchar and binary type =========")
tdLog.debug(
- "create table st(ts timestamp, tbcol1 bool, tbcol2 nchar(10), tbcol3 binary(20)) tags(tagcol1 bool, tagcol2 nchar(10), tagcol3 binary(10))")
+ "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20)) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))")
tdSql.execute(
- "create table st(ts timestamp, tbcol1 bool, tbcol2 nchar(10), tbcol3 binary(20)) tags(tagcol1 bool, tagcol2 nchar(10), tagcol3 binary(10))")
+ "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20)) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))")
tdSql.execute("create table st1 using st tags(true, 'table1', '水表')")
for i in range(1, 6):
diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1a0aaed6b3a8d26ebefd5b5e11cc2ab5493a530
--- /dev/null
+++ b/tests/pytest/query/queryJoin.py
@@ -0,0 +1,128 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ def run(self):
+ tdSql.prepare()
+
+ print("======= Step 1 prepare data=========")
+ tdSql.execute(
+ "create table stb1 (ts timestamp, c1 int, c2 float) tags(t1 int, t2 binary(10), t3 nchar(10))")
+ tdSql.execute(
+ '''insert into tb1 using stb1 tags(1,'tb1', '表1') values ('2020-04-18 15:00:00.000', 1, 0.1)
+ ('2020-04-18 15:00:01.000', 1,0.1) ('2020-04-18 15:00:03.000', 3, 0.3) ('2020-04-18 15:00:04.000', 4,0.4)''')
+ tdSql.execute(
+ '''insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:00.000', 21, 2.1)
+ ('2020-04-18 15:00:01.000', 22,2.2) ('2020-04-18 15:00:02.000', 22, 2.1) ('2020-04-18 15:00:03.000', 23,2.2)''')
+
+ tdSql.execute(
+ "create table stb_t (ts timestamp, temperature int, humidity float) tags(id int, name binary(10), dscrption nchar(10))")
+ tdSql.execute(
+ '''insert into tb_t1 using stb_t tags(1,'tb_t1', '温度表1') values ('2020-04-18 15:00:00.000', 25, 0.5)
+ ('2020-04-18 15:00:01.000', 25, 0.5) ('2020-04-18 15:00:02.000', 26, 0.7) ('2020-04-18 15:00:03.000', 27, 0.8)''')
+ tdSql.execute(
+ '''insert into tb_t2 using stb_t tags(2,'tb_t2', '温度表2') values ('2020-04-18 15:00:00.000', 33, 0.9)
+ ('2020-04-18 15:00:01.000', 35, 1.1) ('2020-04-18 15:00:03.000', 36, 1.3) ('2020-04-18 15:00:04.000', 37, 1.4)''')
+
+ tdSql.execute(
+ "create table stb_p (ts timestamp, pressure float) tags(id int, name binary(10), dscrption nchar(10), location binary(20))")
+ tdSql.execute(
+ '''insert into tb_p1 using stb_p tags(1,'tb_p1', '压力计1', 'beijing') values ('2020-04-18 15:00:00.000', 76.6)
+ ('2020-04-18 15:00:01.000', 76.5) ('2020-04-18 15:00:01.500', 77.1) ('2020-04-18 15:00:02.000', 75.3)
+ ('2020-04-18 15:00:03.000', 75.1) ('2020-04-18 15:00:04.500', 77.3)''')
+ tdSql.execute(
+ '''insert into tb_p2 using stb_p tags(2,'tb_p2', '压力计2', 'shenzhen') values ('2020-04-18 14:59:59.000', 74.6)
+ ('2020-04-18 15:00:01.000', 74.5) ('2020-04-18 15:00:01.500', 73.6) ('2020-04-18 15:00:02.000', 74.5)
+ ('2020-04-18 15:00:02.500', 73.9) ('2020-04-18 15:00:03.000', 73.5)''')
+
+ tdSql.execute(
+ "create table stb_v (ts timestamp, velocity float) tags(id int, name binary(10), dscrption nchar(10), location binary(20))")
+ tdSql.execute(
+ '''insert into tb_v1 using stb_v tags(1,'tb_v1', '速度计1', 'beijing ') values ('2020-04-18 15:00:00.000', 176.6)
+ ('2020-04-18 15:00:01.000', 176.5)''')
+ tdSql.execute(
+ '''insert into tb_v2 using stb_v tags(2,'tb_v2', '速度计2', 'shenzhen') values ('2020-04-18 15:00:00.000', 171.6)
+ ('2020-04-18 15:00:01.000', 171.5)''')
+
+ # explicit join should not work
+ tdSql.error("select * from stb_p join stb_t on (stb_p.id = stb_t.id)")
+ tdSql.error("select * from tb1 join tb2 on (tb1.ts=tb2.ts)")
+ tdSql.error(
+ "select * from stb_p join stb_t on (stb_p.ts=stb_t.ts and stb_p.id = stb_t.id)")
+
+ # alias should not work
+ tdSql.error("select * from stb_p p join stb_t t on (p.id = t.id)")
+
+ # join queries
+ tdSql.query(
+ "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(6)
+
+ tdSql.query(
+ "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id order by ts desc")
+ tdSql.checkColumnSorted(0, "desc")
+
+ tdSql.error(
+ "select ts, pressure, temperature, id, dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+
+ tdSql.query("select stb_p.ts, pressure, stb_t.temperature, stb_p.id, stb_p.dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(6)
+
+ tdSql.query("select stb_t.ts, stb_p.pressure, stb_t.temperature,stb_p.id,stb_p.dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(6)
+
+ tdSql.error(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by name")
+ tdSql.error(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by stb_t.name")
+ tdSql.error(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by stb_t.id")
+ tdSql.error(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;")
+ tdSql.error(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name")
+
+ tdSql.execute("alter table stb_t add tag pid int")
+ tdSql.execute("alter table tb_t1 set tag pid=2")
+ tdSql.execute("alter table tb_t2 set tag pid=1")
+
+ tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.pid")
+ tdSql.checkRows(3)
+
+ tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(6)
+
+ tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(6)
+
+ tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.pid, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id")
+ tdSql.checkRows(2)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/query/querytest.py b/tests/pytest/query/querytest.py
index e2ed26f0f7ec170ecb6b1c9b286bb379161a48fd..d5068884489458f1fdcf000ac8f3e156ccaf10d0 100644
--- a/tests/pytest/query/querytest.py
+++ b/tests/pytest/query/querytest.py
@@ -16,43 +16,42 @@ import time
from datetime import datetime
class DBWriteNonStop:
- def __init(self)__:
+ def __init__(self):
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos"
- def init(self, host, user, password, config):
- self.host = host
- self.user = user
- self.password = password
- self.config = config
-
- def connectDB(self, conn, cursor):
+ def connectDB(self):
self.conn = taos.connect(self.host, self.user, self.password, self.config)
- self.cursor = conn.cursor()
- return self.cursor
-
- def createTable()
-
-
-c1.execute('drop database if exists dbwrite')
-c1.execute('create database dbwrite')
-c1.execute('use dbwrite')
-c1.execute('create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))')
-
-i = 1
-
-startTime = datetime.now()
-while True:
- c1.execute("insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" % (i % 10000, i % 100000))
- i += 1
- i = i % 32000000
- endTime = datetime.now()
- if (endTime - startTime).seconds >= 5 * 2:
- startTime = endTime
- c1.execute("select count(*) from st")
- data = c1.fetchall()
- print(datetime.now())
- print("total records: %d" % data[0][0])
- time.sleep(.001)
\ No newline at end of file
+ self.cursor = self.conn.cursor()
+
+ def createTable(self):
+ self.cursor.execute("drop database if exists dbwrite")
+ self.cursor.execute("create database dbwrite")
+ self.cursor.execute("use dbwrite")
+ self.cursor.execute("create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
+
+ def insertData(self):
+ i = 1
+ startTime = datetime.now()
+ while True:
+ self.cursor.execute("insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" % (i % 10000, i % 100000))
+ i += 1
+ i = i % 32000000
+ endTime = datetime.now()
+ if (endTime - startTime).seconds >= 5 * 2:
+ startTime = endTime
+ self.cursor.execute("select last(ts) from st >> output.txt")
+ self.cursor.execute("select count(*) from st >> output.txt")
+ time.sleep(.001)
+
+ def closeConn(self):
+ self.cursor.close()
+ self.conn.close()
+
+test = DBWriteNonStop()
+test.connectDB()
+test.createTable()
+test.insertData()
+test.closeConn()
\ No newline at end of file
diff --git a/tests/pytest/random-test/random-test.py b/tests/pytest/random-test/random-test.py
index 31ce70e90a291659137d6abbc539cf297ff62411..38cefb3eecadf9f585b6f956a7ea0c0076fe2918 100644
--- a/tests/pytest/random-test/random-test.py
+++ b/tests/pytest/random-test/random-test.py
@@ -17,6 +17,7 @@ from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
+import codecs
class Test:
@@ -93,14 +94,13 @@ class Test:
self.last_stb = current_stb
current_tb = "tb%d" % int(round(time.time() * 1000))
- tdSql.execute(
- "create table %s using %s tags (1, '表1')" %
- (current_tb, self.last_stb))
+ sqlcmd = "create table %s using %s tags (1, 'test')" %(current_tb, self.last_stb)
+ tdSql.execute(sqlcmd)
self.last_tb = current_tb
self.written = 0
tdSql.execute(
- "insert into %s values (now, 27, '我是nchar字符串')" %
+ "insert into %s values (now, 27, 'wsnchar')" %
self.last_tb)
self.written = self.written + 1
diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh
index 1fc3edf4a96ddb33a9ab1613926962e43331129b..955d6aa2ef0e8f60c6b3960817fc5e45acab8a62 100755
--- a/tests/pytest/regressiontest.sh
+++ b/tests/pytest/regressiontest.sh
@@ -13,7 +13,7 @@ python3 ./test.py -f insert/date.py
python3 ./test.py -f insert/binary.py
python3 ./test.py -f insert/nchar.py
# python3 ./test.py -f insert/nchar-boundary.py
-# python3 ./test.py -f insert/nchar-unicode.py
+python3 ./test.py -f insert/nchar-unicode.py
python3 ./test.py -f insert/multi.py
python3 ./test.py -f insert/randomNullCommit.py
diff --git a/tests/pytest/stable/query_after_reset-multi-threading.py b/tests/pytest/stable/query_after_reset-multi-threading.py
new file mode 100644
index 0000000000000000000000000000000000000000..a177f589d166117c0d8de61a01a34040a1b1103b
--- /dev/null
+++ b/tests/pytest/stable/query_after_reset-multi-threading.py
@@ -0,0 +1,259 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import random
+import threading
+import queue
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+current_tb = ""
+last_tb = ""
+written = 0
+
+
+class Test (threading.Thread):
+ def __init__(self, threadId, name, sleepTime, q):
+ threading.Thread.__init__(self)
+ self.threadId = threadId
+ self.name = name
+ self.sleepTime = sleepTime
+ self.q = q
+
+ self.threadLock = threading.Lock()
+
+ def create_table(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("create a table")
+ current_tb = "tb%d" % int(round(time.time() * 1000))
+ tdLog.info("current table %s" % current_tb)
+
+ if (current_tb == last_tb):
+ return
+ else:
+ tdSql.execute(
+ 'create table %s (ts timestamp, speed int)' %
+ current_tb)
+ last_tb = current_tb
+ written = 0
+
+ def insert_data(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("will insert data to table")
+ if (current_tb == ""):
+ tdLog.info("no table, create first")
+ self.create_table()
+
+ tdLog.info("insert data to table")
+ for i in range(0, 10):
+ self.threadLock.acquire()
+ insertRows = 1000
+ tdLog.info("insert %d rows to %s" % (insertRows, current_tb))
+
+ for j in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into %s values (now + %dm, %d)' %
+ (current_tb, j, j))
+ written = written + 1
+ self.threadLock.release()
+
+ def query_data(self):
+ global current_tb
+ global last_tb
+ global written
+
+ if (written > 0):
+ tdLog.info("query data from table")
+ tdSql.query("select * from %s" % last_tb)
+ tdSql.checkRows(written)
+
+ def query_stable(self):
+ tdLog.sleep(10)
+ self.threadLock.acquire()
+ tdLog.info("query super table")
+
+ try:
+ tdSql.query("select * from st")
+ except Exception as e:
+ tdLog.info(repr(e))
+ self.q.put(-1)
+ tdLog.exit("failed")
+ tdSql.checkRows(1)
+ self.threadLock.release()
+
+ def create_stable(self):
+ global current_tb
+ global last_tb
+ global written
+
+ self.threadLock.acquire()
+
+ tdLog.info("create a super table and sub-table and insert data")
+
+ try:
+ tdSql.execute(
+ "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
+ tdSql.execute(
+ 'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
+ tdSql.execute(
+ "INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1)")
+ except Exception as e:
+ tdLog.info(repr(e))
+ self.q.put(-1)
+ tdLog.exit("failed")
+ self.stop_database()
+ self.delete_datafiles()
+ self.restart_database()
+ self.threadLock.release()
+
+ def stop_database(self):
+ tdLog.info("stop databae")
+ tdDnodes.stop(1)
+
+ def restart_database(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("restart databae")
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdLog.sleep(5)
+
+ def force_restart(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("force restart database")
+ tdDnodes.forcestop(1)
+ tdDnodes.start(1)
+ tdLog.sleep(5)
+
+ def drop_table(self):
+ global current_tb
+ global last_tb
+ global written
+
+ for i in range(0, 10):
+ self.threadLock.acquire()
+
+ tdLog.info("current_tb %s" % current_tb)
+
+ if (current_tb != ""):
+ tdLog.info("drop current tb %s" % current_tb)
+ tdSql.execute("drop table %s" % current_tb)
+ current_tb = ""
+ last_tb = ""
+ written = 0
+ tdLog.sleep(self.sleepTime)
+ self.threadLock.release()
+
+ def reset_query_cache(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("reset query cache")
+ tdSql.execute("reset query cache")
+ tdLog.sleep(1)
+
+ def reset_database(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("reset database")
+ tdDnodes.forcestop(1)
+ tdDnodes.deploy(1)
+ current_tb = ""
+ last_tb = ""
+ written = 0
+ tdDnodes.start(1)
+ tdSql.prepare()
+
+ def delete_datafiles(self):
+ global current_tb
+ global last_tb
+ global written
+
+ tdLog.info("delete data files")
+ dnodesDir = tdDnodes.getDnodesRootDir()
+ dataDir = dnodesDir + '/dnode1/*'
+ deleteCmd = 'rm -rf %s' % dataDir
+ os.system(deleteCmd)
+
+ current_tb = ""
+ last_tb = ""
+ written = 0
+ tdDnodes.start(1)
+ tdSql.prepare()
+
+ def run(self):
+ switch = {
+ 1: self.create_table,
+ 2: self.insert_data,
+ 3: self.query_data,
+ 4: self.create_stable,
+ 5: self.restart_database,
+ 6: self.force_restart,
+ 7: self.drop_table,
+ 8: self.reset_query_cache,
+ 9: self.reset_database,
+ 10: self.delete_datafiles,
+ 11: self.query_stable,
+ }
+
+ switch.get(self.threadId, lambda: "ERROR")()
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+
+ q = queue.Queue()
+ test1 = Test(4, "create_stable", 1, q)
+ test2 = Test(11, "query_stable", 2, q)
+
+ test1.start()
+ test2.start()
+ test1.join()
+ test2.join()
+
+ while not q.empty():
+ if (q.get() == -1):
+ tdLog.exit("failed to end of test")
+
+ tdLog.info("end of test")
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/stable/query_after_reset.py b/tests/pytest/stable/query_after_reset.py
new file mode 100644
index 0000000000000000000000000000000000000000..2bc171ae5d53af86bb72b142ba54b52c76a9e80c
--- /dev/null
+++ b/tests/pytest/stable/query_after_reset.py
@@ -0,0 +1,177 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import random
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class Test:
+ def __init__(self):
+ self.current_tb = ""
+ self.last_tb = ""
+ self.written = 0
+
+ def create_table(self):
+ tdLog.info("create a table")
+ self.current_tb = "tb%d" % int(round(time.time() * 1000))
+ tdLog.info("current table %s" % self.current_tb)
+
+ if (self.current_tb == self.last_tb):
+ return
+ else:
+ tdSql.execute(
+ 'create table %s (ts timestamp, speed int)' %
+ self.current_tb)
+ self.last_tb = self.current_tb
+ self.written = 0
+
+ def insert_data(self):
+ tdLog.info("will insert data to table")
+ if (self.current_tb == ""):
+ tdLog.info("no table, create first")
+ self.create_table()
+
+ tdLog.info("insert data to table")
+ insertRows = 10
+ tdLog.info("insert %d rows to %s" % (insertRows, self.last_tb))
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into %s values (now + %dm, %d)' %
+ (self.last_tb, i, i))
+ self.written = self.written + 1
+
+ tdLog.info("insert earlier data")
+ tdSql.execute('insert into %s values (now - 5m , 10)' % self.last_tb)
+ self.written = self.written + 1
+ tdSql.execute('insert into %s values (now - 6m , 10)' % self.last_tb)
+ self.written = self.written + 1
+ tdSql.execute('insert into %s values (now - 7m , 10)' % self.last_tb)
+ self.written = self.written + 1
+ tdSql.execute('insert into %s values (now - 8m , 10)' % self.last_tb)
+ self.written = self.written + 1
+
+ def query_data(self):
+ if (self.written > 0):
+ tdLog.info("query data from table")
+ tdSql.query("select * from %s" % self.last_tb)
+ tdSql.checkRows(self.written)
+
+ def query_stable(self):
+ tdLog.info("query super table")
+ tdSql.query("select * from st")
+ tdSql.checkRows(1)
+
+ def create_stable(self):
+ tdLog.info("create a super table and sub-table and insert data")
+ tdSql.execute(
+ "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
+ tdSql.execute(
+ 'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
+ tdSql.execute(
+ "INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1)")
+
+ def stop_database(self):
+ tdLog.info("stop databae")
+ tdDnodes.stop(1)
+
+ def restart_database(self):
+ tdLog.info("restart databae")
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdLog.sleep(5)
+
+ def force_restart(self):
+ tdLog.info("force restart database")
+ tdDnodes.forcestop(1)
+ tdDnodes.start(1)
+ tdLog.sleep(5)
+
+ def drop_table(self):
+ if (self.current_tb != ""):
+ tdLog.info("drop current tb %s" % self.current_tb)
+ tdSql.execute("drop table %s" % self.current_tb)
+ self.current_tb = ""
+ self.last_tb = ""
+ self.written = 0
+
+ def reset_query_cache(self):
+ tdLog.info("reset query cache")
+ tdSql.execute("reset query cache")
+ tdLog.sleep(1)
+
+ def reset_database(self):
+ tdLog.info("reset database")
+ tdDnodes.forcestop(1)
+ tdDnodes.deploy(1)
+ self.current_tb = ""
+ self.last_tb = ""
+ self.written = 0
+ tdDnodes.start(1)
+ tdSql.prepare()
+
+ def delete_datafiles(self):
+ tdLog.info("delete data files")
+ dnodesDir = tdDnodes.getDnodesRootDir()
+ dataDir = dnodesDir + '/dnode1/*'
+ deleteCmd = 'rm -rf %s' % dataDir
+ os.system(deleteCmd)
+
+ self.current_tb = ""
+ self.last_tb = ""
+ self.written = 0
+ tdDnodes.start(1)
+ tdSql.prepare()
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+
+ test = Test()
+
+ switch = {
+ 1: test.create_table,
+ 2: test.insert_data,
+ 3: test.query_data,
+ 4: test.create_stable,
+ 5: test.restart_database,
+ 6: test.force_restart,
+ 7: test.drop_table,
+ 8: test.reset_query_cache,
+ 9: test.reset_database,
+ 10: test.delete_datafiles,
+ 11: test.query_stable,
+ 12: test.stop_database,
+ }
+
+ switch.get(4, lambda: "ERROR")()
+ switch.get(12, lambda: "ERROR")()
+ switch.get(10, lambda: "ERROR")()
+ switch.get(5, lambda: "ERROR")()
+ switch.get(11, lambda: "ERROR")()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index f3fd178088b41f18b5b7cd31a98853a6c0b79055..3b86a5334396f65b6a5a0f0685403560f3e14499 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -40,10 +40,18 @@ class TDSql:
def prepare(self):
tdLog.info("prepare database:db")
- self.cursor.execute('reset query cache')
- self.cursor.execute('drop database if exists db')
- self.cursor.execute('create database db')
- self.cursor.execute('use db')
+ s = 'reset query cache'
+ print(s)
+ self.cursor.execute(s)
+ s = 'drop database if exists db'
+ print(s)
+ self.cursor.execute(s)
+ s = 'create database db'
+ print(s)
+ self.cursor.execute(s)
+ s = 'use db'
+ print(s)
+ self.cursor.execute(s)
def error(self, sql):
expectErrNotOccured = True
@@ -66,6 +74,7 @@ class TDSql:
def query(self, sql):
self.sql = sql
+ print(sql)
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self.queryRows = len(self.queryResult)
@@ -182,6 +191,7 @@ class TDSql:
def execute(self, sql):
self.sql = sql
+ print(sql)
self.affectedRows = self.cursor.execute(sql)
return self.affectedRows
diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim
index a7d6707e7d66d39f446ad95ed09ee78945481c16..2f45ab077c095a89d04b780052f5c36ac67983ee 100644
--- a/tests/script/fullGeneralSuite.sim
+++ b/tests/script/fullGeneralSuite.sim
@@ -1,10 +1,10 @@
#unsupport run general/alter/cached_schema_after_alter.sim
-#unsupport run general/alter/count.sim
-#unsupport run general/alter/import.sim
+run general/alter/count.sim
+run general/alter/import.sim
#unsupport run general/alter/insert1.sim
-#unsupport run general/alter/insert2.sim
-#unsupport run general/alter/metrics.sim
-#unsupport run general/alter/table.sim
+run general/alter/insert2.sim
+run general/alter/metrics.sim
+run general/alter/table.sim
run general/cache/new_metrics.sim
run general/cache/restart_metrics.sim
run general/cache/restart_table.sim
@@ -87,8 +87,8 @@ run general/insert/query_file_memory.sim
run general/insert/query_multi_file.sim
run general/insert/tcp.sim
#unsupport run general/parser/alter.sim
-#unsupport run general/parser/alter1.sim
-#unsupport run general/parser/alter_stable.sim
+run general/parser/alter1.sim
+run general/parser/alter_stable.sim
run general/parser/auto_create_tb.sim
run general/parser/auto_create_tb_drop_tb.sim
run general/parser/col_arithmetic_operation.sim
@@ -125,17 +125,17 @@ run general/parser/fill.sim
run general/parser/fill_stb.sim
run general/parser/interp.sim
run general/parser/where.sim
-#unsupport run general/parser/join.sim
-#unsupport run general/parser/join_multivnode.sim
+run general/parser/join.sim
+run general/parser/join_multivnode.sim
run general/parser/select_with_tags.sim
-#unsupport run general/parser/groupby.sim
-#unsupport run general/parser/bug.sim
-#unsupport run general/parser/tags_dynamically_specifiy.sim
-#unsupport run general/parser/set_tag_vals.sim
+run general/parser/groupby.sim
+run general/parser/bug.sim
+run general/parser/tags_dynamically_specifiy.sim
+run general/parser/set_tag_vals.sim
#unsupport run general/parser/repeatAlter.sim
#unsupport run general/parser/slimit_alter_tags.sim
#unsupport run general/parser/stream_on_sys.sim
-#unsupport run general/parser/stream.sim
+run general/parser/stream.sim
#unsupport run general/parser/repeatStream.sim
run general/stable/disk.sim
run general/stable/dnode3.sim
@@ -173,25 +173,25 @@ run general/tag/3.sim
run general/tag/4.sim
run general/tag/5.sim
run general/tag/6.sim
-#unsupport run general/tag/add.sim
+run general/tag/add.sim
run general/tag/bigint.sim
run general/tag/binary_binary.sim
run general/tag/binary.sim
run general/tag/bool_binary.sim
run general/tag/bool_int.sim
run general/tag/bool.sim
-#unsupport run general/tag/change.sim
+run general/tag/change.sim
run general/tag/column.sim
#unsupport run general/tag/commit.sim
run general/tag/create.sim
-#unsupport run general/tag/delete.sim
+run general/tag/delete.sim
run general/tag/double.sim
run general/tag/filter.sim
run general/tag/float.sim
run general/tag/int_binary.sim
run general/tag/int_float.sim
run general/tag/int.sim
-#unsupport run general/tag/set.sim
+run general/tag/set.sim
run general/tag/smallint.sim
run general/tag/tinyint.sim
run general/user/authority.sim
@@ -211,6 +211,7 @@ run general/vector/table_field.sim
run general/vector/table_mix.sim
run general/vector/table_query.sim
run general/vector/table_time.sim
+run general/stream/restart_stream.sim
run general/stream/stream_1.sim
run general/stream/stream_2.sim
run general/stream/stream_3.sim
diff --git a/tests/script/general/alter/cached_schema_after_alter.sim b/tests/script/general/alter/cached_schema_after_alter.sim
index d06856d5ff62e5cf96508994906847f690cfa593..2d049ec5955e281d866eaaa15abf06583909c87d 100644
--- a/tests/script/general/alter/cached_schema_after_alter.sim
+++ b/tests/script/general/alter/cached_schema_after_alter.sim
@@ -1,7 +1,7 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
@@ -56,11 +56,12 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 5000
system sh/exec.sh -n dnode1 -s start
print ================== server restart completed
+sleep 5000
sql connect
-sleep 3000
sql use $db
sql select * from $stb
+print select * from $stb ==> $data00 $data01 $data02
if $rows != 1 then
return -1
endi
@@ -72,6 +73,7 @@ if $data02 != 1 then
endi
sql select * from $tb2
+print select * from $tb2 ==> $data00 $data01 $data02
if $rows != 1 then
return -1
endi
diff --git a/tests/script/general/alter/count.sim b/tests/script/general/alter/count.sim
index 4728433dd2af02de7597f33d5bb7b652d4ea5a8b..157b4c137198b70a9e7ad3b1f64e5b4b06a90415 100644
--- a/tests/script/general/alter/count.sim
+++ b/tests/script/general/alter/count.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
-
-system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
diff --git a/tests/script/general/alter/import.sim b/tests/script/general/alter/import.sim
index 0ad52373fc53d8b085c0112dbd2b1e713519a806..76388a26f2b15d1ce9a86331ab2e70cb29a88480 100644
--- a/tests/script/general/alter/import.sim
+++ b/tests/script/general/alter/import.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
-
-system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
@@ -42,9 +41,9 @@ if $data00 != 5 then
return -1
endi
-sql_error import into tb values(now-29d, -29, 0)
+sql import into tb values(now-29d, -29, 0)
sql select count(b) from tb
-if $data00 != 5 then
+if $data00 != 6 then
return -1
endi
diff --git a/tests/script/general/alter/insert1.sim b/tests/script/general/alter/insert1.sim
index 4246bdc18e05bb4f2bf66352d9b8b0e417855191..3b16214465c15133aecc3e2c5977996cbaa22b5f 100644
--- a/tests/script/general/alter/insert1.sim
+++ b/tests/script/general/alter/insert1.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -14,41 +13,42 @@ sql use d3
sql create table tb (ts timestamp, a int)
sql insert into tb values(now-28d, -28)
sql select * from tb order by ts desc
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+
if $rows != 1 then
return -1
endi
if $data01 != -28 then
return -1
endi
-if $data02 != NULL then
+if $data02 != null then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
print ======== step2
-sleep 2500
sql alter table tb add column b smallint
sql insert into tb values(now-25d, -25, 0)
-sleep 3000
sql select * from tb order by ts desc
-print $rows
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
if $rows != 2 then
return -1
endi
@@ -64,31 +64,32 @@ endi
if $data12 != NULL then
return -1
endi
-if $data13 != NULL then
+if $data13 != null then
return -1
endi
-if $data14 != NULL then
+if $data14 != null then
return -1
endi
-if $data15 != NULL then
+if $data15 != null then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
print ======== step3
-sleep 2500
sql alter table tb add column c tinyint
sql insert into tb values(now-22d, -22, 3, 0)
-sleep 3000
sql select * from tb order by ts asc
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26
if $rows != 3 then then
return -1
endi
@@ -101,19 +102,19 @@ endi
if $data03 != NULL then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -25 then
@@ -125,19 +126,19 @@ endi
if $data13 != NULL then
return -1
endi
-if $data14 != NULL then
+if $data14 != null then
return -1
endi
-if $data15 != NULL then
+if $data15 != null then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != -22 then
@@ -149,27 +150,25 @@ endi
if $data23 != 0 then
return -1
endi
-if $data24 != NULL then
+if $data24 != null then
return -1
endi
-if $data25 != NULL then
+if $data25 != null then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
print ======== step4
-sleep 2500
sql alter table tb add column d int
sql insert into tb values(now-19d, -19, 6, 3, 0)
-sleep 3000
sql select * from tb order by ts asc
if $rows != 4 then then
return -1
@@ -186,16 +185,16 @@ endi
if $data04 != NULL then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -25 then
@@ -210,16 +209,16 @@ endi
if $data14 != NULL then
return -1
endi
-if $data15 != NULL then
+if $data15 != null then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != -22 then
@@ -234,16 +233,16 @@ endi
if $data24 != NULL then
return -1
endi
-if $data25 != NULL then
+if $data25 != null then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != -19 then
@@ -258,24 +257,22 @@ endi
if $data34 != 0 then
return -1
endi
-if $data35 != NULL then
+if $data35 != null then
return -1
endi
-if $data36 != NULL then
+if $data36 != null then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
print ======== step5
-sleep 2500
sql alter table tb add column e bigint
sql insert into tb values(now-16d, -16, 9, 5, 4, 3)
-sleep 3000
sql select count(e) from tb
if $data00 != 1 then
return -1
@@ -303,13 +300,13 @@ endi
if $data05 != NULL then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -25 then
@@ -327,13 +324,13 @@ endi
if $data15 != NULL then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != -22 then
@@ -351,13 +348,13 @@ endi
if $data25 != NULL then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != -19 then
@@ -375,13 +372,13 @@ endi
if $data35 != NULL then
return -1
endi
-if $data36 != NULL then
+if $data36 != null then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
if $data41 != -16 then
@@ -399,21 +396,19 @@ endi
if $data45 != 3 then
return -1
endi
-if $data46 != NULL then
+if $data46 != null then
return -1
endi
-if $data47 != NULL then
+if $data47 != null then
return -1
endi
-if $data48 != NULL then
+if $data48 != null then
return -1
endi
print ======== step6
-sleep 2500
sql alter table tb add column f float
sql insert into tb values(now-13d, -13, 12, 11, 10, 9, 8)
-sleep 3000
sql select * from tb order by ts asc
if $rows != 6 then then
return -1
@@ -436,10 +431,10 @@ endi
if $data06 != NULL then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -25 then
@@ -460,10 +455,10 @@ endi
if $data16 != NULL then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != -22 then
@@ -484,10 +479,10 @@ endi
if $data26 != NULL then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != -19 then
@@ -508,10 +503,10 @@ endi
if $data36 != NULL then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
if $data41 != -16 then
@@ -532,10 +527,10 @@ endi
if $data46 != NULL then
return -1
endi
-if $data47 != NULL then
+if $data47 != null then
return -1
endi
-if $data48 != NULL then
+if $data48 != null then
return -1
endi
if $data51 != -13 then
@@ -556,18 +551,16 @@ endi
if $data56 != 8.00000 then
return -1
endi
-if $data57 != NULL then
+if $data57 != null then
return -1
endi
-if $data58 != NULL then
+if $data58 != null then
return -1
endi
print ======== step7
-sleep 2500
sql alter table tb add column g double
sql insert into tb values(now-10d, -10, 15, 14, 13, 12, 11, 10)
-sleep 3000
sql select * from tb order by ts asc
if $rows != 7 then
return -1
@@ -593,7 +586,7 @@ endi
if $data07 != NULL then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -25 then
@@ -617,7 +610,7 @@ endi
if $data17 != NULL then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != -22 then
@@ -641,7 +634,7 @@ endi
if $data27 != NULL then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != -19 then
@@ -665,7 +658,7 @@ endi
if $data37 != NULL then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
if $data41 != -16 then
@@ -689,7 +682,7 @@ endi
if $data47 != NULL then
return -1
endi
-if $data48 != NULL then
+if $data48 != null then
return -1
endi
if $data51 != -13 then
@@ -713,7 +706,7 @@ endi
if $data57 != NULL then
return -1
endi
-if $data58 != NULL then
+if $data58 != null then
return -1
endi
if $data61 != -10 then
@@ -737,15 +730,13 @@ endi
if $data67 != 10.000000000 then
return -1
endi
-if $data68 != NULL then
+if $data68 != null then
return -1
endi
print ======== step8
-sleep 2500
sql alter table tb add column h binary(10)
sql insert into tb values(now-7d, -7, 18, 17, 16, 15, 14, 13, '11')
-sleep 3000
sql select * from tb order by ts asc
if $rows != 8 then
return -1
@@ -942,17 +933,18 @@ endi
if $data78 != 11 then
return -1
endi
-if $data79 != NULL then
+if $data79 != null then
return -1
endi
print ======== step9
+
system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 5000
system sh/exec.sh -n dnode1 -s start
sleep 5000
-sql select * from tb
+sql select * from tb order by ts asc
if $rows != 8 then
return -1
endi
@@ -1148,5 +1140,8 @@ endi
if $data78 != 11 then
return -1
endi
+if $data79 != null then
+ return -1
+endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/alter/insert2.sim b/tests/script/general/alter/insert2.sim
index 007060b7f061f5d50f7af13ddc9e1dfeaeddf8e2..28948b69d231952f3baae0d79a75c1f2c9c6fb60 100644
--- a/tests/script/general/alter/insert2.sim
+++ b/tests/script/general/alter/insert2.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -43,12 +42,10 @@ if $data08 != 8 then
endi
print ======== step2
-sleep 2500
sql_error alter table tb add column b smallint
sql_error alter table tb add column b int
sql alter table tb drop column b
sql insert into tb values(now-25d, 2, 3, 4, 5, 6, 7, 8)
-sleep 3000
sql select * from tb order by ts desc
if $rows != 2 then
return -1
@@ -74,7 +71,7 @@ endi
if $data07 != 8 then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != 1 then
@@ -98,15 +95,13 @@ endi
if $data17 != 8 then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
print ======== step3
-sleep 2500
sql alter table tb drop column c
sql insert into tb values(now-22d, 3, 4, 5, 6, 7, 8)
-sleep 3000
sql select * from tb order by ts desc
if $rows != 3 then
return -1
@@ -129,10 +124,10 @@ endi
if $data06 != 8 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != 2 then
@@ -153,10 +148,10 @@ endi
if $data16 != 8 then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != 1 then
@@ -177,19 +172,17 @@ endi
if $data26 != 8 then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
print ======== step4
-sleep 2500
sql alter table tb drop column d
sql alter table tb drop column e
sql insert into tb values(now-19d, -19, 6, 3, 0)
-sleep 3000
sql select * from tb order by ts desc
if $rows != 4 then
return -1
@@ -206,16 +199,16 @@ endi
if $data04 != 0 then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != 3 then
@@ -230,16 +223,16 @@ endi
if $data14 != 8 then
return -1
endi
-if $data15 != NULL then
+if $data15 != null then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != 2 then
@@ -254,16 +247,16 @@ endi
if $data24 != 8 then
return -1
endi
-if $data25 != NULL then
+if $data25 != null then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != 1 then
@@ -278,24 +271,22 @@ endi
if $data34 != 8 then
return -1
endi
-if $data35 != NULL then
+if $data35 != null then
return -1
endi
-if $data36 != NULL then
+if $data36 != null then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
print ======== step5
-sleep 2500
sql alter table tb drop column g
sql insert into tb values(now-16d, -16, 9, 5)
-sleep 3000
sql select count(f) from tb
if $data00 != 5 then
return -1
@@ -313,19 +304,19 @@ endi
if $data03 != 5 then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data11 != -19 then
@@ -337,19 +328,19 @@ endi
if $data13 != 0 then
return -1
endi
-if $data14 != NULL then
+if $data14 != null then
return -1
endi
-if $data15 != NULL then
+if $data15 != null then
return -1
endi
-if $data16 != NULL then
+if $data16 != null then
return -1
endi
-if $data17 != NULL then
+if $data17 != null then
return -1
endi
-if $data18 != NULL then
+if $data18 != null then
return -1
endi
if $data21 != 3 then
@@ -361,19 +352,19 @@ endi
if $data23 != 8 then
return -1
endi
-if $data24 != NULL then
+if $data24 != null then
return -1
endi
-if $data25 != NULL then
+if $data25 != null then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != 2 then
@@ -385,19 +376,19 @@ endi
if $data33 != 8 then
return -1
endi
-if $data34 != NULL then
+if $data34 != null then
return -1
endi
-if $data35 != NULL then
+if $data35 != null then
return -1
endi
-if $data36 != NULL then
+if $data36 != null then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
if $data41 != 1 then
@@ -409,27 +400,25 @@ endi
if $data43 != 8 then
return -1
endi
-if $data44 != NULL then
+if $data44 != null then
return -1
endi
-if $data45 != NULL then
+if $data45 != null then
return -1
endi
-if $data46 != NULL then
+if $data46 != null then
return -1
endi
-if $data47 != NULL then
+if $data47 != null then
return -1
endi
-if $data48 != NULL then
+if $data48 != null then
return -1
endi
print ======== step6
-sleep 2500
sql alter table tb drop column f
sql insert into tb values(now-13d, -13, 7)
-sleep 3000
sql select * from tb order by ts desc
if $rows != 6 then
return -1
@@ -443,22 +432,22 @@ endi
if $data12 != 5 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
-if $data08 != NULL then
+if $data08 != null then
return -1
endi
if $data21 != -19 then
@@ -467,22 +456,22 @@ endi
if $data22 != 0 then
return -1
endi
-if $data23 != NULL then
+if $data23 != null then
return -1
endi
-if $data24 != NULL then
+if $data24 != null then
return -1
endi
-if $data25 != NULL then
+if $data25 != null then
return -1
endi
-if $data26 != NULL then
+if $data26 != null then
return -1
endi
-if $data27 != NULL then
+if $data27 != null then
return -1
endi
-if $data28 != NULL then
+if $data28 != null then
return -1
endi
if $data31 != 3 then
@@ -491,22 +480,22 @@ endi
if $data32 != 8 then
return -1
endi
-if $data33 != NULL then
+if $data33 != null then
return -1
endi
-if $data34 != NULL then
+if $data34 != null then
return -1
endi
-if $data35 != NULL then
+if $data35 != null then
return -1
endi
-if $data36 != NULL then
+if $data36 != null then
return -1
endi
-if $data37 != NULL then
+if $data37 != null then
return -1
endi
-if $data38 != NULL then
+if $data38 != null then
return -1
endi
if $data41 != 2 then
@@ -515,22 +504,22 @@ endi
if $data42 != 8 then
return -1
endi
-if $data43 != NULL then
+if $data43 != null then
return -1
endi
-if $data44 != NULL then
+if $data44 != null then
return -1
endi
-if $data45 != NULL then
+if $data45 != null then
return -1
endi
-if $data46 != NULL then
+if $data46 != null then
return -1
endi
-if $data47 != NULL then
+if $data47 != null then
return -1
endi
-if $data48 != NULL then
+if $data48 != null then
return -1
endi
if $data51 != 1 then
@@ -539,27 +528,25 @@ endi
if $data52 != 8 then
return -1
endi
-if $data53 != NULL then
+if $data53 != null then
return -1
endi
-if $data54 != NULL then
+if $data54 != null then
return -1
endi
-if $data55 != NULL then
+if $data55 != null then
return -1
endi
-if $data57 != NULL then
+if $data57 != null then
return -1
endi
-if $data58 != NULL then
+if $data58 != null then
return -1
endi
print ======== step7
-sleep 2500
sql alter table tb drop column h
sql insert into tb values(now-10d, -10)
-sleep 3000
sql select * from tb order by ts desc
if $rows != 7 then
return -1
@@ -568,52 +555,52 @@ print data01 = $data01
if $data01 != -10 then
return -1
endi
-if $data02 != NULL then
+if $data02 != null then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
if $data11 != -13 then
return -1
endi
-if $data12 != NULL then
+if $data12 != null then
return -1
endi
if $data21 != -16 then
return -1
endi
-if $data22 != NULL then
+if $data22 != null then
return -1
endi
if $data31 != -19 then
return -1
endi
-if $data32 != NULL then
+if $data32 != null then
return -1
endi
-if $data33 != NULL then
+if $data33 != null then
return -1
endi
if $data41 != 3 then
return -1
endi
-if $data42 != NULL then
+if $data42 != null then
return -1
endi
if $data51 != 2 then
return -1
endi
-if $data52 != NULL then
+if $data52 != null then
return -1
endi
-if $data53 != NULL then
+if $data53 != null then
return -1
endi
if $data61 != 1 then
return -1
endi
-if $data62 != NULL then
+if $data62 != null then
return -1
endi
@@ -632,52 +619,52 @@ endi
if $data01 != -10 then
return -1
endi
-if $data02 != NULL then
+if $data02 != null then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
if $data11 != -13 then
return -1
endi
-if $data12 != NULL then
+if $data12 != null then
return -1
endi
if $data21 != -16 then
return -1
endi
-if $data22 != NULL then
+if $data22 != null then
return -1
endi
if $data31 != -19 then
return -1
endi
-if $data32 != NULL then
+if $data32 != null then
return -1
endi
-if $data33 != NULL then
+if $data33 != null then
return -1
endi
if $data41 != 3 then
return -1
endi
-if $data42 != NULL then
+if $data42 != null then
return -1
endi
if $data51 != 2 then
return -1
endi
-if $data52 != NULL then
+if $data52 != null then
return -1
endi
-if $data53 != NULL then
+if $data53 != null then
return -1
endi
if $data61 != 1 then
return -1
endi
-if $data62 != NULL then
+if $data62 != null then
return -1
endi
diff --git a/tests/script/general/alter/metrics.sim b/tests/script/general/alter/metrics.sim
index a2c5b7b465302cf6e8d5e9c634ca7d80009a652e..5ed8050b96b90d679fcb85ada06c6065b8f0735a 100644
--- a/tests/script/general/alter/metrics.sim
+++ b/tests/script/general/alter/metrics.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -42,7 +41,6 @@ endi
print ======== step2
sql alter table mt add column b smallint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -71,7 +69,6 @@ endi
print ======== step3
sql alter table mt add column c tinyint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -106,7 +103,6 @@ endi
print ======== step4
sql alter table mt add column d int
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -147,7 +143,6 @@ endi
print ======== step5
sql alter table mt add column e bigint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -194,7 +189,6 @@ endi
print ======== step6
sql alter table mt add column f float
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -247,7 +241,6 @@ endi
print ======== step7
sql alter table mt add column g double
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -306,7 +299,6 @@ endi
print ======== step8
sql alter table mt add column h binary(10)
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -468,7 +460,6 @@ print ======== step11
print ======== step12
sql alter table mt drop column b
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -530,7 +521,6 @@ endi
print ======== step13
sql alter table mt drop column c
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -586,7 +576,6 @@ endi
print ======== step14
sql alter table mt drop column d
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -636,7 +625,6 @@ endi
print ======== step15
sql alter table mt drop column e
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -680,7 +668,6 @@ endi
print ======== step16
sql alter table mt drop column f
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -718,7 +705,6 @@ endi
print ======== step17
sql alter table mt drop column g
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -750,7 +736,6 @@ endi
print ============= step18
sql alter table mt drop column h
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -770,7 +755,7 @@ endi
if $data21 != INT then
return -1
endi
-if $data30 != NULL then
+if $data30 != null then
return -1
endi
diff --git a/tests/script/general/alter/table.sim b/tests/script/general/alter/table.sim
index 25d3b4206655ba1a2f1a3397b397fea770411c5f..5a72fc925619e1a0ab40289056f6af355b995ca9 100644
--- a/tests/script/general/alter/table.sim
+++ b/tests/script/general/alter/table.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -35,7 +34,6 @@ endi
print ======== step2
sql alter table tb add column b smallint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -58,7 +56,6 @@ endi
print ======== step3
sql alter table tb add column c tinyint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -87,7 +84,6 @@ endi
print ======== step4
sql alter table tb add column d int
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -122,7 +118,6 @@ endi
print ======== step5
sql alter table tb add column e bigint
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -163,7 +158,6 @@ endi
print ======== step6
sql alter table tb add column f float
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -210,7 +204,6 @@ endi
print ======== step7
sql alter table tb add column g double
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -263,7 +256,6 @@ endi
print ======== step8
sql alter table tb add column h binary(10)
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -413,7 +405,6 @@ step115:
print ======== step12
sql alter table tb drop column b
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -469,7 +460,6 @@ endi
print ======== step13
sql alter table tb drop column c
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -519,7 +509,6 @@ endi
print ======== step14
sql alter table tb drop column d
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -563,7 +552,6 @@ endi
print ======== step15
sql alter table tb drop column e
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -601,7 +589,6 @@ endi
print ======== step16
sql alter table tb drop column f
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -633,7 +620,6 @@ endi
print ======== step17
sql alter table tb drop column g
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -659,7 +645,6 @@ endi
print ============= step18
sql alter table tb drop column h
-sleep 2500
sql describe tb
if $data00 != ts then
return -1
@@ -673,7 +658,7 @@ endi
if $data11 != INT then
return -1
endi
-if $data20 != NULL then
+if $data20 != null then
return -1
endi
diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim
index 44c12fa968d52b7c6e3811d4997e549af3bba7bf..6b28b20fb87d3983fc93a055505415ffa56ac946 100644
--- a/tests/script/general/parser/alter.sim
+++ b/tests/script/general/parser/alter.sim
@@ -114,46 +114,46 @@ sql drop table tb
sql drop table mt
sleep 3000
-## ALTER TABLE WHILE STREAMING [TBASE271]
-sql create table tb1 (ts timestamp, c1 int, c2 nchar(5), c3 int)
-sql create table strm as select count(*), avg(c1), first(c2), sum(c3) from tb1 interval(2s)
-sql select * from strm
-if $rows != 0 then
- return -1
-endi
-#sleep 12000
-sql insert into tb1 values (now, 1, 'taos', 1)
-sleep 20000
-sql select * from strm
-print rows = $rows
-if $rows != 1 then
- return -1
-endi
-if $data04 != 1 then
- return -1
-endi
-sql alter table tb1 drop column c3
-sleep 6000
-sql insert into tb1 values (now, 2, 'taos')
-sleep 3000
-sql select * from strm
-if $rows != 1 then
- return -1
-endi
-if $data04 != 1 then
- return -1
-endi
-sql alter table tb1 add column c3 int
-sleep 6000
-sql insert into tb1 values (now, 3, 'taos', 3);
-sleep 3000
-sql select * from strm
-if $rows != 1 then
- return -1
-endi
-if $data04 != 1 then
- return -1
-endi
+### ALTER TABLE WHILE STREAMING [TBASE271]
+#sql create table tb1 (ts timestamp, c1 int, c2 nchar(5), c3 int)
+#sql create table strm as select count(*), avg(c1), first(c2), sum(c3) from tb1 interval(2s)
+#sql select * from strm
+#if $rows != 0 then
+# return -1
+#endi
+##sleep 12000
+#sql insert into tb1 values (now, 1, 'taos', 1)
+#sleep 20000
+#sql select * from strm
+#print rows = $rows
+#if $rows != 1 then
+# return -1
+#endi
+#if $data04 != 1 then
+# return -1
+#endi
+#sql alter table tb1 drop column c3
+#sleep 6000
+#sql insert into tb1 values (now, 2, 'taos')
+#sleep 30000
+#sql select * from strm
+#if $rows != 2 then
+# return -1
+#endi
+#if $data04 != 1 then
+# return -1
+#endi
+#sql alter table tb1 add column c3 int
+#sleep 6000
+#sql insert into tb1 values (now, 3, 'taos', 3);
+#sleep 3000
+#sql select * from strm
+#if $rows != 3 then
+# return -1
+#endi
+#if $data04 != 1 then
+# return -1
+#endi
## ALTER TABLE AND INSERT BY COLUMNS
sql create table mt (ts timestamp, c1 int, c2 int) tags(t1 int)
@@ -188,9 +188,9 @@ sql create table tb using mt tags(1)
sleep 3000
sql insert into tb values ('2018-11-01 16:30:00.000', 1, 'insert', 1)
sql alter table mt drop column c3
-# the below query should be deleted after bug fix
+
sql insert into tb values ('2018-11-01 16:29:59.000', 1, 'insert')
-sql_error import into tb values ('2018-11-01 16:29:59.000', 1, 'import')
+sql import into tb values ('2018-11-01 16:29:59.000', 1, 'import')
sql select * from tb order by ts desc
if $data01 != 1 then
return -1
@@ -203,13 +203,13 @@ sql select * from tb order by ts desc
if $data03 != NULL then
return -1
endi
-# the query below should be deleted after bug fix
+
sql reset query cache
sql insert into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3)
-sql_error import into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3)
+sql import into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3)
sql import into tb values ('2018-11-01 16:39:58.000', 2, 'import', 3)
sql select * from tb order by ts desc
-if $rows != 2 then
+if $rows != 4 then
return -1
endi
if $data03 != 3 then
diff --git a/tests/script/general/parser/alter1.sim b/tests/script/general/parser/alter1.sim
index 7c4609a87d79db00d835ee517ecb1b7a0162be24..24cf79eae148acaa822974f1d5d681983f72405a 100644
--- a/tests/script/general/parser/alter1.sim
+++ b/tests/script/general/parser/alter1.sim
@@ -108,11 +108,12 @@ endi
print ================== change a tag value
sql alter table car1 set tag carid=10
-sql describe car1
-if $rows != 7 then
+sql select carId, carmodel from car1
+if $rows != 1 then
return -1
endi
-if $data43 != 10 then
+
+if $data00 != 10 then
return -1
endi
diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim
index 2524fbe63127ff2f6d9e6ca2758d460b79f79e1f..1bce6f1950f7d0f935c4594719e9148a90d54ea6 100644
--- a/tests/script/general/parser/join.sim
+++ b/tests/script/general/parser/join.sim
@@ -258,6 +258,7 @@ sql select count(join_tb1.c3), count(join_tb0.ts) from $tb1 , $tb2 where $ts1 =
$val = 2
if $data00 != $val then
+ print expect 2, actaul: $data00
return -1
endi
@@ -353,6 +354,7 @@ sql select count(*) from join_mt0, join_mt1 where join_mt0.ts = join_mt1.ts and
$val = 20
if $data00 != $val then
+ print expect 20, actual:$data00
return -1
endi
@@ -410,7 +412,7 @@ endi
#======================limit offset===================================
# tag values not int
-sql_error select count(*) from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t2=join_mt1.t2;
+sql_error select count(*) from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t2=join_mt1.t2; #!!!!!
# tag type not identical
sql_error select count(*) from join_mt0, join_mt1 where join_mt1.t2 = join_mt0.t1 and join_mt1.ts=join_mt0.ts;
diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim
index 3e32064f3b0b217daccf2dfefc386470dee94f77..4cf1d36672a23f7c1ed6edaca436883de992721b 100644
--- a/tests/script/general/parser/join_multivnode.sim
+++ b/tests/script/general/parser/join_multivnode.sim
@@ -108,6 +108,7 @@ sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where joi
print $row
if $row != 3000 then
+ print expect 3000, actual: $row
return -1
endi
diff --git a/tests/script/general/parser/limit1_tblocks100.sim b/tests/script/general/parser/limit1_tblocks100.sim
index 99ffd31f6dbb405876c9a00b5d903469ab445a20..29b48bdd43409345caae8cf201d757c2d33d3954 100644
--- a/tests/script/general/parser/limit1_tblocks100.sim
+++ b/tests/script/general/parser/limit1_tblocks100.sim
@@ -68,4 +68,4 @@ print ================== server restart completed
run general/parser/limit1_tb.sim
run general/parser/limit1_stb.sim
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/general/parser/limit2.sim b/tests/script/general/parser/limit2.sim
index d16ee29cf789f2e1fe362ee32db8fc10f823ae3e..d22c786aa3001e6f7e275466bd5426b7e275b53c 100644
--- a/tests/script/general/parser/limit2.sim
+++ b/tests/script/general/parser/limit2.sim
@@ -69,8 +69,6 @@ print ====== tables created
print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT
-
-return
sleep 5000
system sh/exec.sh -n dnode1 -s start
print ================== server restart completed
diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim
index 6da419cd4c07a298996563089d86451d732c3883..09e761c85e68b1bc7a4464901080353936a4d8e5 100644
--- a/tests/script/general/parser/null_char.sim
+++ b/tests/script/general/parser/null_char.sim
@@ -206,68 +206,70 @@ endi
################# binary
sql alter table st41 set tag tag_binary = "shanghai"
-sql describe st41
-if $data23 != shanghai then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data00 != shanghai then
return -1
endi
+
##### test 'space' case
-#$tagvalue = '
-#$tagvalue = $tagvalue '
-#sql alter table st41 set tag tag_binary = $tagvalue
+system_content echo ' ' | sed 's/ //g' | tr -d '\n' # Construct an empty result for later result checking
sql alter table st41 set tag tag_binary = ""
-#sql describe st41
-#if $data23 != $tagvalue then
-# return -1
-#endi
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data00 != $system_content then
+ print expect [ $system_content ], actual [ $data00 ]
+ return -1
+endi
+
sql alter table st41 set tag tag_binary = "NULL"
-sql describe st41
-if $data23 != NULL then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data00 != NULL then
return -1
endi
+
sql alter table st41 set tag tag_binary = NULL
-sql describe st41
-if $data23 != NULL then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data00 != NULL then
print ==8== expect: NULL, actually: $data23
return -1
endi
################### nchar
sql alter table st41 set tag tag_nchar = "��˼����"
-sql describe st41
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
#sleep 1000
-#if $data33 != ��˼���� then
-# print ==== expect ��˼����, actually $data33
+#if $data01 != ��˼���� then
+# print ==== expect ��˼����, actually $data01
# return -1
#endi
##### test 'space' case
#$tagvalue = '
#$tagvalue = $tagvalue '
sql alter table st41 set tag tag_nchar = ''
-#sql describe st41
-#if $data33 != $tagvalue then
+#sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+#if $data01 != $tagvalue then
# return -1
#endi
sql alter table st41 set tag tag_nchar = "NULL"
-sql describe st41
-if $data33 != NULL then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data01 != NULL then
return -1
endi
sql alter table st41 set tag tag_nchar = NULL
-#sql describe st41
-#if $data33 != then
-# print ==9== expect , actually $data33
+#sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+#if $data01 != then
+# print ==9== expect , actually $data01
# return -1
#endi
################### int
sql alter table st41 set tag tag_int = -2147483647
-sql describe st41
-if $data43 != -2147483647 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data02 != -2147483647 then
return -1
endi
sql alter table st41 set tag tag_int = 2147483647
-sql describe st41
-if $data43 != 2147483647 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data02 != 2147483647 then
return -1
endi
@@ -275,19 +277,19 @@ sql_error alter table st41 set tag tag_int = -2147483648
sql_error alter table st41 set tag tag_int = 2147483648
sql alter table st41 set tag tag_int = '-379'
-sql describe st41
-if $data43 != -379 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data02 != -379 then
return -1
endi
sql alter table st41 set tag tag_int = -2000
-sql describe st41
-if $data43 != -2000 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data02 != -2000 then
return -1
endi
sql alter table st41 set tag tag_int = NULL
-sql describe st41
-if $data43 != NULL then
- print ==10== expect: NULL, actually: $data43
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data02 != NULL then
+ print ==10== expect: NULL, actually: $data02
return -1
endi
sql alter table st41 set tag tag_int = 'NULL'
@@ -296,34 +298,34 @@ sql_error alter table st41 set tag tag_int = abc379
################### bool
sql alter table st41 set tag tag_bool = 'true'
-sql describe st41
-if $data53 != true then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != 1 then
return -1
endi
sql alter table st41 set tag tag_bool = 'false'
-sql describe st41
-if $data53 != false then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != 0 then
return -1
endi
sql alter table st41 set tag tag_bool = 0
-sql describe st41
-if $data53 != false then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != 0 then
return -1
endi
sql alter table st41 set tag tag_bool = 123
-sql describe st41
-if $data53 != true then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != 1 then
return -1
endi
sql alter table st41 set tag tag_bool = 'NULL'
-sql describe st41
-if $data53 != NULL then
- print ==14== expect: NULL, actually: $data53
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != NULL then
+ print ==14== expect: NULL, actually: $data03
return -1
endi
sql alter table st41 set tag tag_bool = NULL
-sql describe st41
-if $data53 != NULL then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data03 != NULL then
return -1
endi
@@ -333,50 +335,51 @@ sql_error alter table st41 set tag tag_bool = abc379
################### float
sql alter table st41 set tag tag_float = -32
-sql describe st41
-if $data63 != -32.000000 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != -32.00000 then
+ print expect -32.00000 actual $data04
return -1
endi
sql alter table st41 set tag tag_float = 54.123456
-sql describe st41
-if $data63 != 54.123455 then
- print ==15== expect: 54.123455, actually: $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != 54.123455 then
+ print ==15== expect: 54.123455, actually: $data04
# return -1
endi
sql alter table st41 set tag tag_float = 54.12345
-sql describe st41
-if $data63 != 54.123451 then
- print ==16== expect: 54.123451, actually: $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != 54.12345 then
+ print ==16== expect: 54.12345, actually: $data04
return -1
endi
sql alter table st41 set tag tag_float = 54.12345678
-sql describe st41
-if $data63 != 54.123455 then
- print ==11== expect: 54.123455, actually : $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != 54.12346 then
+ print ==11== expect: 54.12346, actually : $data04
return -1
endi
sql alter table st41 set tag tag_float = NULL
-sql describe st41
-if $data63 != NULL then
- print ==12== expect: NULL, actually : $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != NULL then
+ print ==12== expect: NULL, actually : $data04
return -1
endi
sql alter table st41 set tag tag_float = 'NULL'
-sql describe st41
-if $data63 != NULL then
- print ==17== expect: NULL, actually : $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != NULL then
+ print ==17== expect: NULL, actually : $data04
return -1
endi
sql alter table st41 set tag tag_float = '54.123456'
-sql describe st41
-if $data63 != 54.123455 then
- print ==18== expect: 54.123455, actually : $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != 54.12346 then
+ print ==18== expect: 54.12346, actually : $data04
return -1
endi
sql alter table st41 set tag tag_float = '-54.123456'
-sql describe st41
-if $data63 != -54.123455 then
- print ==19== expect: -54.123455, actually : $data63
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data04 != -54.12346 then
+ print ==19== expect: -54.12346, actually : $data04
return -1
endi
sql_error alter table st41 set tag tag_float = ''
@@ -387,30 +390,32 @@ sql_error alter table st41 set tag tag_float = abc
################### double
sql alter table st41 set tag tag_double = -92
-sql describe st41
-if $data73 != -92.000000 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data05 != -92.000000000 then
+ print expect -92.000000000 actual $data05
return -1
endi
sql alter table st41 set tag tag_double = 184
-sql describe st41
-if $data73 != 184.000000 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data05 != 184.000000000 then
+ print expect 184.000000000 actual $data05
return -1
endi
sql alter table st41 set tag tag_double = '-2456'
-sql describe st41
-if $data73 != -2456.000000 then
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data05 != -2456.000000000 then
return -1
endi
sql alter table st41 set tag tag_double = NULL
-sql describe st41
-if $data73 != NULL then
- print ==13== expect: NULL, actually : $data73
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data05 != NULL then
+ print ==13== expect: NULL, actually : $data05
return -1
endi
sql alter table st41 set tag tag_double = 'NULL'
-sql describe st41
-if $data73 != NULL then
- print ==20== expect: NULL, actually : $data73
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
+if $data05 != NULL then
+ print ==20== expect: NULL, actually : $data05
return -1
endi
sql_error alter table st41 set tag tag_double = ''
@@ -427,23 +432,22 @@ sql alter table st51 set tag tag_bigint = '-379'
sql alter table st51 set tag tag_bigint = -2000
sql alter table st51 set tag tag_bigint = NULL
sql alter table st51 set tag tag_bigint = 9223372036854775807
-sql describe st51
-if $data23 != 9223372036854775807 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data00 != 9223372036854775807 then
return -1
endi
sql alter table st51 set tag tag_bigint = 9223372036854775808
-sql describe st51
-if $data23 != 9223372036854775807 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data00 != 9223372036854775807 then
return -1
endi
sql alter table st51 set tag tag_bigint = -9223372036854775807
-sql describe st51
-if $data23 != -9223372036854775807 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data00 != -9223372036854775807 then
return -1
endi
sql_error alter table st51 set tag tag_bigint = -9223372036854775808
-
sql alter table st51 set tag tag_bigint = 'NULL'
sql_error alter table st51 set tag tag_bigint = ''
sql_error alter table st51 set tag tag_bigint = abc379
@@ -452,15 +456,15 @@ sql_error alter table st51 set tag tag_bigint = abc379
sql alter table st51 set tag tag_smallint = -2000
sql alter table st51 set tag tag_smallint = NULL
sql alter table st51 set tag tag_smallint = 32767
-sql describe st51
-if $data33 != 32767 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data01 != 32767 then
return -1
endi
sql_error alter table st51 set tag tag_smallint = 32768
sql alter table st51 set tag tag_smallint = -32767
-sql describe st51
-if $data33 != -32767 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data01 != -32767 then
return -1
endi
sql_error alter table st51 set tag tag_smallint = -32768
@@ -473,13 +477,13 @@ sql_error alter table st51 set tag tag_smallint = abc379
sql alter table st51 set tag tag_tinyint = -127
sql alter table st51 set tag tag_tinyint = NULL
sql alter table st51 set tag tag_tinyint = 127
-sql describe st51
-if $data43 != 127 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data02 != 127 then
return -1
endi
sql alter table st51 set tag tag_tinyint = -127
-sql describe st51
-if $data43 != -127 then
+sql select tag_bigint, tag_smallint, tag_tinyint from st51
+if $data02 != -127 then
return -1
endi
sql_error alter table st51 set tag tag_tinyint = '-128'
@@ -493,4 +497,4 @@ sql_error alter table st51 set tag tag_tinyint = abc379
#sql drop database $db
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim
index a684fd5406950ede213ad065a45a7aa79d355818..9d7ac9678daefde512ac9d126742b43aa3508017 100644
--- a/tests/script/general/parser/set_tag_vals.sim
+++ b/tests/script/general/parser/set_tag_vals.sim
@@ -9,7 +9,7 @@ sql connect
$dbPrefix = db
$tbPrefix = tb
$stbPrefix = stb
-$tbNum = 1000
+$tbNum = 100
$rowNum = 100
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
@@ -26,7 +26,7 @@ $stb = $stbPrefix . $i
sql drop database $db -x step1
step1:
sql create database $db
-print ====== create tables
+print ====== create $tbNum tables
sql use $db
sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 binary(9), t2 binary(8))
diff --git a/tests/script/general/parser/slimit_alter_tags.sim b/tests/script/general/parser/slimit_alter_tags.sim
index 54bc868f6a975ede6f1031a9560b444bd491acad..e8e81e8809b5ef0e53a767a4f7e8621dcbd7aee0 100644
--- a/tests/script/general/parser/slimit_alter_tags.sim
+++ b/tests/script/general/parser/slimit_alter_tags.sim
@@ -84,11 +84,13 @@ while $i < 10
sql alter table $tb set tag tg_added = $tg_added
$i = $i + 1
endw
-sql describe tb0
-if $rows != 7 then
+
+sql select t1,t2,tg_added from tb0
+if $rows != 1 then
return -1
endi
-if $data63 != tb0 then
+
+if $data02 != tb0 then
return -1
endi
diff --git a/tests/script/general/parser/stream.sim b/tests/script/general/parser/stream.sim
index fca5f37d74ce69bd89eb7118b4c19e52f603fcee..2f233b618942cd1dc708ee45b05d0e78326f8c62 100644
--- a/tests/script/general/parser/stream.sim
+++ b/tests/script/general/parser/stream.sim
@@ -192,7 +192,7 @@ sql create table tb4 using stb tags(4, 'tb4')
sql create table strm0 as select count(ts), count(c1), max(c2), min(c4), first(c5), last(c6) from stb where ts < now + 30s interval(4s) sliding(2s)
sleep 1000
sql insert into tb0 values (now, 0, 0, 0, 0, 'binary0', '涛思0', true) tb1 values (now, 1, 1, 1, 1, 'binary1', '涛思1', false) tb2 values (now, 2, 2, 2, 2, 'binary2', '涛思2', true) tb3 values (now, 3, 3, 3, 3, 'binary3', '涛思3', false) tb4 values (now, 4, 4, 4, 4, 'binary4', '涛思4', true)
-sleep 20000
+sleep 30000
sql select * from strm0 order by ts desc
sleep 1000
if $rows != 2 then
@@ -203,9 +203,9 @@ endi
sql insert into tb0 values (now, 10, 10, 10, 10, 'binary0', '涛思0', true) tb1 values (now, 11, 11, 11, 11, 'binary1', '涛思1', false) tb2 values (now, 12, 12, 12, 12, 'binary2', '涛思2', true) tb3 values (now, 13, 13, 13, 13, 'binary3', '涛思3', false) tb4 values (now, 14, 14, 14, 14, 'binary4', '涛思4', true)
sleep 30000
sql select * from strm0 order by ts desc
-sleep 1000
-if $rows != 4 then
- print ==== expect rows = 4, actually returned rows = $rows
+sleep 10000
+if $rows == 4 then
+ print ==== actually returned rows = $rows, expect always not equal to 4
return -1
endi
diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim
index 5170ddb036495e958ccf0eb0b819cf85441a1100..5405e09256242ce18bdb75099283d35497f7fa85 100644
--- a/tests/script/general/parser/stream_on_sys.sim
+++ b/tests/script/general/parser/stream_on_sys.sim
@@ -22,12 +22,12 @@ $i = 0
sql use $db
-sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn_192_168_0_1 interval(4s) sliding(2s)
-sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn_192_168_0_1 interval(4s) sliding(2s)
-sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn_192_168_0_1 interval(4s) sliding(2s)
-sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn_192_168_0_1 interval(4s) sliding(2s)
-sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn_192_168_0_1 interval(4s) sliding(2s)
-sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn_192_168_0_1 interval(4s) sliding(2s)
+sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) sliding(2s)
+sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) sliding(2s)
+sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) sliding(2s)
+sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s)
+sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s)
+sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s)
sleep 20000
sql select * from cpustrm
if $rows <= 0 then
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index 67ef952c41c39ac474d9b1ae7f23f88e9a593033..31acaad1432559662584d777027ec1082a4e7373 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -1,45 +1,45 @@
-#run general/parser/alter.sim
-#sleep 2000
-#run general/parser/alter1.sim
-#sleep 2000
-#run general/parser/alter_stable.sim
-#sleep 2000
-#run general/parser/auto_create_tb.sim
-#sleep 2000
-#run general/parser/auto_create_tb_drop_tb.sim
+run general/parser/alter.sim
+sleep 2000
+run general/parser/alter1.sim
+sleep 2000
+run general/parser/alter_stable.sim
+sleep 2000
+run general/parser/auto_create_tb.sim
+sleep 2000
+run general/parser/auto_create_tb_drop_tb.sim
-#sleep 2000
-#run general/parser/col_arithmetic_operation.sim
-#sleep 2000
-#run general/parser/columnValue.sim
-#sleep 2000
-#run general/parser/commit.sim
-#sleep 2000
-#run general/parser/create_db.sim
-#sleep 2000
-#run general/parser/create_mt.sim
-#sleep 2000
-#run general/parser/create_tb.sim
-#sleep 2000
-#run general/parser/dbtbnameValidate.sim
-#sleep 2000
-#run general/parser/import_commit1.sim
-#sleep 2000
-#run general/parser/import_commit2.sim
-#sleep 2000
-#run general/parser/import_commit3.sim
-#sleep 2000
-#run general/parser/insert_tb.sim
-#sleep 2000
-#run general/parser/first_last.sim
-#sleep 2000
-##run general/parser/import_file.sim
-#sleep 2000
-#run general/parser/lastrow.sim
-#sleep 2000
-#run general/parser/nchar.sim
-#sleep 2000
-##run general/parser/null_char.sim
+sleep 2000
+run general/parser/col_arithmetic_operation.sim
+sleep 2000
+run general/parser/columnValue.sim
+sleep 2000
+run general/parser/commit.sim
+sleep 2000
+run general/parser/create_db.sim
+sleep 2000
+run general/parser/create_mt.sim
+sleep 2000
+run general/parser/create_tb.sim
+sleep 2000
+run general/parser/dbtbnameValidate.sim
+sleep 2000
+run general/parser/import_commit1.sim
+sleep 2000
+run general/parser/import_commit2.sim
+sleep 2000
+run general/parser/import_commit3.sim
+sleep 2000
+run general/parser/insert_tb.sim
+sleep 2000
+run general/parser/first_last.sim
+sleep 2000
+#run general/parser/import_file.sim
+sleep 2000
+run general/parser/lastrow.sim
+sleep 2000
+run general/parser/nchar.sim
+sleep 2000
+#run general/parser/null_char.sim
sleep 2000
run general/parser/single_row_in_tb.sim
sleep 2000
@@ -80,26 +80,24 @@ sleep 2000
run general/parser/tags_dynamically_specifiy.sim
sleep 2000
run general/parser/groupby.sim
-
sleep 2000
run general/parser/set_tag_vals.sim
-
-sleep 2000
-run general/parser/slimit_alter_tags.sim
sleep 2000
-run general/parser/stream_on_sys.sim
-sleep 2000
-run general/parser/stream.sim
-
-sleep 2000
-#run general/parser/repeatAlter.sim
-sleep 2000
-#run general/parser/repeatStream.sim
+run general/parser/slimit_alter_tags.sim # persistent failed
sleep 2000
run general/parser/join.sim
sleep 2000
run general/parser/join_multivnode.sim
sleep 2000
+run general/parser/repeatAlter.sim
+sleep 2000
run general/parser/binary_escapeCharacter.sim
sleep 2000
-#run general/parser/bug.sim
\ No newline at end of file
+run general/parser/bug.sim
+
+#sleep 2000
+#run general/parser/repeatStream.sim
+#sleep 2000
+#run general/parser/stream_on_sys.sim
+#sleep 2000
+#run general/parser/stream.sim
\ No newline at end of file
diff --git a/tests/script/general/stream/new_stream.sim b/tests/script/general/stream/new_stream.sim
index abc06faaef7418e1d86450831853c9e8ad65e434..001602079b367163c1709398da7d23eacb1f9f23 100644
--- a/tests/script/general/stream/new_stream.sim
+++ b/tests/script/general/stream/new_stream.sim
@@ -1,12 +1,11 @@
-#system sh/stop_dnodes.sh
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
+system sh/exec.sh -n dnode1 -s start
-#system sh/deploy.sh -n dnode1 -i 1
-#system sh/cfg.sh -n dnode1 -c walLevel -v 0
-#system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
-#system sh/exec.sh -n dnode1 -s start
-
-#sleep 3000
+sleep 3000
sql connect
print ======================== dnode1 start
diff --git a/tests/script/general/tag/add.sim b/tests/script/general/tag/add.sim
index 69fd909dcf33f0a268fa306955c4beae5da792b6..301ec4f825c9954f9b5790c5e27e417538c3b4e2 100644
--- a/tests/script/general/tag/add.sim
+++ b/tests/script/general/tag/add.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -138,16 +137,17 @@ if $data03 != 2.00000 then
endi
sql describe $tb
+print sql describe $tb
if $data21 != BIGINT then
return -1
endi
if $data31 != FLOAT then
return -1
endi
-if $data23 != 1 then
+if $data23 != TAG then
return -1
endi
-if $data33 != 2.000000 then
+if $data33 != TAG then
return -1
endi
@@ -545,7 +545,7 @@ endi
if $data04 != 0 then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
diff --git a/tests/script/general/tag/change.sim b/tests/script/general/tag/change.sim
index 1577514765e9a851b659d3a1568d99857a2e65c6..a7b8554bd96f6e8857d9b6a6b2de4faa4459bf2e 100644
--- a/tests/script/general/tag/change.sim
+++ b/tests/script/general/tag/change.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -51,9 +50,9 @@ step21:
sql alter table $mt change tag tgcol1 tgcol2 -x step22
return -1
step22:
-sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20
- return -1
-step20:
+#sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20
+# return -1
+#step20:
sql alter table $mt change tag tgcol1 tgcol3
sql alter table $mt change tag tgcol2 tgcol4
@@ -394,7 +393,7 @@ endi
if $data06 != 6 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -421,7 +420,7 @@ endi
if $data06 != 6 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -448,7 +447,7 @@ endi
if $data06 != 6 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -475,7 +474,7 @@ endi
if $data06 != 6 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -502,7 +501,7 @@ endi
if $data06 != 6 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
diff --git a/tests/script/general/tag/commit.sim b/tests/script/general/tag/commit.sim
index fca94ee7d41b58c40128a2bf2774782572c83934..8b07d98afbaf343056ca94955e1b8977e1c46fe1 100644
--- a/tests/script/general/tag/commit.sim
+++ b/tests/script/general/tag/commit.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -144,10 +143,10 @@ endi
if $data31 != FLOAT then
return -1
endi
-if $data23 != 1 then
+if $data23 != TAG then
return -1
endi
-if $data33 != 2.000000 then
+if $data33 != TAG then
return -1
endi
@@ -544,7 +543,7 @@ endi
if $data04 != 0 then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
@@ -1054,7 +1053,7 @@ endi
if $data04 != 0 then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
diff --git a/tests/script/general/tag/delete.sim b/tests/script/general/tag/delete.sim
index 3454ce9d822b74bf24430f227217b4e545adcdba..f3d0735b5c2c158b1bbc9725755c7349a474178a 100644
--- a/tests/script/general/tag/delete.sim
+++ b/tests/script/general/tag/delete.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -98,7 +97,7 @@ endi
if $data31 != FLOAT then
return -1
endi
-if $data23 != 1 then
+if $data23 != TAG then
return -1
endi
@@ -203,13 +202,13 @@ endi
if $data42 != 10 then
return -1
endi
-if $data23 != 1 then
+if $data23 != TAG then
return -1
endi
-if $data33 != 2 then
+if $data33 != TAG then
return -1
endi
-if $data43 != 3 then
+if $data43 != TAG then
return -1
endi
@@ -428,7 +427,7 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
@@ -452,7 +451,7 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
@@ -476,7 +475,7 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
@@ -500,7 +499,7 @@ endi
if $data02 != 1.000000000 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
@@ -524,10 +523,10 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
@@ -554,10 +553,10 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
@@ -584,10 +583,10 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
@@ -614,10 +613,10 @@ endi
if $data02 != 1.000000000 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
@@ -644,13 +643,13 @@ endi
if $data02 != 1 then
return -1
endi
-if $data03 != NULL then
+if $data03 != null then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
@@ -683,13 +682,13 @@ endi
if $data03 != 4.00000 then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
@@ -722,16 +721,16 @@ endi
if $data03 != 4.000000000 then
return -1
endi
-if $data04 != NULL then
+if $data04 != null then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -771,13 +770,13 @@ endi
if $data04 != 5.000000000 then
return -1
endi
-if $data05 != NULL then
+if $data05 != null then
return -1
endi
-if $data06 != NULL then
+if $data06 != null then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
diff --git a/tests/script/general/tag/set.sim b/tests/script/general/tag/set.sim
index 917844c3e3272c9983d7bd24caeebdf92e821273..112695d37b0a924779812485e22ffb5680a8856e 100644
--- a/tests/script/general/tag/set.sim
+++ b/tests/script/general/tag/set.sim
@@ -1,8 +1,7 @@
system sh/stop_dnodes.sh
-
system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/exec.sh -n dnode1 -s start
sleep 3000
@@ -91,10 +90,10 @@ endi
if $data31 != INT then
return -1
endi
-if $data23 != false then
+if $data23 != TAG then
return -1
endi
-if $data33 != 4 then
+if $data33 != TAG then
return -1
endi
@@ -338,7 +337,7 @@ endi
if $data06 != 11 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -365,7 +364,7 @@ endi
if $data06 != 11 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -392,7 +391,7 @@ endi
if $data06 != 11 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -419,7 +418,7 @@ endi
if $data06 != 11 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
@@ -446,7 +445,7 @@ endi
if $data06 != 11 then
return -1
endi
-if $data07 != NULL then
+if $data07 != null then
return -1
endi
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index fe91e4bbde0c92373e3ac05e3af4ee2c7fa568b8..9cb9dc594e06f294c4e9be5f9ef92e1d904147e5 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -1,13 +1,13 @@
cd ../../../debug; cmake ..
cd ../../../debug; make
-#unsupport ./test.sh -f general/alter/cached_schema_after_alter.sim
-#unsupport ./test.sh -f general/alter/count.sim
-#unsupport ./test.sh -f general/alter/import.sim
-#unsupport ./test.sh -f general/alter/insert1.sim
-#unsupport ./test.sh -f general/alter/insert2.sim
-#unsupport ./test.sh -f general/alter/metrics.sim
-#unsupport ./test.sh -f general/alter/table.sim
+#./test.sh -f general/alter/cached_schema_after_alter.sim
+./test.sh -f general/alter/count.sim
+./test.sh -f general/alter/import.sim
+#./test.sh -f general/alter/insert1.sim
+./test.sh -f general/alter/insert2.sim
+./test.sh -f general/alter/metrics.sim
+./test.sh -f general/alter/table.sim
./test.sh -f general/cache/new_metrics.sim
./test.sh -f general/cache/restart_metrics.sim
@@ -100,9 +100,9 @@ cd ../../../debug; make
./test.sh -f general/insert/query_multi_file.sim
./test.sh -f general/insert/tcp.sim
-#unsupport ./test.sh -f general/parser/alter.sim
-#unsupport ./test.sh -f general/parser/alter1.sim
-#unsupport ./test.sh -f general/parser/alter_stable.sim
+./test.sh -f general/parser/alter.sim
+./test.sh -f general/parser/alter1.sim
+./test.sh -f general/parser/alter_stable.sim
./test.sh -f general/parser/auto_create_tb.sim
./test.sh -f general/parser/auto_create_tb_drop_tb.sim
./test.sh -f general/parser/col_arithmetic_operation.sim
@@ -117,40 +117,40 @@ cd ../../../debug; make
./test.sh -f general/parser/import_commit3.sim
./test.sh -f general/parser/insert_tb.sim
./test.sh -f general/parser/first_last.sim
-#unsupport ./test.sh -f general/parser/import_file.sim
+#./test.sh -f general/parser/import_file.sim
./test.sh -f general/parser/lastrow.sim
./test.sh -f general/parser/nchar.sim
-#unsupport ./test.sh -f general/parser/null_char.sim
+./test.sh -f general/parser/null_char.sim
./test.sh -f general/parser/single_row_in_tb.sim
./test.sh -f general/parser/select_from_cache_disk.sim
+./test.sh -f general/parser/mixed_blocks.sim
+./test.sh -f general/parser/selectResNum.sim
./test.sh -f general/parser/limit.sim
./test.sh -f general/parser/limit1.sim
./test.sh -f general/parser/limit1_tblocks100.sim
-./test.sh -f general/parser/mixed_blocks.sim
-./test.sh -f general/parser/selectResNum.sim
./test.sh -f general/parser/select_across_vnodes.sim
./test.sh -f general/parser/slimit1.sim
./test.sh -f general/parser/tbnameIn.sim
-./test.sh -f general/parser/binary_escapeCharacter.sim
./test.sh -f general/parser/projection_limit_offset.sim
./test.sh -f general/parser/limit2.sim
-./test.sh -f general/parser/slimit.sim
./test.sh -f general/parser/fill.sim
./test.sh -f general/parser/fill_stb.sim
-./test.sh -f general/parser/interp.sim
./test.sh -f general/parser/where.sim
-#unsupport ./test.sh -f general/parser/join.sim
-#unsupport ./test.sh -f general/parser/join_multivnode.sim
+./test.sh -f general/parser/slimit.sim
./test.sh -f general/parser/select_with_tags.sim
-#unsupport ./test.sh -f general/parser/groupby.sim
-#unsupport ./test.sh -f general/parser/bug.sim
-#unsupport ./test.sh -f general/parser/tags_dynamically_specifiy.sim
-#unsupport ./test.sh -f general/parser/set_tag_vals.sim
-#unsupport ./test.sh -f general/parser/repeatAlter.sim
-#unsupport ./test.sh -f general/parser/slimit_alter_tags.sim
-#unsupport ./test.sh -f general/parser/stream_on_sys.sim
-#unsupport ./test.sh -f general/parser/stream.sim
-#unsupport ./test.sh -f general/parser/repeatStream.sim
+./test.sh -f general/parser/interp.sim
+./test.sh -f general/parser/tags_dynamically_specifiy.sim
+./test.sh -f general/parser/groupby.sim
+./test.sh -f general/parser/set_tag_vals.sim
+#./test.sh -f general/parser/slimit_alter_tags.sim
+./test.sh -f general/parser/join.sim
+./test.sh -f general/parser/join_multivnode.sim
+./test.sh -f general/parser/binary_escapeCharacter.sim
+./test.sh -f general/parser/bug.sim
+#./test.sh -f general/parser/stream_on_sys.sim
+./test.sh -f general/parser/stream.sim
+./test.sh -f general/parser/repeatAlter.sim
+#./test.sh -f general/parser/repeatStream.sim
./test.sh -f general/stable/disk.sim
./test.sh -f general/stable/dnode3.sim
@@ -159,21 +159,6 @@ cd ../../../debug; make
./test.sh -f general/stable/values.sim
./test.sh -f general/stable/vnode3.sim
-#./test.sh -f general/stream/metrics_1.sim
-#./test.sh -f general/stream/metrics_del.sim
-#./test.sh -f general/stream/metrics_n.sim
-#./test.sh -f general/stream/metrics_replica1_vnoden.sim
-#./test.sh -f general/stream/new_stream.sim
-#./test.sh -f general/stream/restart_stream.sim
-#./test.sh -f general/stream/stream_1.sim
-#./test.sh -f general/stream/stream_2.sim
-#./test.sh -f general/stream/stream_3.sim
-#./test.sh -f general/stream/stream_restart.sim
-#./test.sh -f general/stream/table_1.sim
-#./test.sh -f general/stream/table_del.sim
-#./test.sh -f general/stream/table_n.sim
-#./test.sh -f general/stream/table_replica1_vnoden.sim
-
./test.sh -f general/table/autocreate.sim
./test.sh -f general/table/basic1.sim
./test.sh -f general/table/basic2.sim
@@ -206,25 +191,25 @@ cd ../../../debug; make
./test.sh -f general/tag/4.sim
./test.sh -f general/tag/5.sim
./test.sh -f general/tag/6.sim
-#unsupport ./test.sh -f general/tag/add.sim
+./test.sh -f general/tag/add.sim
./test.sh -f general/tag/bigint.sim
./test.sh -f general/tag/binary_binary.sim
./test.sh -f general/tag/binary.sim
./test.sh -f general/tag/bool_binary.sim
./test.sh -f general/tag/bool_int.sim
./test.sh -f general/tag/bool.sim
-#unsupport ./test.sh -f general/tag/change.sim
+./test.sh -f general/tag/change.sim
./test.sh -f general/tag/column.sim
-#unsupport ./test.sh -f general/tag/commit.sim
+#./test.sh -f general/tag/commit.sim
./test.sh -f general/tag/create.sim
-#unsupport ./test.sh -f general/tag/delete.sim
+./test.sh -f general/tag/delete.sim
./test.sh -f general/tag/double.sim
./test.sh -f general/tag/filter.sim
./test.sh -f general/tag/float.sim
./test.sh -f general/tag/int_binary.sim
./test.sh -f general/tag/int_float.sim
./test.sh -f general/tag/int.sim
-#unsupport ./test.sh -f general/tag/set.sim
+./test.sh -f general/tag/set.sim
./test.sh -f general/tag/smallint.sim
./test.sh -f general/tag/tinyint.sim
@@ -323,8 +308,23 @@ cd ../../../debug; make
./test.sh -f unique/vnode/replica3_repeat.sim
./test.sh -f unique/vnode/replica3_vgroup.sim
+./test.sh -f general/stream/metrics_1.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+#./test.sh -f general/stream/new_stream.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_1.sim
+./test.sh -f general/stream/stream_2.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim
+
+./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
-#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync_second.sim
./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
@@ -336,6 +336,10 @@ cd ../../../debug; make
./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
./test.sh -f unique/arbitrator/insert_duplicationTs.sim
@@ -358,3 +362,4 @@ cd ../../../debug; make
./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim
+
diff --git a/tests/script/jenkins/general.txt b/tests/script/jenkins/general.txt
deleted file mode 100644
index 0c14e1c7a0ffd9bf01b11312e93320dbd883380d..0000000000000000000000000000000000000000
--- a/tests/script/jenkins/general.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-cd ../../debug; cmake ..
-cd ../../debug; make
-
-./test.sh -f general/db/dropdnodes.sim
-./test.sh -f general/user/basic1.sim
diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt
index ae48994c2a1e61f7c4933989a83574b0a1aab2c2..0ee98aa1f9f003bf440a2cfdd2ad016f2f461c80 100644
--- a/tests/script/jenkins/simple.txt
+++ b/tests/script/jenkins/simple.txt
@@ -1,48 +1,267 @@
cd ../../../debug; cmake ..
cd ../../../debug; make
-#./test.sh -f general/parser/lastrow.sim
-#./test.sh -f general/parser/nchar.sim
-#./test.sh -f general/parser/limit.sim
-#./test.sh -f general/parser/limit1.sim
-#./test.sh -f general/parser/limit1_tblocks100.sim
-#./test.sh -f general/parser/binary_escapeCharacter.sim
-#./test.sh -f general/parser/projection_limit_offset.sim
-#./test.sh -f general/parser/limit2.sim
-#./test.sh -f general/stable/metrics.sim
-#./test.sh -f general/table/date.sim
-#./test.sh -f unique/big/balance.sim
-#./test.sh -f unique/column/replica3.sim
-#./test.sh -f unique/db/commit.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete.sim
+
+#./test.sh -f general/alter/cached_schema_after_alter.sim
+./test.sh -f general/alter/count.sim
+./test.sh -f general/alter/import.sim
+#./test.sh -f general/alter/insert1.sim
+./test.sh -f general/alter/insert2.sim
+./test.sh -f general/alter/metrics.sim
+./test.sh -f general/alter/table.sim
+
+./test.sh -f general/cache/new_metrics.sim
+./test.sh -f general/cache/restart_metrics.sim
+./test.sh -f general/cache/restart_table.sim
+
+./test.sh -f general/connection/connection.sim
+
+./test.sh -f general/column/commit.sim
+./test.sh -f general/column/metrics.sim
+./test.sh -f general/column/table.sim
+
+./test.sh -f general/compress/commitlog.sim
+./test.sh -f general/compress/compress.sim
+./test.sh -f general/compress/compress2.sim
+./test.sh -f general/compress/uncompress.sim
+
+./test.sh -f general/compute/avg.sim
+./test.sh -f general/compute/bottom.sim
+./test.sh -f general/compute/count.sim
+./test.sh -f general/compute/diff.sim
+./test.sh -f general/compute/diff2.sim
+./test.sh -f general/compute/first.sim
+./test.sh -f general/compute/interval.sim
+./test.sh -f general/compute/last.sim
+./test.sh -f general/compute/leastsquare.sim
+./test.sh -f general/compute/max.sim
+./test.sh -f general/compute/min.sim
+./test.sh -f general/compute/null.sim
+./test.sh -f general/compute/percentile.sim
+./test.sh -f general/compute/stddev.sim
+./test.sh -f general/compute/sum.sim
+./test.sh -f general/compute/top.sim
+
+./test.sh -f general/db/alter_option.sim
+./test.sh -f general/db/alter_tables_d2.sim
+./test.sh -f general/db/alter_tables_v1.sim
+./test.sh -f general/db/alter_tables_v4.sim
+./test.sh -f general/db/alter_vgroups.sim
+./test.sh -f general/db/basic.sim
+./test.sh -f general/db/basic1.sim
+./test.sh -f general/db/basic2.sim
+./test.sh -f general/db/basic3.sim
+./test.sh -f general/db/basic4.sim
+./test.sh -f general/db/basic5.sim
+./test.sh -f general/db/delete_writing2.sim
+./test.sh -f general/db/delete.sim
+./test.sh -f general/db/len.sim
+./test.sh -f general/db/repeat.sim
+./test.sh -f general/db/tables.sim
+./test.sh -f general/db/vnodes.sim
+
+./test.sh -f general/field/2.sim
+./test.sh -f general/field/3.sim
+./test.sh -f general/field/4.sim
+./test.sh -f general/field/5.sim
+./test.sh -f general/field/6.sim
+./test.sh -f general/field/bigint.sim
+./test.sh -f general/field/binary.sim
+./test.sh -f general/field/bool.sim
+./test.sh -f general/field/single.sim
+./test.sh -f general/field/smallint.sim
+./test.sh -f general/field/tinyint.sim
+
+./test.sh -f general/http/restful.sim
+./test.sh -f general/http/restful_insert.sim
+./test.sh -f general/http/restful_limit.sim
+./test.sh -f general/http/restful_full.sim
+./test.sh -f general/http/prepare.sim
+./test.sh -f general/http/telegraf.sim
+./test.sh -f general/http/grafana_bug.sim
+./test.sh -f general/http/grafana.sim
+
+./test.sh -f general/import/basic.sim
+./test.sh -f general/import/commit.sim
+./test.sh -f general/import/large.sim
+./test.sh -f general/import/replica1.sim
+
+./test.sh -f general/insert/basic.sim
+./test.sh -f general/insert/insert_drop.sim
+./test.sh -f general/insert/query_block1_memory.sim
+./test.sh -f general/insert/query_block2_memory.sim
+./test.sh -f general/insert/query_block1_file.sim
+./test.sh -f general/insert/query_block2_file.sim
+./test.sh -f general/insert/query_file_memory.sim
+./test.sh -f general/insert/query_multi_file.sim
+./test.sh -f general/insert/tcp.sim
+
+./test.sh -f general/parser/alter.sim
+./test.sh -f general/parser/alter1.sim
+./test.sh -f general/parser/alter_stable.sim
+./test.sh -f general/parser/auto_create_tb.sim
+./test.sh -f general/parser/auto_create_tb_drop_tb.sim
+./test.sh -f general/parser/col_arithmetic_operation.sim
+./test.sh -f general/parser/columnValue.sim
+./test.sh -f general/parser/commit.sim
+./test.sh -f general/parser/create_db.sim
+./test.sh -f general/parser/create_mt.sim
+./test.sh -f general/parser/create_tb.sim
+./test.sh -f general/parser/dbtbnameValidate.sim
+./test.sh -f general/parser/import_commit1.sim
+./test.sh -f general/parser/import_commit2.sim
+./test.sh -f general/parser/import_commit3.sim
+./test.sh -f general/parser/insert_tb.sim
+./test.sh -f general/parser/first_last.sim
+#./test.sh -f general/parser/import_file.sim
+./test.sh -f general/parser/lastrow.sim
+./test.sh -f general/parser/nchar.sim
+./test.sh -f general/parser/null_char.sim
+./test.sh -f general/parser/single_row_in_tb.sim
+./test.sh -f general/parser/select_from_cache_disk.sim
+./test.sh -f general/parser/mixed_blocks.sim
+./test.sh -f general/parser/selectResNum.sim
+./test.sh -f general/parser/limit.sim
+./test.sh -f general/parser/limit1.sim
+./test.sh -f general/parser/limit1_tblocks100.sim
+./test.sh -f general/parser/select_across_vnodes.sim
+./test.sh -f general/parser/slimit1.sim
+./test.sh -f general/parser/tbnameIn.sim
+./test.sh -f general/parser/projection_limit_offset.sim
+./test.sh -f general/parser/limit2.sim
+./test.sh -f general/parser/fill.sim
+./test.sh -f general/parser/fill_stb.sim
+./test.sh -f general/parser/where.sim
+./test.sh -f general/parser/slimit.sim
+./test.sh -f general/parser/select_with_tags.sim
+./test.sh -f general/parser/interp.sim
+./test.sh -f general/parser/tags_dynamically_specifiy.sim
+./test.sh -f general/parser/groupby.sim
+./test.sh -f general/parser/set_tag_vals.sim
+#./test.sh -f general/parser/slimit_alter_tags.sim
+./test.sh -f general/parser/join.sim
+./test.sh -f general/parser/join_multivnode.sim
+./test.sh -f general/parser/binary_escapeCharacter.sim
+./test.sh -f general/parser/bug.sim
+
+./test.sh -f general/stable/disk.sim
+./test.sh -f general/stable/dnode3.sim
+./test.sh -f general/stable/metrics.sim
+./test.sh -f general/stable/show.sim
+./test.sh -f general/stable/values.sim
+./test.sh -f general/stable/vnode3.sim
+
+./test.sh -f general/table/autocreate.sim
+./test.sh -f general/table/basic1.sim
+./test.sh -f general/table/basic2.sim
+./test.sh -f general/table/basic3.sim
+./test.sh -f general/table/bigint.sim
+./test.sh -f general/table/binary.sim
+./test.sh -f general/table/bool.sim
+./test.sh -f general/table/column_name.sim
+./test.sh -f general/table/column_num.sim
+./test.sh -f general/table/column_value.sim
+./test.sh -f general/table/column2.sim
+./test.sh -f general/table/date.sim
+./test.sh -f general/table/db.table.sim
+./test.sh -f general/table/delete_writing.sim
+./test.sh -f general/table/describe.sim
+./test.sh -f general/table/double.sim
+./test.sh -f general/table/fill.sim
+./test.sh -f general/table/float.sim
+./test.sh -f general/table/int.sim
+./test.sh -f general/table/limit.sim
+./test.sh -f general/table/smallint.sim
+./test.sh -f general/table/table_len.sim
+./test.sh -f general/table/table.sim
+./test.sh -f general/table/tinyint.sim
+./test.sh -f general/table/vgroup.sim
+
+./test.sh -f general/tag/3.sim
+./test.sh -f general/tag/4.sim
+./test.sh -f general/tag/5.sim
+./test.sh -f general/tag/6.sim
+./test.sh -f general/tag/add.sim
+./test.sh -f general/tag/bigint.sim
+./test.sh -f general/tag/binary_binary.sim
+./test.sh -f general/tag/binary.sim
+./test.sh -f general/tag/bool_binary.sim
+./test.sh -f general/tag/bool_int.sim
+./test.sh -f general/tag/bool.sim
+./test.sh -f general/tag/change.sim
+./test.sh -f general/tag/column.sim
+#./test.sh -f general/tag/commit.sim
+./test.sh -f general/tag/create.sim
+./test.sh -f general/tag/delete.sim
+./test.sh -f general/tag/double.sim
+./test.sh -f general/tag/filter.sim
+./test.sh -f general/tag/float.sim
+./test.sh -f general/tag/int_binary.sim
+./test.sh -f general/tag/int_float.sim
+./test.sh -f general/tag/int.sim
+./test.sh -f general/tag/set.sim
+./test.sh -f general/tag/smallint.sim
+./test.sh -f general/tag/tinyint.sim
+
+./test.sh -f general/user/authority.sim
+./test.sh -f general/user/monitor.sim
+./test.sh -f general/user/pass_alter.sim
+./test.sh -f general/user/pass_len.sim
+./test.sh -f general/user/user_create.sim
+./test.sh -f general/user/user_len.sim
+
+./test.sh -f general/vector/metrics_field.sim
+./test.sh -f general/vector/metrics_mix.sim
+./test.sh -f general/vector/metrics_query.sim
+./test.sh -f general/vector/metrics_tag.sim
+./test.sh -f general/vector/metrics_time.sim
+./test.sh -f general/vector/multi.sim
+./test.sh -f general/vector/single.sim
+./test.sh -f general/vector/table_field.sim
+./test.sh -f general/vector/table_mix.sim
+./test.sh -f general/vector/table_query.sim
+./test.sh -f general/vector/table_time.sim
+
+./test.sh -f unique/account/account_create.sim
+./test.sh -f unique/account/account_delete.sim
+./test.sh -f unique/account/account_len.sim
+./test.sh -f unique/account/authority.sim
+./test.sh -f unique/account/basic.sim
+./test.sh -f unique/account/paras.sim
+./test.sh -f unique/account/pass_alter.sim
+./test.sh -f unique/account/pass_len.sim
+./test.sh -f unique/account/usage.sim
+./test.sh -f unique/account/user_create.sim
+./test.sh -f unique/account/user_len.sim
+
+./test.sh -f unique/cluster/balance1.sim
+./test.sh -f unique/cluster/balance2.sim
+
+./test.sh -f unique/column/replica3.sim
+
+./test.sh -f unique/db/commit.sim
./test.sh -f unique/db/delete.sim
-#./test.sh -f unique/db/replica_add12.sim
-#./test.sh -f unique/db/replica_add13.sim
-#./test.sh -f unique/vnode/replica3_basic.sim
-#./test.sh -f unique/dnode/balance1.sim
-#./test.sh -f unique/dnode/balance2.sim
-#./test.sh -f unique/dnode/balance3.sim
-#./test.sh -f unique/cluster/balance1.sim
-#./test.sh -f unique/cluster/balance2.sim
-#./test.sh -f unique/cluster/balance3.sim
\ No newline at end of file
+./test.sh -f unique/dnode/alternativeRole.sim
+./test.sh -f unique/dnode/balance1.sim
+./test.sh -f unique/dnode/balance2.sim
+./test.sh -f unique/dnode/offline1.sim
+./test.sh -f unique/dnode/remove2.sim
+./test.sh -f unique/dnode/vnode_clean.sim
+
+./test.sh -f unique/http/admin.sim
+./test.sh -f unique/http/opentsdb.sim
+
+./test.sh -f unique/import/replica2.sim
+./test.sh -f unique/import/replica3.sim
+
+./test.sh -f unique/stable/balance_replica1.sim
+./test.sh -f unique/stable/dnode2_stop.sim
+./test.sh -f unique/stable/dnode3.sim
+./test.sh -f unique/stable/replica2_dnode4.smallint
+
+./test.sh -f unique/mnode/mgmt22.sim
+./test.sh -f unique/mnode/mgmt33.sim
+
+./test.sh -f unique/vnode/many.sim
+./test.sh -f unique/vnode/replica2_basic2.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt
index 4f194cde93dfe29280d1e11ccf84308f0d9c3644..da0aee3d343f652562426d8d077ab56734729fed 100644
--- a/tests/script/jenkins/unique.txt
+++ b/tests/script/jenkins/unique.txt
@@ -35,6 +35,7 @@ cd ../../../debug; make
./test.sh -f unique/db/replica_reduce31.sim
./test.sh -f unique/db/replica_part.sim
+./test.sh -f unique/dnode/alternativeRole.sim
./test.sh -f unique/dnode/balance1.sim
./test.sh -f unique/dnode/balance2.sim
./test.sh -f unique/dnode/balance3.sim
@@ -75,3 +76,59 @@ cd ../../../debug; make
./test.sh -f unique/vnode/replica3_basic.sim
./test.sh -f unique/vnode/replica3_repeat.sim
./test.sh -f unique/vnode/replica3_vgroup.sim
+
+./test.sh -f general/stream/metrics_1.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+#./test.sh -f general/stream/new_stream.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_1.sim
+./test.sh -f general/stream/stream_2.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim
+
+./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
+./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
+./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
+./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
+./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
+./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim
+./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
+./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
+./test.sh -f unique/arbitrator/insert_duplicationTs.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim
+./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim
+./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim
+
diff --git a/tests/script/loop.sh b/tests/script/loop.sh
new file mode 100755
index 0000000000000000000000000000000000000000..adafe2ed605ffeacbd7dba4e8e13672323fc5bb6
--- /dev/null
+++ b/tests/script/loop.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+##################################################
+#
+# Do simulation test
+#
+##################################################
+
+set -e
+#set -x
+
+CMD_NAME=
+LOOP_TIMES=5
+
+while getopts "f:t:" arg
+do
+ case $arg in
+ f)
+ CMD_NAME=$OPTARG
+ ;;
+ t)
+ LOOP_TIMES=$OPTARG
+ ;;
+ ?)
+ echo "unknow argument"
+ ;;
+ esac
+done
+
+echo LOOP_TIMES ${LOOP_TIMES}
+echo CMD_NAME ${CMD_NAME}
+
+for ((i=0; i<$LOOP_TIMES; i++ ))
+do
+ echo loop $i
+ echo cmd $CMD_NAME
+ $CMD_NAME
+ sleep 2
+done
diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim
index 6ef400149507558f105545959437c121c6796316..0fbec7ebd851c6dec2323c488279a0a5066a9b72 100644
--- a/tests/script/regressionSuite.sim
+++ b/tests/script/regressionSuite.sim
@@ -1,10 +1,10 @@
-#unsupport run general/alter/cached_schema_after_alter.sim
-#unsupport run general/alter/count.sim
-#unsupport run general/alter/import.sim
-#unsupport run general/alter/insert1.sim
-#unsupport run general/alter/insert2.sim
-#unsupport run general/alter/metrics.sim
-#unsupport run general/alter/table.sim
+##unsupport run general/alter/cached_schema_after_alter.sim
+unsupport run general/alter/count.sim
+unsupport run general/alter/import.sim
+##unsupport run general/alter/insert1.sim
+unsupport run general/alter/insert2.sim
+unsupport run general/alter/metrics.sim
+unsupport run general/alter/table.sim
run general/cache/new_metrics.sim
run general/cache/restart_metrics.sim
run general/cache/restart_table.sim
@@ -86,14 +86,14 @@ run general/insert/query_block2_file.sim
run general/insert/query_file_memory.sim
run general/insert/query_multi_file.sim
run general/insert/tcp.sim
-#unsupport run general/parser/alter.sim
-#unsupport run general/parser/alter1.sim
-#unsupport run general/parser/alter_stable.sim
+##unsupport run general/parser/alter.sim
+run general/parser/alter1.sim
+run general/parser/alter_stable.sim
run general/parser/auto_create_tb.sim
run general/parser/auto_create_tb_drop_tb.sim
run general/parser/col_arithmetic_operation.sim
run general/parser/columnValue.sim
-# run general/parser/commit.sim
+#run general/parser/commit.sim
run general/parser/create_db.sim
run general/parser/create_mt.sim
run general/parser/create_tb.sim
@@ -103,17 +103,17 @@ run general/parser/import_commit2.sim
run general/parser/import_commit3.sim
run general/parser/insert_tb.sim
run general/parser/first_last.sim
-#unsupport run general/parser/import_file.sim
+##unsupport run general/parser/import_file.sim
run general/parser/lastrow.sim
run general/parser/nchar.sim
-#unsupport run general/parser/null_char.sim
+##unsupport run general/parser/null_char.sim
run general/parser/single_row_in_tb.sim
run general/parser/select_from_cache_disk.sim
run general/parser/limit.sim
-# run general/parser/limit1.sim
+run general/parser/limit1.sim
run general/parser/limit1_tblocks100.sim
run general/parser/mixed_blocks.sim
-# run general/parser/selectResNum.sim
+run general/parser/selectResNum.sim
run general/parser/select_across_vnodes.sim
run general/parser/slimit1.sim
run general/parser/tbnameIn.sim
@@ -123,19 +123,19 @@ run general/parser/limit2.sim
run general/parser/slimit.sim
run general/parser/fill.sim
run general/parser/fill_stb.sim
-# run general/parser/interp.sim
+run general/parser/interp.sim
# run general/parser/where.sim
-#unsupport run general/parser/join.sim
-#unsupport run general/parser/join_multivnode.sim
+run general/parser/join.sim
+run general/parser/join_multivnode.sim
run general/parser/select_with_tags.sim
-#unsupport run general/parser/groupby.sim
-#unsupport run general/parser/bug.sim
-#unsupport run general/parser/tags_dynamically_specifiy.sim
-#unsupport run general/parser/set_tag_vals.sim
-#unsupport run general/parser/repeatAlter.sim
-#unsupport run general/parser/slimit_alter_tags.sim
-#unsupport run general/parser/stream_on_sys.sim
-#unsupport run general/parser/stream.sim
+run general/parser/groupby.sim
+run general/parser/bug.sim
+run general/parser/tags_dynamically_specifiy.sim
+run general/parser/set_tag_vals.sim
+##unsupport run general/parser/repeatAlter.sim
+##unsupport run general/parser/slimit_alter_tags.sim
+##unsupport run general/parser/stream_on_sys.sim
+run general/parser/stream.sim
#unsupport run general/parser/repeatStream.sim
run general/stable/disk.sim
run general/stable/dnode3.sim
@@ -173,25 +173,25 @@ run general/tag/3.sim
run general/tag/4.sim
run general/tag/5.sim
run general/tag/6.sim
-#unsupport run general/tag/add.sim
+run general/tag/add.sim
run general/tag/bigint.sim
run general/tag/binary_binary.sim
run general/tag/binary.sim
run general/tag/bool_binary.sim
run general/tag/bool_int.sim
run general/tag/bool.sim
-#unsupport run general/tag/change.sim
+run general/tag/change.sim
run general/tag/column.sim
-#unsupport run general/tag/commit.sim
+##unsupport run general/tag/commit.sim
run general/tag/create.sim
-#unsupport run general/tag/delete.sim
+run general/tag/delete.sim
run general/tag/double.sim
run general/tag/filter.sim
run general/tag/float.sim
run general/tag/int_binary.sim
run general/tag/int_float.sim
run general/tag/int.sim
-#unsupport run general/tag/set.sim
+run general/tag/set.sim
run general/tag/smallint.sim
run general/tag/tinyint.sim
run general/user/authority.sim
@@ -211,6 +211,7 @@ run general/vector/table_field.sim
run general/vector/table_mix.sim
run general/vector/table_query.sim
run general/vector/table_time.sim
+run general/stream/restart_stream.sim
run general/stream/stream_1.sim
run general/stream/stream_2.sim
run general/stream/stream_3.sim
diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh
index 1e033071f7610cd11d2c9ac69cb272a52d181300..6928039be11642b82e9a50d2f32a38d442abfbce 100755
--- a/tests/script/sh/exec.sh
+++ b/tests/script/sh/exec.sh
@@ -97,13 +97,17 @@ else
#relative path
RCFG_DIR=sim/$NODE_NAME/cfg
PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
- if [ -n "$PID" ]; then
+ while [ -n "$PID" ]
+ do
if [ "$SIGNAL" = "SIGINT" ]; then
- echo killed by signal
+ echo try to kill by signal SIGINT
kill -SIGINT $PID
else
+ echo try to kill by signal SIGKILL
kill -9 $PID
fi
- fi
+ sleep 1
+ PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
+ done
fi
diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim
index a71401d6e4709feec4bf60c3b1bb6eceba349179..2480c20709be97840b691d2d90e99d2e74d5722a 100644
--- a/tests/script/tmp/prepare.sim
+++ b/tests/script/tmp/prepare.sim
@@ -20,10 +20,10 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 20
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim
index c1c5d7572888bae7d0360fc901a9688885e510b9..3f12c4613736fa3022a9608c4703fbd2baec9e06 100644
--- a/tests/script/unique/account/usage.sim
+++ b/tests/script/unique/account/usage.sim
@@ -1,6 +1,8 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
+#system sh/exec.sh -n monitor -s 1
+system sh/exec.sh -n monitorInterval -s 1
sleep 3000
sql connect
@@ -25,13 +27,16 @@ endi
if $data05 != 0/10 then
return -1
endi
-
+if $data06 != 0.000/unlimited then
+ return -1
+endi
+
print =============== check usage account
-sql create database d1
-sql create database d2
-sql create database d3
-sql create database d4
-sql create database d5
+sql create database d1 wal 2
+sql create database d2 wal 2
+sql create database d3 wal 2
+sql create database d4 wal 2
+sql create database d5 wal 2
sql create table d1.t1 (ts timestamp, i int);
sql create user u1 pass "u1"
@@ -53,6 +58,75 @@ endi
if $data05 != 0/10 then
return -1
endi
+if $data06 != 0.000/unlimited then
+ return -1
+endi
+
+print =============== step2
+sql alter account root pass "taosdata" tseries 10 storage 1073741824 streams 10 dbs 5 users 5
+sql show accounts
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+if $data00 != root then
+ return -1
+endi
+if $data02 != 4/5 then
+ return -1
+endi
+if $data03 != 5/5 then
+ return -1
+endi
+if $data04 != 1/10 then
+ return -1
+endi
+if $data05 != 0/10 then
+ return -1
+endi
+if $data06 != 0.000/1.000 then
+ return -1
+endi
+
+print =============== step3
+sql alter account root pass "taosdata" tseries 10 storage 16 streams 10 dbs 5 users 5
+sql show accounts
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+if $data00 != root then
+ return -1
+endi
+if $data02 != 4/5 then
+ return -1
+endi
+if $data03 != 5/5 then
+ return -1
+endi
+if $data04 != 1/10 then
+ return -1
+endi
+if $data05 != 0/10 then
+ return -1
+endi
+if $data06 != 0.000/0.000 then
+ return -1
+endi
+
+print =============== step4
+sql insert into d1.t1 values(now + 1s, 1)
+sql insert into d1.t1 values(now + 2s, 2)
+
+# no write auth
+sleep 3000
+sql_error insert into d1.t1 values(now + 3s, 2)
+sql_error insert into d1.t1 values(now + 4s, 2)
+
+sql alter account root pass "taosdata" tseries 10 storage 36 streams 10 dbs 5 users 5
+sleep 3000
+sql insert into d1.t1 values(now + 5s, 1)
+sql insert into d1.t1 values(now + 6s, 2)
+
+# no write auth
+sleep 3000
+sql_error insert into d1.t1 values(now + 7s, 2)
+sql_error insert into d1.t1 values(now + 8s, 2)
+
print =============== check grant
sql_error create database d6
diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim
index e74717c0778284519171716a3214bb8a1de9fcb9..af2b47d1eafd69b9adf5b3677f588c56e546a74e 100644
--- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim
+++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim
@@ -7,84 +7,83 @@ system sh/deploy.sh -n dnode5 -i 5
system sh/deploy.sh -n dnode6 -i 6
system sh/deploy.sh -n dnode7 -i 7
-
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode1 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode1 -c statusInterval -v 3
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode1 -c timezone -v ""
#system sh/cfg.sh -n dnode1 -c locale -v ""
#system sh/cfg.sh -n dnode1 -c charset -v ""
-system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 3
######## dnode 2 the same with dnode1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode2 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode2 -c statusInterval -v 3
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode2 -c timezone -v ""
#system sh/cfg.sh -n dnode2 -c locale -v ""
#system sh/cfg.sh -n dnode2 -c charset -v ""
-system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 3
######## dnode 3 one para no same with dnode1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode3 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode3 -c statusInterval -v 3
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode3 -c timezone -v ""
#system sh/cfg.sh -n dnode3 -c locale -v ""
#system sh/cfg.sh -n dnode3 -c charset -v ""
-system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 3
######## dnode 4 one para no same with dnode1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 5
-system sh/cfg.sh -n dnode4 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode4 -c statusInterval -v 3
system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode4 -c timezone -v ""
#system sh/cfg.sh -n dnode4 -c locale -v ""
#system sh/cfg.sh -n dnode4 -c charset -v ""
-system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 3
######## dnode 5 one para no same with dnode1
system sh/cfg.sh -n dnode5 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 16
+system sh/cfg.sh -n dnode5 -c offlineThreshold -v 7
system sh/cfg.sh -n dnode5 -c statusInterval -v 3
system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode5 -c timezone -v ""
#system sh/cfg.sh -n dnode5 -c locale -v ""
#system sh/cfg.sh -n dnode5 -c charset -v ""
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 3
######## dnode 6 one para no same with dnode1
system sh/cfg.sh -n dnode6 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode6 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode6 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode6 -c statusInterval -v 2
system sh/cfg.sh -n dnode6 -c arbitrator -v $arbitrator
#system sh/cfg.sh -n dnode6 -c timezone -v ""
#system sh/cfg.sh -n dnode6 -c locale -v ""
#system sh/cfg.sh -n dnode6 -c charset -v ""
-system sh/cfg.sh -n dnode6 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode6 -c balanceInterval -v 3
######## dnode 7 one para no same with dnode1
system sh/cfg.sh -n dnode7 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode7 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode7 -c offlineThreshold -v 8
system sh/cfg.sh -n dnode7 -c statusInterval -v 3
system sh/cfg.sh -n dnode7 -c arbitrator -v "plum-VirtualBox:8001"
#system sh/cfg.sh -n dnode7 -c timezone -v ""
#system sh/cfg.sh -n dnode7 -c locale -v ""
#system sh/cfg.sh -n dnode7 -c charset -v ""
-system sh/cfg.sh -n dnode7 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode7 -c balanceInterval -v 3
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -109,7 +108,12 @@ sql create dnode $hostname6
sql create dnode $hostname7
sleep 10000
+$loopCnt = 0
wait_dnode_created:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 7 then
sleep 2000
@@ -154,7 +158,12 @@ endi
sleep 10000
+$loopCnt = 0
wait_dnode_offline_overtime_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim
index 2620746d382f7c51433f969a9a2bf83f275115c0..193a54ac6ab46f4c0da52f1b521fb0b12f77f0d0 100644
--- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim
+++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim
@@ -117,7 +117,13 @@ print ============== step4: stop dnode3, so date rows falling disc, generate two
system sh/exec.sh -n dnode3 -s stop -x SIGINT
sleep $sleepTimer
+$loopCnt = 0
wait_dnode3_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -152,11 +158,11 @@ if $data00 != $totalRows then
return -1
endi
-system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ---->dnode3 data files: $system_content , expect is 9
-#if $system_content != @9@ then
-# return -1
-#endi
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ---->dnode3 data files: $system_content expect: 9
+if $system_content != 9 then
+ return -1
+endi
print ============== step5: insert two data rows: now-16d, now+16d,
sql insert into $tb values ( now - 21d , -21 )
@@ -169,17 +175,23 @@ if $data00 != $totalRows then
return -1
endi
-system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ---->dnode2 data files: $system_content , expect is 3
-#if $system_content != @3@ then
-# return -1
-#endi
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ---->dnode2 data files: $system_content expect: 3
+if $system_content != 3 then
+ return -1
+endi
print ============== step7: restart dnode3, waiting sync end
system sh/exec.sh -n dnode3 -s start
sleep 3000
+$loopCnt = 0
wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -203,13 +215,13 @@ if $data00 != $totalRows then
return -1
endi
-system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ---->dnode2 data files: $system_content , expect is 3
-#if $system_content != @3@ then
-# return -1
-#endi
-system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ---->dnode3 data files: $system_content , expect is 3
-#if $system_content != @3@ then
-# return -1
-#endi
\ No newline at end of file
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ---->dnode2 data files: $system_content expect: 3
+if $system_content != 3 then
+ return -1
+endi
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ---->dnode3 data files: $system_content expect: 3
+if $system_content != 3 then
+ return -1
+endi
\ No newline at end of file
diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim
index 7bcde02ba857bf98eae4e9343d95321df4d67398..c373bce188b08cb2f669063feb9c31cac218404c 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
index 97c2f024276b5f6155c32195b7310675cf464284..29fa60de94fdb5d3110cb57174e71c7631b8f88b 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -37,6 +38,7 @@ system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
diff --git a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
index 67164908c7f66511961233f1336e1a3a9c5488c2..c3bae9005a3dde2d48b326f72dfebab34db4f84b 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
diff --git a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
index 9b1e320946673885e3cdd4d81458bbdff2da4084..ac2b8365f99f25a17aa8a37e43074bec526b3c37 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
@@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
-system sh/deploy.sh -n dnode5 -i 5
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
-system sh/cfg.sh -n dnode5 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode2 -c walLevel -v 1
system sh/cfg.sh -n dnode3 -c walLevel -v 1
system sh/cfg.sh -n dnode4 -c walLevel -v 1
-system sh/cfg.sh -n dnode5 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
@@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
-system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1
-system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -126,7 +117,13 @@ system sh/exec.sh -n dnode4 -s stop
#system sh/port.sh -p 7400 -s down
sleep 12000
+$loopCnt = 0
wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
new file mode 100644
index 0000000000000000000000000000000000000000..807709b6752a91c1b47d12f5e608480ea721181a
--- /dev/null
+++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
@@ -0,0 +1,357 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+ $tb = tb . $i
+ sql create table $tb using $stb tags( $i )
+
+ $x = 0
+ while $x < $rowNum
+ $ts = $tsStart + $x
+ sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x )
+ $x = $x + 60
+ endw
+ $totalRows = $totalRows + $x
+ print info: inserted $x rows into $tb and totalRows: $totalRows
+ $i = $i + 1
+endw
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step3: stop dnode3, and remove its vnodeX subdirector
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode3Status != offline then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+if $dnode3Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+if $dnode2Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step4: restart dnode3, waiting sync end
+system sh/exec.sh -n dnode3 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode2Vtatus: $dnode3Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode3Vtatus != slave then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+if $dnode2Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step5: stop dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode2Status != offline then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+
+print ============== step6: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode2_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode2_reready
+endi
+if $dnode2Status != ready then
+ sleep 2000
+ goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode3Vtatus != master then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+if $dnode2Vtatus != slave then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
\ No newline at end of file
diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
new file mode 100644
index 0000000000000000000000000000000000000000..a05681ece231a4b47dfdb89d6ec95a36a1a651a1
--- /dev/null
+++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
@@ -0,0 +1,407 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sql create dnode $hostname4
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 3 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+ $tb = tb . $i
+ sql create table $tb using $stb tags( $i )
+
+ $x = 0
+ while $x < $rowNum
+ $ts = $tsStart + $x
+ sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x )
+ $x = $x + 60
+ endw
+ $totalRows = $totalRows + $x
+ print info: inserted $x rows into $tb and totalRows: $totalRows
+ $i = $i + 1
+endw
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step3: stop dnode4, and remove its vnodeX subdirector
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode4_offline_0
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode4Status != offline then
+ sleep 2000
+ goto wait_dnode4_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode4_vgroup_offline
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+if $dnode4Vtatus != offline then
+ sleep 2000
+ goto wait_dnode4_vgroup_offline
+endi
+if $dnode3Vtatus != master then
+ sleep 2000
+ goto wait_dnode4_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode4/data
+#system rm -rf ../../../sim/dnode4/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step4: restart dnode4, waiting sync end
+system sh/exec.sh -n dnode4 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode4_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode4Status != ready then
+ sleep 2000
+ goto wait_dnode4_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode4_vgroup_slave
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode4Vtatus != slave then
+ sleep 2000
+ goto wait_dnode4_vgroup_slave
+endi
+if $dnode3Vtatus != master then
+ sleep 2000
+ goto wait_dnode4_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step5: stop dnode3/dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode23_offline
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode2Status != offline then
+ sleep 2000
+ goto wait_dnode23_offline
+endi
+if $dnode3Status != offline then
+ sleep 2000
+ goto wait_dnode23_offline
+endi
+if $dnode4Status != ready then
+ sleep 2000
+ goto wait_dnode23_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode2/dnode3, and check rows
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode23_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode2Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready
+endi
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready
+endi
+if $dnode4Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode4_vgroup_master
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode4Vtatus != master then
+ sleep 2000
+ goto wait_dnode4_vgroup_master
+endi
+if $dnode3Vtatus != slave then
+ sleep 2000
+ goto wait_dnode4_vgroup_master
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+
+print ============== step7: stop dnode3/dnode2, and cluster unable to provide services
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step8: restart dnode2, and cluster Still unable to provide services
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step9: restart dnode3, and cluster Resume service delivery
+system sh/exec.sh -n dnode3 -s start
+
+$loopCnt = 0
+wait_dnode4_vgroup_master_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode4_vgroup_master_2
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode4Vtatus != master then
+ sleep 2000
+ goto wait_dnode4_vgroup_master_2
+endi
+if $dnode3Vtatus != slave then
+ sleep 2000
+ goto wait_dnode4_vgroup_master_2
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
index ef6954d4ee82e6b7c128ff7006ca04a26073511f..cacbcbb132817eab099d14dc6f2f0660e43044ab 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
@@ -36,14 +36,16 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 200
system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode5 -c alternativeRole -v 2
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
@@ -71,7 +73,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
-print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data
+print ============== step2: start dnode2/dnode3 and add into cluster, then create database replica 2, create table , and insert data
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname2
@@ -84,7 +86,7 @@ $totalRows = 0
$tsStart = 1420041600000
$db = db
-sql create database $db replica 2 maxTables 4
+sql create database $db replica 2
sql use $db
# create table , insert data
@@ -164,7 +166,13 @@ endi
sleep 15000
+$loopCnt = 0
wait_drop:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 3000
@@ -193,14 +201,17 @@ if $dnode4Status != ready then
endi
print ============== step4-1: restart dnode3, adn add into cluster
-system rm -rf ../../sim/dnode3
-sleep 3000
+system rm -rf ../../../sim/dnode3
+sleep 1000
system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c walLevel -v 1
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
@@ -209,7 +220,13 @@ system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname3
sleep 3000
+$loopCnt = 0
wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
print rows: $rows
if $rows != 4 then
@@ -228,19 +245,19 @@ $dnode4Status = $data4_4
$dnode5Status = $data4_5
if $dnode1Status != ready then
- return -1
+ goto wait_dnode3_ready
endi
if $dnode2Status != ready then
- return -1
+ goto wait_dnode3_ready
endi
if $dnode3Status != null then
- return -1
+ goto wait_dnode3_ready
endi
if $dnode4Status != ready then
- return -1
+ goto wait_dnode3_ready
endi
if $dnode5Status != ready then
- return -1
+ goto wait_dnode3_ready
endi
sql select count(*) from $stb
@@ -255,7 +272,13 @@ sleep 1000
system sh/exec.sh -n dnode5 -s start
sql create dnode $hostname5
sleep 3000
+$loopCnt = 0
wait_dnode5:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 5 then
sleep 3000
@@ -274,22 +297,22 @@ $dnode5Status = $data4_5
$dnode6Status = $data4_6
if $dnode1Status != ready then
- return -1
+ goto wait_dnode5
endi
if $dnode2Status != ready then
- return -1
+ goto wait_dnode5
endi
if $dnode3Status != null then
- return -1
+ goto wait_dnode5
endi
if $dnode4Status != ready then
- return -1
+ goto wait_dnode5
endi
if $dnode5Status != ready then
- return -1
+ goto wait_dnode5
endi
if $dnode6Status != ready then
- return -1
+ goto wait_dnode5
endi
print ============== step6: create database and table until not free vnodes
diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim
index 1652470346ec747cc325b7a3923e6afa344fdb53..9b01b2fae38a63c38ba89872b3217afb2b24b9f5 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -92,7 +93,14 @@ endi
print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
+
+
+$loopCnt = 0
wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 2 then
sleep 2000
@@ -117,7 +125,14 @@ endi
system sh/exec.sh -n dnode2 -s start
sleep $sleepTimer
+
+
+$loopCnt = 0
wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 2 then
sleep 2000
@@ -155,8 +170,13 @@ sleep 3000
sql alter database $db replica 2
sleep $sleepTimer
-
+$loopCnt = 0
wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -195,8 +215,13 @@ endi
print ============== step4: stop dnode2 for checking if sync ok
system sh/exec.sh -n dnode2 -s stop
sleep $sleepTimer
-
+$loopCnt = 0
wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -235,7 +260,13 @@ print ============== step5: restart dnode2
system sh/exec.sh -n dnode2 -s start
sleep 3000
+$loopCnt = 0
wait_dnode2_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -273,8 +304,13 @@ sleep 3000
sql alter database $db replica 3
sleep $sleepTimer
-
+$loopCnt = 0
wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -308,8 +344,13 @@ endi
print ============== step7: alter replica from 3 to 2, and waiting sync
sql alter database $db replica 2
sleep $sleepTimer
-
+$loopCnt = 0
wait_vgroups_replic_to_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2
@@ -350,8 +391,13 @@ endi
print ============== step8: alter replica from 2 to 1, and waiting sync
sql alter database $db replica 1
sleep $sleepTimer
-
+$loopCnt = 0
wait_vgroups_replic_to_1:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2
@@ -381,7 +427,13 @@ if $sencodDnode_5 != null then
goto wait_vgroups_replic_to_1
endi
+$loopCnt = 0
all_dnodes_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -429,7 +481,13 @@ sql drop dnode $hostname2
sql drop dnode $hostname3
sleep $sleepTimer
+$loopCnt = 0
wait_dnode23_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 2 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
index b870d07c1d261af47207bcba21c7bdef33016bd7..9b001820f603da35c7661c0426e5b84e5929afea 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -97,8 +98,13 @@ print ============== step3: drop dnode4, then check rows
#system sh/exec.sh -n dnode4 -s stop -x SIGINT
sql drop dnode $hostname4
sleep $sleepTimer
-
+$loopCnt = 0
wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
index 1b0a184690f7bbcaf0a49b1473451dd90043981a..152a65a75aec326c3314e079da7c562113be0557 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
@@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
-system sh/deploy.sh -n dnode5 -i 5
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode5 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode2 -c walLevel -v 1
system sh/cfg.sh -n dnode3 -c walLevel -v 1
system sh/cfg.sh -n dnode4 -c walLevel -v 1
-system sh/cfg.sh -n dnode5 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
@@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
-system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1
-system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -124,7 +115,13 @@ print ============== step3: stop dnode4, after timerout dnode4 will be auto-drop
system sh/exec.sh -n dnode4 -s stop
sleep 12000
+$loopCnt = 0
wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -178,7 +175,7 @@ system sh/exec.sh -n dnode4 -s stop
system rm -rf ../../../sim/dnode4
system sh/deploy.sh -n dnode4 -i 4
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c walLevel -v 1
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
@@ -192,7 +189,13 @@ system sh/exec.sh -n dnode4 -s start
sql create dnode $hostname4
sleep 6000
+$loopCnt = 0
wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim
index d2bd5c6b266ab398eaf1e3306d63bc09d6233dcb..77dbdb0c4c40d3701f4aaf11cbb6eb0a2947a857 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
-print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
@@ -59,7 +60,7 @@ $totalTableNum = 10
$sleepTimer = 3000
$db = db
-sql create database $db replica 3 maxTables $totalTableNum
+sql create database $db replica 2 maxTables $totalTableNum
sql use $db
# create table , insert data
@@ -87,13 +88,18 @@ while $i < $tblNum
endw
sql select count(*) from $stb
-sleep 1000
print data00 $data00
if $data00 != $totalRows then
return -1
endi
-print ============== step3: stop dnode4, and remove its vnodeX subdirector
+sql show dnodes
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+
+print ============== step3: stop dnode4, then destroy the contents of its data file
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
$loopCnt = 0
@@ -110,14 +116,11 @@ endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
-#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
-#$dnode1Status = $data4_1
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
if $dnode4Status != offline then
sleep 2000
@@ -146,10 +149,20 @@ if $dnode3Vtatus != master then
goto wait_dnode4_vgroup_offline
endi
-
-system rm -rf ../../../sim/dnode4/data/vnode/*
+system echo "haha, nothing......" > ../../../sim/dnode4/data/vnode/vnode2/tsdb/data/f1643.data
+#system rm -rf ../../../sim/dnode4/data/vnode/*
sleep 1000
+print ============== step3-1: insert new data
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
print ============== step4: restart dnode4, waiting sync end
system sh/exec.sh -n dnode4 -s start
sleep $sleepTimer
@@ -213,12 +226,17 @@ if $dnode3Vtatus != master then
goto wait_dnode4_vgroup_slave
endi
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
print ============== step5: stop dnode3/dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
-system sh/exec.sh -n dnode3 -s stop
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
sleep $sleepTimer
-
$loopCnt = 0
wait_dnode23_offline:
$loopCnt = $loopCnt + 1
@@ -234,13 +252,10 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
-#$dnode1Status = $data4_1
+$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
if $dnode2Status != offline then
sleep 2000
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
index a7529d5bbcb366bbbd92f461e67124aefb7d8b5b..50d9bbbecbb7229773d40fc25f12c8bedb283127 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -96,7 +97,14 @@ endi
print ============== step3: stop dnode3 for falling disc, then corrupt vnode data file in dnode3
system sh/exec.sh -n dnode3 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -119,7 +127,13 @@ if $dnode3Status != offline then
goto wait_dnode3_offline_0
endi
+$loopCnt = 0
wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -141,27 +155,47 @@ if $dnode2Vtatus != master then
goto wait_dnode3_vgroup_offline
endi
-#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | sed 's/^[ \t]*//g'
-#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | sed 's/[ \t]*$//g'
+#$expectCnt = 3 . :
+#print expectCnt: [ $expectCnt ]
+#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':'
+#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':'
#print --2-->dnode3 data files: [ $system_content ]
-system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l
-print ---->dnode2 data files: [ $system_content ], expect is 0
-system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l
-print ---->dnode3 data files: [ $system_content ], expect is 3
-#if $system_content != 3 then
-# return -1
-#endi
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n'
+print ---->dnode2 data files: $system_content expect: 0
+if $system_content != 0 then
+ return -1
+endi
+
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n'
+print ---->dnode3 data files: $system_content expect: 3
+if $system_content != 3 then
+ return -1
+endi
+
+system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
-#system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
-#sleep 1000
+print ============== step3-1: insert some news data for let version changed
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+sql insert into $tb values ( now + 10a , $x ) ( now + 11a , $x ) ( now + 12a , $x )
+$totalRows = $totalRows + 6
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
print ============== step4: restart dnode3, and run query
system sh/exec.sh -n dnode3 -s start
sleep $sleepTimer
-
+$loopCnt = 0
wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -183,8 +217,13 @@ if $dnode3Status != ready then
sleep 2000
goto wait_dnode3_reready
endi
-
+$loopCnt = 0
wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -212,25 +251,29 @@ if $data00 != $totalRows then
return -1
endi
-system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ----> dnode2 data files: [ $system_content ], expect is 0
-#if $system_content != 0 then
-# return -1
-#endi
-
-system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l
-print ----> dnode3 data files: [ $system_content ], expect is 0
-#if $system_content != 0 then
-# print there should be no data file in dnode3 after sync
-# return -1
-#endi
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ----> dnode2 data files: $system_content expect: 0
+if $system_content != 0 then
+ return -1
+endi
-return -1
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ----> dnode3 data files: $system_content expect: 0
+if $system_content != 0 then
+ print there should be no data file in dnode3 after sync
+ return -1
+endi
print ============== step5: stop dnode2, and check if dnode3 sync ok
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -253,7 +296,13 @@ if $dnode2Status != offline then
goto wait_dnode2_offline_0
endi
+$loopCnt = 0
wait_dnode3_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -297,10 +346,11 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat
print ============== step7: restart dnode3, and run query
+system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
sleep $sleepTimer
$loopCnt = 0
-wait_dnode3_reready_2:
+wait_dnode23_reready_2:
$loopCnt = $loopCnt + 1
if $loopCnt == 10 then
return -1
@@ -308,7 +358,7 @@ endi
sql show dnodes
if $rows != 3 then
sleep 2000
- goto wait_dnode3_reready_2
+ goto wait_dnode23_reready_2
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
@@ -317,13 +367,18 @@ $dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
+if $dnode2Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready_2
+endi
+
if $dnode3Status != ready then
sleep 2000
- goto wait_dnode3_reready_2
+ goto wait_dnode23_reready_2
endi
$loopCnt = 0
-wait_dnode3_vgroup_master_2:
+wait_dnode23_vgroup_ok:
$loopCnt = $loopCnt + 1
if $loopCnt == 10 then
return -1
@@ -331,7 +386,7 @@ endi
sql show vgroups
if $rows != 1 then
sleep 2000
- goto wait_dnode3_vgroup_master_2
+ goto wait_dnode23_vgroup_ok
endi
print show vgroups:
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
@@ -340,13 +395,13 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat
$dnode2Vtatus = $data7_2
$dnode3Vtatus = $data4_2
-if $dnode2Vtatus != offline then
+if $dnode2Vtatus == offline then
sleep 2000
- goto wait_dnode3_vgroup_master_2
+ goto wait_dnode23_vgroup_ok
endi
-if $dnode3Vtatus != master then
+if $dnode3Vtatus == offline then
sleep 2000
- goto wait_dnode3_vgroup_master_2
+ goto wait_dnode23_vgroup_ok
endi
sql select count(*) from $stb
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
index fc8577964dc3baa1eeb30b3b50a26df5687b4c81..a526192b15cb4f93ef9408cec19e84ca50774d50 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -94,11 +95,13 @@ if $data00 != $totalRows then
endi
print ============== step3: corrupt vnode data file in dnode3, not stop dnode3
-# del the second row
-system sed '2d' ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/v1849.data
+system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
sleep 1000
-print ============== step4: run query
+print ============== step4: insert new data, and run query
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+$totalRows = $totalRows + 3
+
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
@@ -108,7 +111,14 @@ endi
print ============== step5: stop dnode2, and check if dnode3 sync ok
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -131,7 +141,13 @@ if $dnode2Status != offline then
goto wait_dnode2_offline_0
endi
+$loopCnt = 0
wait_dnode3_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -157,4 +173,142 @@ sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
-endi
\ No newline at end of file
+endi
+
+print ============== step6: stop dnode3 for falling disc
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
+#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
+#$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+
+if $dnode3Status != offline then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+
+
+$loopCnt = 0
+wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode2Vtatus = $data7_2
+$dnode3Vtatus = $data4_2
+
+if $dnode2Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+if $dnode3Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+
+print ============== step7: restart dnode3, and run query
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_master_1:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_master_1
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode2Vtatus = $data7_2
+$dnode3Vtatus = $data4_2
+
+if $dnode2Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_master_1
+endi
+if $dnode3Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_master_1
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
index 82f2aad07b79858379e4b3d9e8355b1ad693f1d1..cf20122d4c6a4a4d0b795af287bd156fe1681708 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
-print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
@@ -96,7 +97,14 @@ endi
print ============== step3: stop dnode4, and remove its vnodeX subdirector
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -105,21 +113,25 @@ endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
-#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
-#$dnode1Status = $data4_1
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
if $dnode4Status != offline then
sleep 2000
goto wait_dnode4_offline_0
endi
+
+$loopCnt = 0
wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -141,14 +153,19 @@ if $dnode3Vtatus != master then
goto wait_dnode4_vgroup_offline
endi
-
system rm -rf ../../../sim/dnode4/data/vnode/*
-sleep 1000
print ============== step4: restart dnode4, waiting sync end
system sh/exec.sh -n dnode4 -s start
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -157,21 +174,24 @@ endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
-#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
-#$dnode1Status = $data4_1
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
if $dnode4Status != ready then
sleep 2000
goto wait_dnode4_reready
endi
+$loopCnt = 0
wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -195,47 +215,87 @@ if $dnode3Vtatus != master then
goto wait_dnode4_vgroup_slave
endi
-print ============== step5: stop dnode3/dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
-system sh/exec.sh -n dnode3 -s stop
+print ============== step5: stop dnode3, and remove its vnodeX subdirector
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
sleep $sleepTimer
-wait_dnode23_offline:
+$loopCnt = 0
+wait_dnode3_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 4 then
sleep 2000
- goto wait_dnode23_offline
+ goto wait_dnode3_offline
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
-#$dnode1Status = $data4_1
+$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
-if $dnode2Status != offline then
+if $dnode2Status != ready then
sleep 2000
- goto wait_dnode23_offline
+ goto wait_dnode3_offline
endi
if $dnode3Status != offline then
sleep 2000
- goto wait_dnode23_offline
+ goto wait_dnode3_offline
endi
if $dnode4Status != ready then
sleep 2000
- goto wait_dnode23_offline
+ goto wait_dnode3_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode3, and check rows
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
endi
-wait_dnode4_vgroup_master:
sql show vgroups
if $rows != 1 then
sleep 2000
- goto wait_dnode4_vgroup_master
+ goto wait_dnode3_vgroup_slave
endi
print show vgroups:
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
@@ -248,11 +308,11 @@ print dnode4Vtatus: $dnode4Vtatus
print dnode3Vtatus: $dnode3Vtatus
if $dnode4Vtatus != master then
sleep 2000
- goto wait_dnode4_vgroup_master
+ goto wait_dnode3_vgroup_slave
endi
-if $dnode3Vtatus != offline then
+if $dnode3Vtatus != slave then
sleep 2000
- goto wait_dnode4_vgroup_master
+ goto wait_dnode3_vgroup_slave
endi
# check using select
@@ -271,3 +331,128 @@ if $data00 != $totalRows then
return -1
endi
+print ============== step7: stop dnode2, and remove its vnodeX subdirector
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode2Status != offline then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+if $dnode4Status != ready then
+ sleep 2000
+ goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data/vnode/*
+
+print ============== step8: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+ sleep 2000
+ goto wait_dnode2_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+
+if $dnode2Status != ready then
+ sleep 2000
+ goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus
+if $dnode4Vtatus != master then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+if $dnode3Vtatus != slave then
+ sleep 2000
+ goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
new file mode 100644
index 0000000000000000000000000000000000000000..ce05b0b5ff7eff84e90a753cedb7bf08c04c83c7
--- /dev/null
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
@@ -0,0 +1,433 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+#system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+#sql create dnode $hostname4
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+ $tb = tb . $i
+ sql create table $tb using $stb tags( $i )
+
+ $x = 0
+ while $x < $rowNum
+ $ts = $tsStart + $x
+ sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x )
+ $x = $x + 60
+ endw
+ $totalRows = $totalRows + $x
+ print info: inserted $x rows into $tb and totalRows: $totalRows
+ $i = $i + 1
+endw
+
+sql select count(*) from $stb
+sleep 1000
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step3: stop dnode3 for falling disc, then corrupt vnode data file in dnode3
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
+#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
+#$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+#$dnode5Status = $data4_5
+
+if $dnode3Status != offline then
+ sleep 2000
+ goto wait_dnode3_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+if $dnode3Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+if $dnode2Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_offline
+endi
+
+#$expectCnt = 3 . :
+#print expectCnt: [ $expectCnt ]
+#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':'
+#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':'
+#print --2-->dnode3 data files: [ $system_content ]
+
+
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n'
+print ---->dnode2 data files: $system_content expect: 0
+if $system_content != 0 then
+ return -1
+endi
+
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n'
+print ---->dnode3 data files: $system_content expect: 3
+if $system_content != 3 then
+ return -1
+endi
+
+#system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
+
+print ============== step3-1: insert some news data for let version changed
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+sql insert into $tb values ( now + 10a , $x ) ( now + 11a , $x ) ( now + 12a , $x )
+$totalRows = $totalRows + 6
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step4: restart dnode3, and run query
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
+#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
+#$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+#$dnode5Status = $data4_5
+
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode3_reready
+endi
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode2Vtatus = $data7_2
+$dnode3Vtatus = $data4_2
+
+if $dnode2Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+if $dnode3Vtatus != slave then
+ sleep 2000
+ goto wait_dnode3_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ----> dnode2 data files: $system_content expect: 0
+if $system_content != 0 then
+ return -1
+endi
+
+system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n'
+print ----> dnode3 data files: $system_content expect: 0
+if $system_content != 0 then
+ print there should be no data file in dnode3 after sync
+ return -1
+endi
+
+print ============== step5: stop dnode2, and check if dnode3 sync ok
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode2_offline_0
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
+#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
+#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
+#$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+$dnode4Status = $data4_4
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+ sleep 2000
+ goto wait_dnode2_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode3_vgroup_master
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode2Vtatus = $data7_2
+$dnode3Vtatus = $data4_2
+
+if $dnode2Vtatus != offline then
+ sleep 2000
+ goto wait_dnode3_vgroup_master
+endi
+if $dnode3Vtatus != master then
+ sleep 2000
+ goto wait_dnode3_vgroup_master
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+print ============== step6: stop dnode3 for falling disck
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+sql show dnodes
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+
+sql show vgroups
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+
+
+print ============== step7: restart dnode2/dnode3, and run query
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+$loopCnt = 0
+wait_dnode23_reready_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+sql show dnodes
+if $rows != 3 then
+ sleep 2000
+ goto wait_dnode23_reready_2
+endi
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
+$dnode1Status = $data4_1
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3
+
+if $dnode2Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready_2
+endi
+if $dnode3Status != ready then
+ sleep 2000
+ goto wait_dnode23_reready_2
+endi
+
+$loopCnt = 0
+wait_dnode23_vgroup_ok:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+sql show vgroups
+if $rows != 1 then
+ sleep 2000
+ goto wait_dnode23_vgroup_ok
+endi
+print show vgroups:
+print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1
+print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2
+print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3
+$dnode2Vtatus = $data7_2
+$dnode3Vtatus = $data4_2
+
+if $dnode2Vtatus == offline then
+ sleep 2000
+ goto wait_dnode23_vgroup_ok
+endi
+if $dnode3Vtatus == offline then
+ sleep 2000
+ goto wait_dnode23_vgroup_ok
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+ return -1
+endi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
index 4e9afbf31ae32fe2dacb43a4d367a94212f711c3..f5bcd055a03144c8e054618eb3e760f1c72b8d21 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -96,7 +97,13 @@ print ============== step3: stop dnode4/dnode2
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -123,7 +130,13 @@ if $dnode2Status != offline then
goto wait_dnode4_offline_0
endi
+$loopCnt = 0
wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -192,7 +205,12 @@ endi
print ============== step5: stop dnode3
system sh/exec.sh -n dnode3 -s stop -x SIGINT
sleep $sleepTimer
+$loopCnt = 0
wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -221,7 +239,12 @@ if $dnode2Status != ready then
goto wait_dnode3_offline_0
endi
+$loopCnt = 0
wait_dnode2_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -243,8 +266,8 @@ if $dnode2Vtatus != master then
goto wait_dnode2_vgroup_master
endi
-sql insert into tb98 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002) tb99 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002)
-$totalRows = $totalRows + 6
+sql insert into $tb values (now, 9000) (now + 1s, 9001) (now + 2s, 9002)
+$totalRows = $totalRows + 3
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
diff --git a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim
index 57a833b8de3f956b3d8e0422fa23750362c15bfd..acf157a93a80a8517615b202e7801f1495ce6f88 100644
--- a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim
+++ b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
diff --git a/tests/script/unique/arbitrator/insert_duplicationTs.sim b/tests/script/unique/arbitrator/insert_duplicationTs.sim
index aae1f46180f3749a15ed28afcad943548add20de..b9aff29f0cb1706065b8a0837267e2ff48d8973e 100644
--- a/tests/script/unique/arbitrator/insert_duplicationTs.sim
+++ b/tests/script/unique/arbitrator/insert_duplicationTs.sim
@@ -43,11 +43,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -117,7 +117,13 @@ print ============== step4: stop dnode2, so date rows falling disc, generate two
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
+$loopCnt = 0
wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -160,8 +166,13 @@ $totalRows = $totalRows + 2
print ============== step5: restart dnode2, waiting sync end
system sh/exec.sh -n dnode2 -s start
sleep 3000
-
+$loopCnt = 0
wait_dnode2_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
+
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim
index 26a6359a5c526a88ea8e84ae2cb6f1c43fefac41..18ffb32eed6d980be4e742cf307225eaa4c12229 100644
--- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim
@@ -99,7 +99,13 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+
+$loopCnt = 0
wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -160,7 +166,12 @@ $totalRows = $totalRows + $addRows
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$loopCnt = 0
wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim
index b883f077c2d1821a104efb579bdda866d7940b33..6e85429b773b53af9b5b57f632287795becedf51 100644
--- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim
@@ -100,7 +100,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -187,7 +197,12 @@ endi
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim
index e7b2c70c782a6ffd260c5c4b12dfc94ba6331def..0356f72a972c83e0179acfbbf69e9adb25bd2d6c 100644
--- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim
@@ -101,7 +101,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -124,7 +129,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -174,7 +184,12 @@ endi
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim
index 20c575d382e71a9340240ba2fc7a142ee8927b0f..dc2586b43b917be410619d8f6b15be07874cb648 100644
--- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim
index 9f72cde440ac53cc7e5c246905022c0f51e5ee7b..721c7d60ba9b7a8745071963795fb4b5864e0a6c 100644
--- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -158,7 +168,12 @@ $tblNum = $tblNum - 5
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim
index 43f9dfff144d3622ff738db6c5bdb61215bee357..897c662b5123d5f5f5d2a8f9df4c23a9c214db88 100644
--- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -160,7 +170,12 @@ $totalRows = $totalRows + $addRows
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim
index e27bbafba7a6d6cc01f83f2a594f9f25b1f8a5d8..95ae8047fa8b6dd5eb48d2d26f689d679cc569bf 100644
--- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim
@@ -100,7 +100,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -187,7 +197,12 @@ endi
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim
index 1e47157975e5f9183787f7874f7461b6d7492598..a60f810eeaa7774446c4ec3320c18136adcb1e91 100644
--- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim
@@ -100,7 +100,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -173,7 +183,12 @@ endi
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim
index c9fc91527d27a2e66d17b2d385b097b9f837b7de..a619cbe826464d941b20ed4599305398fb1223f8 100644
--- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim
index aecf41b892ea2eabd428a88b803d9a81c8be830c..3b7ee869ed67e034dc5ec61941cadf019d6b69e8 100644
--- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim
+++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -158,7 +168,12 @@ $tblNum = $tblNum - 5
print ============== step5: restart dnode4, waiting dnode4 synced
system sh/exec.sh -n dnode4 -s start
+$cnt = 0
wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim
index 4a133d2aac875428624da7b1ba1b28021062dd74..1b2c8d1db6b3068f8bf637466c72edfb2593ff2c 100644
--- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim
+++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -24,9 +25,9 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
-#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
-#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
-#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
@@ -37,11 +38,21 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8
+
+system sh/cfg.sh -n dnode1 -c statusInterval -v 3
+system sh/cfg.sh -n dnode2 -c statusInterval -v 3
+system sh/cfg.sh -n dnode3 -c statusInterval -v 3
+system sh/cfg.sh -n dnode4 -c statusInterval -v 3
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
@@ -96,7 +107,12 @@ sql create dnode $hostname2
sleep 3000
# expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2
+$cnt = 0
wait_dnode2_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 2 then
sleep 2000
@@ -105,14 +121,8 @@ endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
-#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
-#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
-#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
$dnode1Status = $data4_1
$dnode2Status = $data4_2
-#$dnode3Status = $data4_3
-#$dnode4Status = $data4_4
-#$dnode5Status = $data4_5
if $dnode1Status != ready then
sleep 2000
@@ -123,7 +133,7 @@ if $dnode2Status != ready then
goto wait_dnode2_ready
endi
-print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2
+print ============== step3: stop dnode1/dnode2, modify cfg numOfMnodes to 2, and restart dnode1/dnode2
system sh/exec.sh -n dnode1 -s stop
system sh/exec.sh -n dnode2 -s stop
sleep 3000
@@ -143,7 +153,12 @@ sleep 5000
print ============= step4: wait dnode ready
+$cnt = 0
wait_dnode_ready:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 2 then
sleep 2000
@@ -180,7 +195,12 @@ print ============== step5: stop dnode1
system sh/exec.sh -n dnode1 -s stop
sleep 3000
+$cnt = 0
wait_dnode2_master:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show mnodes
if $rows != 2 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim
index 9730842938f018bb89cb40b16d960530eee3a19f..4f66e5c13562a98e63ccefeacdfda1fb7cbc7cce 100644
--- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim
+++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim
index 21957871a522d9e2cd25ac9009d4b8021a9e302b..7bcd148bc9316625af6cb75c9387cdb3e047d8fb 100644
--- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim
+++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start
run_back unique/arbitrator/sync_replica_alterTable_background_drop.sim
print ============== step6: check result
-#in background.sim, drop one column and add one new column, then insert 200 rows
+#in background.sim, drop one column and add one new column, then insert 36 rows
$totalRows = $totalRows + 36
$cnt = 0
diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim
index 9836284af99358825b5705bf191d21fe628aad5f..f61b1e3cb9efad8878cd6cacc8d830133bf6791a 100644
--- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim
+++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim
index 4793e8e535a4439e2601028b0a8390685f6905d3..b27558fdf051bf66fecf3ec38bdf835121c76977 100644
--- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim
+++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 3 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim
index 9277ad2c85a6873b848e6022ebf006eb5107d605..2a283a6a0c6f1abc9594b7e721d7bd495d0b9014 100644
--- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim
+++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -162,7 +172,7 @@ while $i < $tblNum
endw
sql select count(*) from $stb
-print data00 $data00
+print data00:$data00 totalRows:$totalRows
if $data00 != $totalRows then
return -1
endi
@@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start
run_back unique/arbitrator/sync_replica_alterTable_background_add.sim
print ============== step6: check result
-#in background.sim, add one column and insert 200 rows
+#in background.sim, add one column and insert 36 rows
$totalRows = $totalRows + 36
$cnt = 0
@@ -183,7 +193,7 @@ if $cnt == 20 then
endi
sql select count(*) from $stb
if $data00 != $totalRows then
- print data00: $data00 totalRows: $totalRows
+ print data00:$data00 totalRows:$totalRows
sleep 2000
goto wait_table_altered
endi
diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim
index 6593d6933b65120901a6a708068b04b6e1d21c51..7d321ed381adaa1485f86dc23d8b42f43260dce9 100644
--- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim
+++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim
index 7099b1dc8e04e1d03d2e9c85b03579b8939e54f8..0f7ee50c3943c966e9284461189d585a22729fa4 100644
--- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim
+++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim
index f902b41de57c1c6f687f12f1eb3a8cd1e94ed8b2..9c33dd8bb474bbc7d4c34eb9f62f62cc79951b06 100644
--- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim
+++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim
@@ -99,7 +99,12 @@ endi
print ============== step3: stop dnode4
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sleep $sleepTimer
+$cnt = 0
wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show dnodes
if $rows != 4 then
sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
goto wait_dnode4_offline_0
endi
+$cnt = 0
wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+ return -1
+endi
sql show vgroups
if $rows != 1 then
sleep 2000
@@ -143,7 +153,7 @@ if $dnode3Vtatus != master then
sleep 2000
goto wait_dnode4_vgroup_offline
endi
-sleep 2000
+
print ============== step4: insert more data rows
$tsStart = $tsEnd + 1000
$i = 0
@@ -162,17 +172,17 @@ while $i < $tblNum
endw
sql select count(*) from $stb
-print data00 $data00
+print data00:$data00 totalRows:$totalRows
if $data00 != $totalRows then
return -1
endi
-print ============== step5: restart dnode4, while drop database in other thead when dnode4 is syncing
+print ============== step5: restart dnode4, while drop some tables in other thread when dnode4 is syncing
system sh/exec.sh -n dnode4 -s start
run_back unique/arbitrator/sync_replica_dropTable_background.sim
print ============== step6: check result
-#in background.sim, drop 10 tables
+#in background.sim, drop 5 tables
$totalRows = $totalRows - 5400
$cnt = 0
@@ -183,7 +193,7 @@ if $cnt == 20 then
endi
sql select count(*) from $stb
if $data00 != $totalRows then
- print data00: $data00
+ print data00:$data00 totalRows:$totalRows
sleep 2000
goto wait_table_dropped
endi
@@ -191,7 +201,7 @@ endi
$tblNum = $tblNum - 5
sql select count(tbname) from $stb
if $data00 != $tblNum then
- print data00: $data00
+ print data00: $data00 tblNum: $tblNum
sleep 2000
goto wait_table_dropped
endi
diff --git a/tests/script/unique/arbitrator/testSuite.sim b/tests/script/unique/arbitrator/testSuite.sim
index d286ecfaf29ea68efc1197f03da9e8f4fb571917..b6678633276aa3bc7f22c325fede37a8660d3f9c 100644
--- a/tests/script/unique/arbitrator/testSuite.sim
+++ b/tests/script/unique/arbitrator/testSuite.sim
@@ -1,5 +1,5 @@
+run unique/arbitrator/check_cluster_cfg_para.sim
run unique/arbitrator/dn2_mn1_cache_file_sync.sim
-run unique/arbitrator/dn2_mn1_cache_file_sync_second.sim
run unique/arbitrator/dn3_mn1_full_createTableFail.sim
run unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
run unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
@@ -9,9 +9,12 @@ run unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
run unique/arbitrator/dn3_mn1_replica_change.sim
run unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
run unique/arbitrator/dn3_mn1_vnode_change.sim
-run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
run unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
-####run unique/arbitrator/dn3_mn1_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
+run unique/arbitrator/dn3_mn1_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
run unique/arbitrator/dn3_mn1_vnode_nomaster.sim
run unique/arbitrator/dn3_mn2_killDnode.sim
run unique/arbitrator/insert_duplicationTs.sim
diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim
index bb27a2aadbdfb351eeac2524ef4890415ba1db94..0b3e1374ec61d595e04b03f3c6aa3036098f2220 100644
--- a/tests/script/unique/cluster/balance2.sim
+++ b/tests/script/unique/cluster/balance2.sim
@@ -425,7 +425,7 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
-system sh/exec.sh -n dnode5 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGKILL
system sh/exec.sh -n dnode6 -s stop -x SIGINT
system sh/exec.sh -n dnode7 -s stop -x SIGINT
system sh/exec.sh -n dnode8 -s stop -x SIGINT
diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim
index a09fd7e4ea99a3ac0ed0fee159111c3b8e1d98b1..b2adb24dfa6c85ec09e37c6c415160963f8e1378 100644
--- a/tests/script/unique/dnode/balance3.sim
+++ b/tests/script/unique/dnode/balance3.sim
@@ -177,7 +177,6 @@ endi
print ========== step5
sql create dnode $hostname6
-system sh/deploy.sh -n dnode6 -i 6
system sh/exec.sh -n dnode6 -s start
$x = 0
diff --git a/tests/test-all.sh b/tests/test-all.sh
index 0c91229120e2d6497bbf4f10e65fc8cf70ae8b68..275c6b16773bb3df00271ea1628dad706a1f4252 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -1,10 +1,22 @@
#!/bin/bash
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
function runSimCaseOneByOne {
while read -r line; do
if [[ $line =~ ^run.* ]]; then
- case=`echo $line | awk '{print $2}'`
- ./test.sh -f $case 2>&1 | grep 'success\|failed\|fault' | grep -v 'default' | tee -a out.log
+ case=`echo $line | awk '{print $NF}'`
+ start_time=`date +%s`
+ ./test.sh -f $case > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
+ echo -e "${RED}$case failed${NC}" | tee -a out.log
+ end_time=`date +%s`
+ echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
fi
done < $1
}
@@ -12,27 +24,29 @@ function runSimCaseOneByOne {
function runPyCaseOneByOne {
while read -r line; do
if [[ $line =~ ^python.* ]]; then
- $line 2>&1 | grep 'successfully executed\|failed\|fault' | grep -v 'default'| tee -a pytest-out.log
+ if [[ $line != *sleep* ]]; then
+ case=`echo $line|awk '{print $NF}'`
+ start_time=`date +%s`
+ $line > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \
+ echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log
+ end_time=`date +%s`
+ echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+ else
+ $line > /dev/null 2>&1
+ fi
fi
done < $1
}
-# Color setting
-RED='\033[0;31m'
-GREEN='\033[1;32m'
-GREEN_DARK='\033[0;32m'
-GREEN_UNDERLINE='\033[4;32m'
-NC='\033[0m'
-
totalFailed=0
totalPyFailed=0
-
-current_dir=`pwd`
+tests_dir=`pwd`
if [ "$2" != "python" ]; then
echo "### run TSIM test case ###"
- cd $current_dir/script
+ cd $tests_dir/script
[ -f out.log ] && rm -f out.log
if [ "$1" == "cron" ]; then
@@ -53,13 +67,13 @@ if [ "$2" != "python" ]; then
totalSuccess=`expr $totalSuccess - $totalBasic`
fi
- echo -e "${GREEN} ### Total $totalSuccess TSIM case(s) succeed! ### ${NC}"
+ echo -e "\n${GREEN} ### Total $totalSuccess TSIM case(s) succeed! ### ${NC}"
totalFailed=`grep 'failed\|fault' out.log | wc -l`
# echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}"
if [ "$totalFailed" -ne "0" ]; then
- echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}"
+ echo -e "\n${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}"
# exit $totalFailed
fi
@@ -67,7 +81,27 @@ fi
if [ "$2" != "sim" ]; then
echo "### run Python test case ###"
- cd $current_dir/pytest
+
+ cd $tests_dir
+ IN_TDINTERNAL="community"
+
+ if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
+ cd ../..
+ else
+ cd ../
+ fi
+
+ TOP_DIR=`pwd`
+ TAOSLIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1`
+ if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then
+ LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4,5`
+ else
+ LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4`
+ fi
+
+ export LD_LIBRARY_PATH=$TOP_DIR/$LIB_DIR:$LD_LIBRARY_PATH
+
+ cd $tests_dir/pytest
[ -f pytest-out.log ] && rm -f pytest-out.log
@@ -81,15 +115,15 @@ if [ "$2" != "sim" ]; then
echo "### run Python smoke test ###"
runPyCaseOneByOne smoketest.sh
fi
- totalPySuccess=`grep 'successfully executed' pytest-out.log | wc -l`
+ totalPySuccess=`grep 'success' pytest-out.log | wc -l`
if [ "$totalPySuccess" -gt "0" ]; then
- echo -e "${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
+ echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
fi
totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
if [ "$totalPyFailed" -ne "0" ]; then
- echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
+ echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
# exit $totalPyFailed
fi
fi
diff --git a/tests/test/c/importOneRow.c b/tests/test/c/importOneRow.c
index 0690d8983786341c09d43ad1d69cf527e758203f..858926106c14afa6fb35bb66d44075c04add89f0 100644
--- a/tests/test/c/importOneRow.c
+++ b/tests/test/c/importOneRow.c
@@ -109,7 +109,7 @@ void* taos_execute(void *param) {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- void *taos = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ void *taos = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (taos == NULL) taos_error(taos);
char sql[1024] = {0};
diff --git a/tests/test/c/importPerTable.c b/tests/test/c/importPerTable.c
index 15b3fc2572e69a0894bc0f6ed6a90cf7af278a73..b4e8b6870342845ddcc75c2703fbe54119c423f8 100644
--- a/tests/test/c/importPerTable.c
+++ b/tests/test/c/importPerTable.c
@@ -73,7 +73,7 @@ void createDbAndTable() {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(1);
@@ -110,7 +110,7 @@ void createDbAndTable() {
for (int64_t t = 0; t < totalTables; ++t) {
sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
if (taos_query(con, qstr)) {
- pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
+ pError("failed to create table %s%" PRId64 ", reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
}
@@ -141,7 +141,7 @@ void insertData() {
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- pPrint("%d threads are spawned to import data", numOfThreads);
+ pPrint("%" PRId64 " threads are spawned to import data", numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
@@ -200,7 +200,7 @@ void *syncTest(void *param) {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
exit(1);
@@ -323,8 +323,8 @@ void shellParseArgument(int argc, char *argv[]) {
pPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
- pPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
- pPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
+ pPrint("%scache:%d%s", GREEN, cache, NC);
+ pPrint("%stables:%d%s", GREEN, tables, NC);
pPrint("%sdbName:%s%s", GREEN, dbName, NC);
pPrint("%stableName:%s%s", GREEN, stableName, NC);
pPrint("%sstart to run%s", GREEN, NC);
diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c
index 6b5a6780938f534ffc1bcab2618301350c6c09e4..9b71ce50dd60fa8dddb7579d9f8a052b812d7b13 100644
--- a/tests/test/c/insertPerRow.c
+++ b/tests/test/c/insertPerRow.c
@@ -33,6 +33,7 @@ typedef struct {
int threadIndex;
char dbName[32];
char stableName[64];
+ float createTableSpeed;
pthread_t thread;
} SInfo;
@@ -49,8 +50,8 @@ int64_t numOfThreads = 1;
int64_t numOfTablesPerThread = 200;
char dbName[32] = "db";
char stableName[64] = "st";
-int32_t cache = 16384;
-int32_t tables = 1000;
+int32_t cache = 16;
+int32_t tables = 5000;
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
@@ -63,36 +64,37 @@ int main(int argc, char *argv[]) {
void createDbAndTable() {
pPrint("start to create table");
+ TAOS_RES * pSql;
TAOS * con;
- struct timeval systemTime;
- int64_t st, et;
char qstr[64000];
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(1);
}
- sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
- if (taos_query(con, qstr)) {
- pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
+ sprintf(qstr, "create database if not exists %s cache %d maxtables %d", dbName, cache, tables);
+ pSql = taos_query(con, qstr);
+ int32_t code = taos_errno(pSql);
+ if (code != 0) {
+ pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con));
exit(0);
}
+ taos_free_result(pSql);
sprintf(qstr, "use %s", dbName);
- if (taos_query(con, qstr)) {
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
+ taos_free_result(pSql);
- gettimeofday(&systemTime, NULL);
- st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- int64_t totalTables = numOfTablesPerThread * numOfThreads;
-
if (strcmp(stableName, "no") != 0) {
int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
for (int64_t f = 0; f < pointsPerTable; ++f) {
@@ -100,36 +102,14 @@ void createDbAndTable() {
}
sprintf(qstr + len, ") tags(t int)");
- if (taos_query(con, qstr)) {
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
-
- for (int64_t t = 0; t < totalTables; ++t) {
- sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
- if (taos_query(con, qstr)) {
- pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
- exit(0);
- }
- }
- } else {
- for (int64_t t = 0; t < totalTables; ++t) {
- int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
- for (int64_t f = 0; f < pointsPerTable; ++f) {
- len += sprintf(qstr + len, ", f%ld double", f);
- }
- sprintf(qstr + len, ")");
-
- if (taos_query(con, qstr)) {
- pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
- exit(0);
- }
- }
+ taos_free_result(pSql);
}
-
- gettimeofday(&systemTime, NULL);
- et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- pPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
}
void insertData() {
@@ -139,12 +119,12 @@ void insertData() {
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- pPrint("%d threads are spawned to insert data", numOfThreads);
+ pPrint("%" PRId64 " threads are spawned to insert data", numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
- SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads);
+ SInfo *pInfo = (SInfo *)calloc(numOfThreads, sizeof(SInfo));
// Start threads to write
for (int i = 0; i < numOfThreads; ++i) {
@@ -173,10 +153,15 @@ void insertData() {
double speedOfRows = totalRows / seconds;
double speedOfPoints = totalPoints / seconds;
+ float createTableSpeed = 0;
+ for (int i = 0; i < numOfThreads; ++i) {
+ createTableSpeed += pInfo[i].createTableSpeed;
+ }
+
pPrint(
"%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf "
- "PointsPerSecond:%.1lf%s",
- GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC);
+ "PointsPerSecond:%.1lf CreateTableSpeed:%.1f t/s %s",
+ GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, createTableSpeed, NC);
pPrint("threads exit");
@@ -191,6 +176,7 @@ void *syncTest(void *param) {
int64_t st, et;
char qstr[65000];
int maxBytes = 60000;
+ int code;
pPrint("thread:%d, start to run", pInfo->threadIndex);
@@ -198,7 +184,7 @@ void *syncTest(void *param) {
uint16_t port;
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
exit(1);
@@ -210,6 +196,48 @@ void *syncTest(void *param) {
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ if (strcmp(stableName, "no") != 0) {
+ for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
+ sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
+ TAOS_RES *pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
+ pError("failed to create table %s%" PRId64 ", reason:%s", stableName, t, taos_errstr(con));
+ exit(0);
+ }
+ taos_free_result(pSql);
+ }
+ } else {
+ for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
+ int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
+ for (int64_t f = 0; f < pointsPerTable; ++f) {
+ len += sprintf(qstr + len, ", f%ld double", f);
+ }
+ sprintf(qstr + len, ")");
+
+ TAOS_RES *pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
+ pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
+ exit(0);
+ }
+ taos_free_result(pSql);
+ }
+ }
+
+ gettimeofday(&systemTime, NULL);
+ et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ float seconds = (et - st) / 1000.0 / 1000.0;
+ int64_t tables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
+ pInfo->createTableSpeed = (float)tables / seconds;
+ pPrint("thread:%d, %.1f seconds to create %ld tables, speed:%.1f", pInfo->threadIndex, seconds, tables,
+ pInfo->createTableSpeed);
+
+ if (pInfo->rowsPerTable == 0) return NULL;
+
+ gettimeofday(&systemTime, NULL);
+ st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+
int64_t start = 1430000000000;
int64_t interval = 1000; // 1000 ms
@@ -227,10 +255,13 @@ void *syncTest(void *param) {
}
len += sprintf(sql + len, ")");
if (len > maxBytes) {
- if (taos_query(con, qstr)) {
+ TAOS_RES *pSql = taos_query(con, qstr);
+ int32_t code = taos_errno(pSql);
+ if (code != 0) {
pError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
table, row, taos_errstr(con));
}
+ taos_free_result(pSql);
// "insert into"
len = sprintf(sql, "%s", inserStr);
@@ -239,7 +270,8 @@ void *syncTest(void *param) {
}
if (len != strlen(inserStr)) {
- taos_query(con, qstr);
+ TAOS_RES *pSql = taos_query(con, qstr);
+ taos_free_result(pSql);
}
gettimeofday(&systemTime, NULL);
@@ -316,8 +348,8 @@ void shellParseArgument(int argc, char *argv[]) {
pPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
- pPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
- pPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
+ pPrint("%scache:%" PRId32 "%s", GREEN, cache, NC);
+ pPrint("%stables:%" PRId32 "%s", GREEN, tables, NC);
pPrint("%sdbName:%s%s", GREEN, dbName, NC);
pPrint("%stableName:%s%s", GREEN, stableName, NC);
pPrint("%sstart to run%s", GREEN, NC);
diff --git a/tests/test/c/insertPerTable.c b/tests/test/c/insertPerTable.c
index a439abec881d21a7d2a1ae51e701f0da59afed44..a5e2c4b966b99734c9137a11c8e410ad76d27ce4 100644
--- a/tests/test/c/insertPerTable.c
+++ b/tests/test/c/insertPerTable.c
@@ -49,8 +49,8 @@ int64_t numOfThreads = 1;
int64_t numOfTablesPerThread = 1;
char dbName[32] = "db";
char stableName[64] = "st";
-int32_t cache = 16384;
-int32_t tables = 1000;
+int32_t cache = 16;
+int32_t tables = 5000;
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
@@ -63,6 +63,7 @@ int main(int argc, char *argv[]) {
void createDbAndTable() {
pPrint("start to create table");
+ TAOS_RES * pSql;
TAOS * con;
struct timeval systemTime;
int64_t st, et;
@@ -73,23 +74,28 @@ void createDbAndTable() {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(1);
}
- sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
- if (taos_query(con, qstr)) {
- pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
+ sprintf(qstr, "create database if not exists %s cache %d maxtables %d", dbName, cache, tables);
+ pSql = taos_query(con, qstr);
+ int32_t code = taos_errno(pSql);
+ if (code != 0) {
+ pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con));
exit(0);
}
sprintf(qstr, "use %s", dbName);
- if (taos_query(con, qstr)) {
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
+ taos_stop_query(pSql);
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
@@ -102,17 +108,23 @@ void createDbAndTable() {
}
sprintf(qstr + len, ") tags(t int)");
- if (taos_query(con, qstr)) {
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
-
+ taos_stop_query(pSql);
+
for (int64_t t = 0; t < totalTables; ++t) {
sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
- if (taos_query(con, qstr)) {
- pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
+ pError("failed to create table %s%" PRId64 ", reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
+ taos_stop_query(pSql);
}
} else {
for (int64_t t = 0; t < totalTables; ++t) {
@@ -122,16 +134,20 @@ void createDbAndTable() {
}
sprintf(qstr + len, ")");
- if (taos_query(con, qstr)) {
+ pSql = taos_query(con, qstr);
+ code = taos_errno(pSql);
+ if (code != 0) {
pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
+ taos_stop_query(pSql);
}
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- pPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
+ float seconds = (et - st) / 1000.0 / 1000.0;
+ pPrint("%.1f seconds to create %ld tables, speed:%.1f", seconds, totalTables, totalTables / seconds);
}
void insertData() {
@@ -141,7 +157,12 @@ void insertData() {
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
- pPrint("%d threads are spawned to insert data", numOfThreads);
+ if (rowsPerTable <= 0) {
+ pPrint("not insert data for rowsPerTable is :%" PRId64, rowsPerTable);
+ exit(0);
+ } else {
+ pPrint("%" PRId64 " threads are spawned to insert data", numOfThreads);
+ }
pthread_attr_t thattr;
pthread_attr_init(&thattr);
@@ -201,7 +222,7 @@ void *syncTest(void *param) {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
- con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port);
+ con = taos_connect(fqdn, "root", "taosdata", NULL, port);
if (con == NULL) {
pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
exit(1);
@@ -230,10 +251,13 @@ void *syncTest(void *param) {
}
len += sprintf(sql + len, ")");
if (len > maxBytes) {
- if (taos_query(con, qstr)) {
+ TAOS_RES *pSql = taos_query(con, qstr);
+ int32_t code = taos_errno(pSql);
+ if (code != 0) {
pError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
table, row, taos_errstr(con));
}
+ taos_stop_query(pSql);
// "insert into"
len = sprintf(sql, "%s", inserStr);
@@ -324,8 +348,8 @@ void shellParseArgument(int argc, char *argv[]) {
pPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
- pPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
- pPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
+ pPrint("%scache:%" PRId32 "%s", GREEN, cache, NC);
+ pPrint("%stables:%" PRId32 "%s", GREEN, tables, NC);
pPrint("%sdbName:%s%s", GREEN, dbName, NC);
pPrint("%stableName:%s%s", GREEN, stableName, NC);
pPrint("%sstart to run%s", GREEN, NC);
diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index f13e861907a8b4701b3556960b0797469f9d4bcc..048ee0486642217b5b8093b8c1571092c5017a39 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -612,7 +612,7 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
}
bool simCreateTaosdConnect(SScript *script, char *rest) {
- char *user = tsDefaultUser;
+ char *user = TSDB_DEFAULT_USER;
char *token;
int tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@@ -622,9 +622,9 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
}
if (simAsyncQuery) {
- return simCreateRestFulConnect(script, user, tsDefaultPass);
+ return simCreateRestFulConnect(script, user, TSDB_DEFAULT_PASS);
} else {
- return simCreateNativeConnect(script, user, tsDefaultPass);
+ return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS);
}
}
@@ -652,6 +652,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
+ taos_free_result(pSql);
return true;
}
taosMsleep(1000);