Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 6a504d36 编写于 作者: H hjxilinx
浏览文件
操作

Merge branch 'develop' into feature/query

上级 53b8867a 89b72161
显示空白变更内容
内联 并排
Showing with 3104 addition and 6578 deletion
.gitignore
浏览文件 @ 6a504d36
......@@ -19,7 +19,6 @@ tests/test/
tests/taoshebei/
tests/taoscsv/
tests/taosdalipu/
tests/pytest/
tests/jenkins/
tests/hdfs/
*.iml
......
.gitmodules 0 → 100644
浏览文件 @ 6a504d36
[submodule "src/connector/go"]
path = src/connector/go
url = https://github.com/taosdata/driver-go
.travis.yml
浏览文件 @ 6a504d36
......@@ -14,6 +14,15 @@ os:
- linux
# - osx
before_install:
- |-
case $TRAVIS_OS_NAME in
linux)
sudo apt -y update
sudo apt -y install python-pip python3-pip python-setuptools python3-setuptools
;;
esac
addons:
coverity_scan:
......@@ -50,6 +59,15 @@ script:
- |-
case $TRAVIS_OS_NAME in
linux)
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
sudo make install
cd ../tests/script
sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt
......@@ -57,16 +75,32 @@ script:
if [ "$total_success" -gt "0" ]; then
total_success=`expr $total_success - 1`
echo -e "${GREEN} ### Total $total_success TSIM case(s) succeed! ### ${NC}"
fi
echo "Total $total_success success"
total_failed=`grep failed out.txt | wc -l`
echo "Total $total_failed failed"
if [ "$total_failed" -ne "0" ]; then
echo -e "${RED} ### Total $total_failed TSIM case(s) failed! ### ${NC}"
exit $total_failed
fi
pip install --user ../../src/connector/python/linux/python2/
pip3 install --user ../../src/connector/python/linux/python3/
cd ../pytest
sudo ./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt
total_py_success=`grep 'successfully executed' pytest-out.txt | wc -l`
if [ "$total_py_success" -gt "0" ]; then
echo -e "${GREEN} ### Total $total_py_success python case(s) succeed! ### ${NC}"
fi
total_py_failed=`grep 'failed' pytest-out.txt | wc -l`
if [ "$total_py_failed" -ne "0" ]; then
echo -e "${RED} ### Total $total_py_failed python case(s) failed! ### ${NC}"
exit $total_py_failed
fi
;;
esac
......@@ -81,6 +115,10 @@ matrix:
- build-essential
- cmake
- net-tools
- python-pip
- python-setuptools
- python3-pip
- python3-setuptools
# - os: osx
# addons:
......
cmake/define.inc
浏览文件 @ 6a504d36
......@@ -11,9 +11,9 @@ ENDIF ()
IF (TD_VPEER)
ADD_DEFINITIONS(-D_VPEER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
#ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
#ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_ACCOUNT)
......
deps/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -5,3 +5,5 @@ ADD_SUBDIRECTORY(zlib-1.2.11)
ADD_SUBDIRECTORY(pthread)
ADD_SUBDIRECTORY(regex)
ADD_SUBDIRECTORY(iconv)
ADD_SUBDIRECTORY(lz4)
ADD_SUBDIRECTORY(cJson)
src/thirdparty/CMakeLists.txt → deps/cJson/CMakeLists.txt
浏览文件 @ 6a504d36
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
add_library(thirdparty ${SOURCE_LIST})
target_include_directories(thirdparty PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)
\ No newline at end of file
add_library(cJson ${SOURCE_LIST})
target_include_directories(cJson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)
\ No newline at end of file
deps/lz4/CMakeLists.txt 0 → 100644
浏览文件 @ 6a504d36
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
add_library(lz4 ${SOURCE_LIST})
target_include_directories(lz4 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)
\ No newline at end of file
documentation/webdocs/markdowndocs/administrator-ch.md
浏览文件 @ 6a504d36
......@@ -105,7 +105,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接
**shellActivityTimer**
- 默认值：3
系统在服务端保持结果集的最长时间，范围[1-120]。
系统在服务端保持结果集的最长时间，单位：秒，范围[1-120]。
**maxUsers**
- 默认值：10,000
......@@ -138,7 +138,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接
系统（服务端和客户端）运行日志开关：
- 131 仅输出错误和警告信息
- 135 输入错误（ERROR）、警告（WARN）、信息（Info）
- 135 输出错误（ERROR）、警告（WARN）、信息（Info）
不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数：
......
src/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -3,8 +3,6 @@ PROJECT(TDengine)
# Base compile
ADD_SUBDIRECTORY(os)
ADD_SUBDIRECTORY(thirdparty)
ADD_SUBDIRECTORY(common)
ADD_SUBDIRECTORY(util)
ADD_SUBDIRECTORY(rpc)
......@@ -14,5 +12,7 @@ ADD_SUBDIRECTORY(kit)
ADD_SUBDIRECTORY(plugins)
ADD_SUBDIRECTORY(mnode)
ADD_SUBDIRECTORY(vnode)
ADD_SUBDIRECTORY(tsdb)
ADD_SUBDIRECTORY(wal)
ADD_SUBDIRECTORY(dnode)
#ADD_SUBDIRECTORY(connector/jdbc)
src/client/src/tscServer.c
浏览文件 @ 6a504d36
......@@ -191,7 +191,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
}
if (pSql->cmd.command < TSDB_SQL_MGMT) {
tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {
......
go @ 8c58c512
Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766
src/connector/go/src/taosSql/connection.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import "C"
import (
"context"
"errors"
"database/sql/driver"
"unsafe"
"strconv"
"strings"
"time"
)
type taosConn struct {
taos unsafe.Pointer
affectedRows int
insertId int
cfg *config
status statusFlag
parseTime bool
reset bool // set when the Go SQL package calls ResetSession
}
type taosSqlResult struct {
affectedRows int64
insertId int64
}
func (res *taosSqlResult) LastInsertId() (int64, error) {
return res.insertId, nil
}
func (res *taosSqlResult) RowsAffected() (int64, error) {
return res.affectedRows, nil
}
func (mc *taosConn) Begin() (driver.Tx, error) {
taosLog.Println("taosSql not support transaction")
return nil, errors.New("taosSql not support transaction")
}
func (mc *taosConn) Close() (err error) {
if mc.taos == nil {
return errConnNoExist
}
mc.taos_close()
return nil
}
func (mc *taosConn) Prepare(query string) (driver.Stmt, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
stmt := &taosSqlStmt{
mc: mc,
pSql: query,
}
// find ? count and save to stmt.paramCount
stmt.paramCount = strings.Count(query, "?")
//fmt.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query)
taosLog.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query)
return stmt, nil
}
func (mc *taosConn) interpolateParams(query string, args []driver.Value) (string, error) {
// Number of ? should be same to len(args)
if strings.Count(query, "?") != len(args) {
return "", driver.ErrSkip
}
buf := make([]byte, defaultBufSize)
buf = buf[:0] // clear buf
argPos := 0
for i := 0; i < len(query); i++ {
q := strings.IndexByte(query[i:], '?')
if q == -1 {
buf = append(buf, query[i:]...)
break
}
buf = append(buf, query[i:i+q]...)
i += q
arg := args[argPos]
argPos++
if arg == nil {
buf = append(buf, "NULL"...)
continue
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
// Handle uint64 explicitly because our custom ConvertValue emits unsigned values
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
v := v.In(mc.cfg.loc)
v = v.Add(time.Nanosecond * 500) // To round under microsecond
year := v.Year()
year100 := year / 100
year1 := year % 100
month := v.Month()
day := v.Day()
hour := v.Hour()
minute := v.Minute()
second := v.Second()
micro := v.Nanosecond() / 1000
buf = append(buf, []byte{
'\'',
digits10[year100], digits01[year100],
digits10[year1], digits01[year1],
'-',
digits10[month], digits01[month],
'-',
digits10[day], digits01[day],
' ',
digits10[hour], digits01[hour],
':',
digits10[minute], digits01[minute],
':',
digits10[second], digits01[second],
}...)
if micro != 0 {
micro10000 := micro / 10000
micro100 := micro / 100 % 100
micro1 := micro % 100
buf = append(buf, []byte{
'.',
digits10[micro10000], digits01[micro10000],
digits10[micro100], digits01[micro100],
digits10[micro1], digits01[micro1],
}...)
}
buf = append(buf, '\'')
}
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
buf = append(buf, "_binary'"...)
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
} else {
buf = escapeBytesQuotes(buf, v)
}
buf = append(buf, '\'')
}
case string:
//buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeStringBackslash(buf, v)
} else {
buf = escapeStringQuotes(buf, v)
}
//buf = append(buf, '\'')
default:
return "", driver.ErrSkip
}
//if len(buf)+4 > mc.maxAllowedPacket {
if len(buf)+4 > maxTaosSqlLen {
return "", driver.ErrSkip
}
}
if argPos != len(args) {
return "", driver.ErrSkip
}
return string(buf), nil
}
func (mc *taosConn) Exec(query string, args []driver.Value) (driver.Result, error) {
if mc.taos == nil {
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try to interpolate the parameters to save extra roundtrips for preparing and closing a statement
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
mc.affectedRows = 0
mc.insertId = 0
_, err := mc.taosQuery(query)
if err == nil {
return &taosSqlResult{
affectedRows: int64(mc.affectedRows),
insertId: int64(mc.insertId),
}, err
}
return nil, err
}
func (mc *taosConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return mc.query(query, args)
}
func (mc *taosConn) query(query string, args []driver.Value) (*textRows, error) {
if mc.taos == nil {
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try client-side prepare to reduce roundtrip
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
num_fields, err := mc.taosQuery(query)
if err == nil {
// Read Result
rows := new(textRows)
rows.mc = mc
// Columns field
rows.rs.columns, err = mc.readColumns(num_fields)
return rows, err
}
return nil, err
}
// Ping implements driver.Pinger interface
func (mc *taosConn) Ping(ctx context.Context) (err error) {
if mc.taos != nil {
return nil
}
return errInvalidConn
}
// BeginTx implements driver.ConnBeginTx interface
func (mc *taosConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
taosLog.Println("taosSql not support transaction")
return nil, errors.New("taosSql not support transaction")
}
func (mc *taosConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
rows, err := mc.query(query, dargs)
if err != nil {
return nil, err
}
return rows, err
}
func (mc *taosConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
return mc.Exec(query, dargs)
}
func (mc *taosConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
stmt, err := mc.Prepare(query)
if err != nil {
return nil, err
}
return stmt, nil
}
func (stmt *taosSqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
if stmt.mc == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
rows, err := stmt.query(dargs)
if err != nil {
return nil, err
}
return rows, err
}
func (stmt *taosSqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
if stmt.mc == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
return stmt.Exec(dargs)
}
func (mc *taosConn) CheckNamedValue(nv *driver.NamedValue) (err error) {
nv.Value, err = converter{}.ConvertValue(nv.Value)
return
}
// ResetSession implements driver.SessionResetter.
// (From Go 1.10)
func (mc *taosConn) ResetSession(ctx context.Context) error {
if mc.taos == nil {
return driver.ErrBadConn
}
mc.reset = true
return nil
}
src/connector/go/src/taosSql/connector.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"context"
"database/sql/driver"
)
type connector struct {
cfg *config
}
// Connect implements driver.Connector interface.
// Connect returns a connection to the database.
func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
var err error
// New taosConn
mc := &taosConn{
cfg: c.cfg,
parseTime: c.cfg.parseTime,
}
// Connect to Server
mc.taos, err = mc.taosConnect(mc.cfg.addr, mc.cfg.user, mc.cfg.passwd, mc.cfg.dbName, mc.cfg.port)
if err != nil {
return nil, err
}
return mc, nil
}
// Driver implements driver.Connector interface.
// Driver returns &taosSQLDriver{}.
func (c *connector) Driver() driver.Driver {
return &taosSQLDriver{}
}
src/connector/go/src/taosSql/const.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
const (
timeFormat = "2006-01-02 15:04:05"
maxTaosSqlLen = 65380
defaultBufSize = maxTaosSqlLen + 32
)
type fieldType byte
type fieldFlag uint16
const (
flagNotNULL fieldFlag = 1 << iota
)
type statusFlag uint16
const (
statusInTrans statusFlag = 1 << iota
statusInAutocommit
statusReserved // Not in documentation
statusMoreResultsExists
statusNoGoodIndexUsed
statusNoIndexUsed
statusCursorExists
statusLastRowSent
statusDbDropped
statusNoBackslashEscapes
statusMetadataChanged
statusQueryWasSlow
statusPsOutParams
statusInTransReadonly
statusSessionStateChanged
)
src/connector/go/src/taosSql/driver.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"context"
"database/sql"
"database/sql/driver"
)
// taosSqlDriver is exported to make the driver directly accessible.
// In general the driver is used via the database/sql package.
type taosSQLDriver struct{}
// Open new Connection.
// the DSN string is formatted
func (d taosSQLDriver) Open(dsn string) (driver.Conn, error) {
cfg, err := parseDSN(dsn)
if err != nil {
return nil, err
}
c := &connector{
cfg: cfg,
}
return c.Connect(context.Background())
}
func init() {
sql.Register("taosSql", &taosSQLDriver{})
taosLogInit()
}
src/connector/go/src/taosSql/dsn.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"errors"
"net/url"
"strconv"
"strings"
"time"
)
var (
errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?")
errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)")
errInvalidDSNPort = errors.New("invalid DSN: network port is not a valid number")
errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name")
)
// Config is a configuration parsed from a DSN string.
// If a new Config is created instead of being parsed from a DSN string,
// the NewConfig function should be used, which sets default values.
type config struct {
user string // Username
passwd string // Password (requires User)
net string // Network type
addr string // Network address (requires Net)
port int
dbName string // Database name
params map[string]string // Connection parameters
loc *time.Location // Location for time.Time values
columnsWithAlias bool // Prepend table alias to column names
interpolateParams bool // Interpolate placeholders into query string
parseTime bool // Parse time values to time.Time
}
// NewConfig creates a new Config and sets default values.
func newConfig() *config {
return &config{
loc: time.UTC,
interpolateParams: true,
parseTime: true,
}
}
// ParseDSN parses the DSN string to a Config
func parseDSN(dsn string) (cfg *config, err error) {
taosLog.Println("input dsn:", dsn)
// New config with some default values
cfg = newConfig()
// [user[:password]@][net[(addr)]]/dbname[?param1=value1&paramN=valueN]
// Find the last '/' (since the password or the net addr might contain a '/')
foundSlash := false
for i := len(dsn) - 1; i >= 0; i-- {
if dsn[i] == '/' {
foundSlash = true
var j, k int
// left part is empty if i <= 0
if i > 0 {
// [username[:password]@][protocol[(address)]]
// Find the last '@' in dsn[:i]
for j = i; j >= 0; j-- {
if dsn[j] == '@' {
// username[:password]
// Find the first ':' in dsn[:j]
for k = 0; k < j; k++ {
if dsn[k] == ':' {
cfg.passwd = dsn[k+1 : j]
break
}
}
cfg.user = dsn[:k]
break
}
}
// [protocol[(address)]]
// Find the first '(' in dsn[j+1:i]
for k = j + 1; k < i; k++ {
if dsn[k] == '(' {
// dsn[i-1] must be == ')' if an address is specified
if dsn[i-1] != ')' {
if strings.ContainsRune(dsn[k+1:i], ')') {
return nil, errInvalidDSNUnescaped
}
return nil, errInvalidDSNAddr
}
strs := strings.Split(dsn[k+1:i-1], ":")
if len(strs) == 1 {
return nil, errInvalidDSNAddr
}
cfg.addr = strs[0]
cfg.port, err = strconv.Atoi(strs[1])
if err != nil {
return nil, errInvalidDSNPort
}
break
}
}
cfg.net = dsn[j+1 : k]
}
// dbname[?param1=value1&...&paramN=valueN]
// Find the first '?' in dsn[i+1:]
for j = i + 1; j < len(dsn); j++ {
if dsn[j] == '?' {
if err = parseDSNParams(cfg, dsn[j+1:]); err != nil {
return
}
break
}
}
cfg.dbName = dsn[i+1 : j]
break
}
}
if !foundSlash && len(dsn) > 0 {
return nil, errInvalidDSNNoSlash
}
taosLog.Printf("cfg info: %+v", cfg)
return
}
// parseDSNParams parses the DSN "query string"
// Values must be url.QueryEscape'ed
func parseDSNParams(cfg *config, params string) (err error) {
for _, v := range strings.Split(params, "&") {
param := strings.SplitN(v, "=", 2)
if len(param) != 2 {
continue
}
// cfg params
switch value := param[1]; param[0] {
case "columnsWithAlias":
var isBool bool
cfg.columnsWithAlias, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
// Enable client side placeholder substitution
case "interpolateParams":
var isBool bool
cfg.interpolateParams, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
// Time Location
case "loc":
if value, err = url.QueryUnescape(value); err != nil {
return
}
cfg.loc, err = time.LoadLocation(value)
if err != nil {
return
}
// time.Time parsing
case "parseTime":
var isBool bool
cfg.parseTime, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
default:
// lazy init
if cfg.params == nil {
cfg.params = make(map[string]string)
}
if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil {
return
}
}
}
return
}
src/connector/go/src/taosSql/result.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"database/sql/driver"
"errors"
"fmt"
"io"
"strconv"
"time"
"unsafe"
)
/******************************************************************************
* Result *
******************************************************************************/
// Read Packets as Field Packets until EOF-Packet or an Error appears
func (mc *taosConn) readColumns(count int) ([]taosSqlField, error) {
columns := make([]taosSqlField, count)
var result unsafe.Pointer
result = C.taos_use_result(mc.taos)
if result == nil {
return nil, errors.New("invalid result")
}
pFields := (*C.struct_taosField)(C.taos_fetch_fields(result))
// TODO: Optimized rewriting !!!!
fields := (*[1 << 30]C.struct_taosField)(unsafe.Pointer(pFields))
for i := 0; i < count; i++ {
//columns[i].tableName = ms.taos.
//fmt.Println(reflect.TypeOf(fields[i].name))
var charray []byte
for j := range fields[i].name {
//fmt.Println("fields[i].name[j]: ", fields[i].name[j])
if fields[i].name[j] != 0 {
charray = append(charray, byte(fields[i].name[j]))
} else {
break
}
}
columns[i].name = string(charray)
columns[i].length = (uint32)(fields[i].bytes)
columns[i].fieldType = fieldType(fields[i]._type)
columns[i].flags = 0
// columns[i].decimals = 0
//columns[i].charSet = 0
}
return columns, nil
}
func (rows *taosSqlRows) readRow(dest []driver.Value) error {
mc := rows.mc
if rows.rs.done || mc == nil {
return io.EOF
}
var result unsafe.Pointer
result = C.taos_use_result(mc.taos)
if result == nil {
return errors.New(C.GoString(C.taos_errstr(mc.taos)))
}
//var row *unsafe.Pointer
row := C.taos_fetch_row(result)
if row == nil {
rows.rs.done = true
C.taos_free_result(result)
rows.mc = nil
return io.EOF
}
// because sizeof(void*) == sizeof(int*) == 8
// notes: sizeof(int) == 8 in go, but sizeof(int) == 4 in C.
for i := range dest {
currentRow := (unsafe.Pointer)(uintptr(*((*int)(unsafe.Pointer(uintptr(unsafe.Pointer(row)) + uintptr(i)*unsafe.Sizeof(int(0)))))))
if currentRow == nil {
dest[i] = nil
continue
}
switch rows.rs.columns[i].fieldType {
case C.TSDB_DATA_TYPE_BOOL:
if (*((*byte)(currentRow))) != 0 {
dest[i] = true
} else {
dest[i] = false
}
break
case C.TSDB_DATA_TYPE_TINYINT:
dest[i] = (int)(*((*byte)(currentRow)))
break
case C.TSDB_DATA_TYPE_SMALLINT:
dest[i] = (int16)(*((*int16)(currentRow)))
break
case C.TSDB_DATA_TYPE_INT:
dest[i] = (int)(*((*int32)(currentRow))) // notes int32 of go <----> int of C
break
case C.TSDB_DATA_TYPE_BIGINT:
dest[i] = (int64)(*((*int64)(currentRow)))
break
case C.TSDB_DATA_TYPE_FLOAT:
dest[i] = (*((*float32)(currentRow)))
break
case C.TSDB_DATA_TYPE_DOUBLE:
dest[i] = (*((*float64)(currentRow)))
break
case C.TSDB_DATA_TYPE_BINARY, C.TSDB_DATA_TYPE_NCHAR:
charLen := rows.rs.columns[i].length
var index uint32
binaryVal := make([]byte, charLen)
for index = 0; index < charLen; index++ {
binaryVal[index] = *((*byte)(unsafe.Pointer(uintptr(currentRow) + uintptr(index))))
}
dest[i] = string(binaryVal[:])
break
case C.TSDB_DATA_TYPE_TIMESTAMP:
if mc.cfg.parseTime == true {
timestamp := (int64)(*((*int64)(currentRow)))
dest[i] = timestampConvertToString(timestamp, int(C.taos_result_precision(result)))
} else {
dest[i] = (int64)(*((*int64)(currentRow)))
}
break
default:
fmt.Println("default fieldType: set dest[] to nil")
dest[i] = nil
break
}
}
return nil
}
// Read result as Field format until all rows or an Error appears
// call this func in conn mode
func (rows *textRows) readRow(dest []driver.Value) error {
return rows.taosSqlRows.readRow(dest)
}
// call thsi func in stmt mode
func (rows *binaryRows) readRow(dest []driver.Value) error {
return rows.taosSqlRows.readRow(dest)
}
func timestampConvertToString(timestamp int64, precision int) string {
var decimal, sVal, nsVal int64
if precision == 0 {
decimal = timestamp % 1000
sVal = timestamp / 1000
nsVal = decimal * 1000
} else {
decimal = timestamp % 1000000
sVal = timestamp / 1000000
nsVal = decimal * 1000000
}
date_time := time.Unix(sVal, nsVal)
//const base_format = "2006-01-02 15:04:05"
str_time := date_time.Format(timeFormat)
return (str_time + "." + strconv.Itoa(int(decimal)))
}
src/connector/go/src/taosSql/rows.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"database/sql"
"database/sql/driver"
"io"
"math"
"reflect"
)
type taosSqlField struct {
tableName string
name string
length uint32
flags fieldFlag // indicate whether this field can is null
fieldType fieldType
decimals byte
charSet uint8
}
type resultSet struct {
columns []taosSqlField
columnNames []string
done bool
}
type taosSqlRows struct {
mc *taosConn
rs resultSet
}
type binaryRows struct {
taosSqlRows
}
type textRows struct {
taosSqlRows
}
func (rows *taosSqlRows) Columns() []string {
if rows.rs.columnNames != nil {
return rows.rs.columnNames
}
columns := make([]string, len(rows.rs.columns))
if rows.mc != nil && rows.mc.cfg.columnsWithAlias {
for i := range columns {
if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 {
columns[i] = tableName + "." + rows.rs.columns[i].name
} else {
columns[i] = rows.rs.columns[i].name
}
}
} else {
for i := range columns {
columns[i] = rows.rs.columns[i].name
}
}
rows.rs.columnNames = columns
return columns
}
func (rows *taosSqlRows) ColumnTypeDatabaseTypeName(i int) string {
return rows.rs.columns[i].typeDatabaseName()
}
func (rows *taosSqlRows) ColumnTypeLength(i int) (length int64, ok bool) {
return int64(rows.rs.columns[i].length), true
}
func (rows *taosSqlRows) ColumnTypeNullable(i int) (nullable, ok bool) {
return rows.rs.columns[i].flags&flagNotNULL == 0, true
}
func (rows *taosSqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) {
column := rows.rs.columns[i]
decimals := int64(column.decimals)
switch column.fieldType {
case C.TSDB_DATA_TYPE_FLOAT:
fallthrough
case C.TSDB_DATA_TYPE_DOUBLE:
if decimals == 0x1f {
return math.MaxInt64, math.MaxInt64, true
}
return math.MaxInt64, decimals, true
}
return 0, 0, false
}
func (rows *taosSqlRows) ColumnTypeScanType(i int) reflect.Type {
return rows.rs.columns[i].scanType()
}
func (rows *taosSqlRows) Close() error {
if rows.mc != nil {
result := C.taos_use_result(rows.mc.taos)
if result != nil {
C.taos_free_result(result)
}
rows.mc = nil
}
return nil
}
func (rows *taosSqlRows) HasNextResultSet() (b bool) {
if rows.mc == nil {
return false
}
return rows.mc.status&statusMoreResultsExists != 0
}
func (rows *taosSqlRows) nextResultSet() (int, error) {
if rows.mc == nil {
return 0, io.EOF
}
// Remove unread packets from stream
if !rows.rs.done {
rows.rs.done = true
}
if !rows.HasNextResultSet() {
rows.mc = nil
return 0, io.EOF
}
rows.rs = resultSet{}
return 0,nil
}
func (rows *taosSqlRows) nextNotEmptyResultSet() (int, error) {
for {
resLen, err := rows.nextResultSet()
if err != nil {
return 0, err
}
if resLen > 0 {
return resLen, nil
}
rows.rs.done = true
}
}
func (rows *binaryRows) NextResultSet() error {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
// stmt.Query return binary rows, and get row from this func
func (rows *binaryRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
func (rows *textRows) NextResultSet() (err error) {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
// db.Query return text rows, and get row from this func
func (rows *textRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
func (mf *taosSqlField) typeDatabaseName() string {
//fmt.Println("######## (mf *taosSqlField) typeDatabaseName() mf.fieldType:", mf.fieldType)
switch mf.fieldType {
case C.TSDB_DATA_TYPE_BOOL:
return "BOOL"
case C.TSDB_DATA_TYPE_TINYINT:
return "TINYINT"
case C.TSDB_DATA_TYPE_SMALLINT:
return "SMALLINT"
case C.TSDB_DATA_TYPE_INT:
return "INT"
case C.TSDB_DATA_TYPE_BIGINT:
return "BIGINT"
case C.TSDB_DATA_TYPE_FLOAT:
return "FLOAT"
case C.TSDB_DATA_TYPE_DOUBLE:
return "DOUBLE"
case C.TSDB_DATA_TYPE_BINARY:
return "BINARY"
case C.TSDB_DATA_TYPE_NCHAR:
return "NCHAR"
case C.TSDB_DATA_TYPE_TIMESTAMP:
return "TIMESTAMP"
default:
return ""
}
}
var (
scanTypeFloat32 = reflect.TypeOf(float32(0))
scanTypeFloat64 = reflect.TypeOf(float64(0))
scanTypeInt8 = reflect.TypeOf(int8(0))
scanTypeInt16 = reflect.TypeOf(int16(0))
scanTypeInt32 = reflect.TypeOf(int32(0))
scanTypeInt64 = reflect.TypeOf(int64(0))
scanTypeNullTime = reflect.TypeOf(NullTime{})
scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{})
scanTypeUnknown = reflect.TypeOf(new(interface{}))
)
func (mf *taosSqlField) scanType() reflect.Type {
//fmt.Println("######## (mf *taosSqlField) scanType() mf.fieldType:", mf.fieldType)
switch mf.fieldType {
case C.TSDB_DATA_TYPE_BOOL:
return scanTypeInt8
case C.TSDB_DATA_TYPE_TINYINT:
return scanTypeInt8
case C.TSDB_DATA_TYPE_SMALLINT:
return scanTypeInt16
case C.TSDB_DATA_TYPE_INT:
return scanTypeInt32
case C.TSDB_DATA_TYPE_BIGINT:
return scanTypeInt64
case C.TSDB_DATA_TYPE_FLOAT:
return scanTypeFloat32
case C.TSDB_DATA_TYPE_DOUBLE:
return scanTypeFloat64
case C.TSDB_DATA_TYPE_BINARY:
return scanTypeRawBytes
case C.TSDB_DATA_TYPE_NCHAR:
return scanTypeRawBytes
case C.TSDB_DATA_TYPE_TIMESTAMP:
return scanTypeNullTime
default:
return scanTypeUnknown
}
}
src/connector/go/src/taosSql/statement.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"database/sql/driver"
"fmt"
"reflect"
)
type taosSqlStmt struct {
mc *taosConn
id uint32
pSql string
paramCount int
}
func (stmt *taosSqlStmt) Close() error {
return nil
}
func (stmt *taosSqlStmt) NumInput() int {
return stmt.paramCount
}
func (stmt *taosSqlStmt) Exec(args []driver.Value) (driver.Result, error) {
if stmt.mc == nil || stmt.mc.taos == nil {
return nil, errInvalidConn
}
return stmt.mc.Exec(stmt.pSql, args)
}
func (stmt *taosSqlStmt) Query(args []driver.Value) (driver.Rows, error) {
if stmt.mc == nil || stmt.mc.taos == nil {
return nil, errInvalidConn
}
return stmt.query(args)
}
func (stmt *taosSqlStmt) query(args []driver.Value) (*binaryRows, error) {
mc := stmt.mc
if mc == nil || mc.taos == nil {
return nil, errInvalidConn
}
querySql := stmt.pSql
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try client-side prepare to reduce roundtrip
prepared, err := mc.interpolateParams(stmt.pSql, args)
if err != nil {
return nil, err
}
querySql = prepared
}
num_fields, err := mc.taosQuery(querySql)
if err == nil {
// Read Result
rows := new(binaryRows)
rows.mc = mc
// Columns field
rows.rs.columns, err = mc.readColumns(num_fields)
return rows, err
}
return nil, err
}
type converter struct{}
// ConvertValue mirrors the reference/default converter in database/sql/driver
// with _one_ exception. We support uint64 with their high bit and the default
// implementation does not. This function should be kept in sync with
// database/sql/driver defaultConverter.ConvertValue() except for that
// deliberate difference.
func (c converter) ConvertValue(v interface{}) (driver.Value, error) {
if driver.IsValue(v) {
return v, nil
}
if vr, ok := v.(driver.Valuer); ok {
sv, err := callValuerValue(vr)
if err != nil {
return nil, err
}
if !driver.IsValue(sv) {
return nil, fmt.Errorf("non-Value type %T returned from Value", sv)
}
return sv, nil
}
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Ptr:
// indirect pointers
if rv.IsNil() {
return nil, nil
} else {
return c.ConvertValue(rv.Elem().Interface())
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return rv.Int(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return rv.Uint(), nil
case reflect.Float32, reflect.Float64:
return rv.Float(), nil
case reflect.Bool:
return rv.Bool(), nil
case reflect.Slice:
ek := rv.Type().Elem().Kind()
if ek == reflect.Uint8 {
return rv.Bytes(), nil
}
return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek)
case reflect.String:
return rv.String(), nil
}
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
}
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
// callValuerValue returns vr.Value(), with one exception:
// If vr.Value is an auto-generated method on a pointer type and the
// pointer is nil, it would panic at runtime in the panicwrap
// method. Treat it like nil instead.
//
// This is so people can implement driver.Value on value types and
// still use nil pointers to those types to mean nil/NULL, just like
// string/*string.
//
// This is an exact copy of the same-named unexported function from the
// database/sql package.
func callValuerValue(vr driver.Valuer) (v driver.Value, err error) {
if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr &&
rv.IsNil() &&
rv.Type().Elem().Implements(valuerReflectType) {
return nil, nil
}
return vr.Value()
}
src/connector/go/src/taosSql/taosLog.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"bufio"
"errors"
"fmt"
"io"
"log"
"os"
"strings"
)
// Various errors the driver might return.
var (
errInvalidConn = errors.New("invalid connection")
errConnNoExist = errors.New("no existent connection ")
)
var taosLog *log.Logger
// SetLogger is used to set the logger for critical errors.
// The initial logger
func taosLogInit() {
cfgName := "/etc/taos/taos.cfg"
logNameDefault := "/var/log/taos/taosgo.log"
var logName string
// get log path from cfg file
cfgFile, err := os.OpenFile(cfgName, os.O_RDONLY, 0644)
defer cfgFile.Close()
if err != nil {
fmt.Println(err)
logName = logNameDefault
} else {
logName, err = getLogNameFromCfg(cfgFile)
if err != nil {
fmt.Println(err)
logName = logNameDefault
}
}
logFile, err := os.OpenFile(logName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
taosLog = log.New(logFile, "", log.LstdFlags)
taosLog.SetPrefix("TAOS DRIVER ")
taosLog.SetFlags(log.LstdFlags|log.Lshortfile)
}
func getLogNameFromCfg(f *os.File) (string, error) {
// Create file buf, *Reader
r := bufio.NewReader(f)
for {
//read one line, return to slice b
b, _, err := r.ReadLine()
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
// Remove space of left and right
s := strings.TrimSpace(string(b))
if strings.Index(s, "#") == 0 {
// comment line
continue
}
if len(s) == 0 {
continue
}
var ns string
// If there is a comment on the right of the line, must be remove
index := strings.Index(s, "#")
if index > 0 {
// Gets the string to the left of the comment to determine whether it is empty
ns = s[:index]
if len(ns) == 0 {
continue
}
} else {
ns = s;
}
ss := strings.Fields(ns)
if strings.Compare("logDir", ss[0]) != 0 {
continue
}
if len(ss) < 2 {
break
}
// Add a filename after the path
logName := ss[1] + "/taosgo.log"
return logName,nil
}
return "", errors.New("no config log path, use default")
}
src/connector/go/src/taosSql/taosSqlCgo.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"errors"
"unsafe"
)
func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) {
cuser := C.CString(user)
cpass := C.CString(pass)
cip := C.CString(ip)
cdb := C.CString(db)
defer C.free(unsafe.Pointer(cip))
defer C.free(unsafe.Pointer(cuser))
defer C.free(unsafe.Pointer(cpass))
defer C.free(unsafe.Pointer(cdb))
taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.ushort)(port))
if taosObj == nil {
return nil, errors.New("taos_connect() fail!")
}
return (unsafe.Pointer)(taosObj), nil
}
func (mc *taosConn) taosQuery(sqlstr string) (int, error) {
//taosLog.Printf("taosQuery() input sql:%s\n", sqlstr)
csqlstr := C.CString(sqlstr)
defer C.free(unsafe.Pointer(csqlstr))
code := int(C.taos_query(mc.taos, csqlstr))
if 0 != code {
mc.taos_error()
errStr := C.GoString(C.taos_errstr(mc.taos))
taosLog.Println("taos_query() failed:", errStr)
taosLog.Printf("taosQuery() input sql:%s\n", sqlstr)
return 0, errors.New(errStr)
}
// read result and save into mc struct
num_fields := int(C.taos_field_count(mc.taos))
if 0 == num_fields { // there are no select and show kinds of commands
mc.affectedRows = int(C.taos_affected_rows(mc.taos))
mc.insertId = 0
}
return num_fields, nil
}
func (mc *taosConn) taos_close() {
C.taos_close(mc.taos)
}
func (mc *taosConn) taos_error() {
// free local resouce: allocated memory/metric-meta refcnt
//var pRes unsafe.Pointer
pRes := C.taos_use_result(mc.taos)
C.taos_free_result(pRes)
}
src/connector/go/src/taosSql/utils.go 已删除 100755 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#include <stdlib.h>
#cgo LDFLAGS: -L/usr/lib -ltaos
void taosSetAllocMode(int mode, const char* path, _Bool autoDump);
void taosDumpMemoryLeak();
*/
import "C"
import (
"database/sql/driver"
"errors"
"fmt"
"sync/atomic"
"time"
"unsafe"
)
// Returns the bool value of the input.
// The 2nd return value indicates if the input was a valid bool value
func readBool(input string) (value bool, valid bool) {
switch input {
case "1", "true", "TRUE", "True":
return true, true
case "0", "false", "FALSE", "False":
return false, true
}
// Not a valid bool value
return
}
/******************************************************************************
* Time related utils *
******************************************************************************/
// NullTime represents a time.Time that may be NULL.
// NullTime implements the Scanner interface so
// it can be used as a scan destination:
//
// var nt NullTime
// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
// ...
// if nt.Valid {
// // use nt.Time
// } else {
// // NULL value
// }
//
// This NullTime implementation is not driver-specific
type NullTime struct {
Time time.Time
Valid bool // Valid is true if Time is not NULL
}
// Scan implements the Scanner interface.
// The value type must be time.Time or string / []byte (formatted time-string),
// otherwise Scan fails.
func (nt *NullTime) Scan(value interface{}) (err error) {
if value == nil {
nt.Time, nt.Valid = time.Time{}, false
return
}
switch v := value.(type) {
case time.Time:
nt.Time, nt.Valid = v, true
return
case []byte:
nt.Time, err = parseDateTime(string(v), time.UTC)
nt.Valid = (err == nil)
return
case string:
nt.Time, err = parseDateTime(v, time.UTC)
nt.Valid = (err == nil)
return
}
nt.Valid = false
return fmt.Errorf("Can't convert %T to time.Time", value)
}
// Value implements the driver Valuer interface.
func (nt NullTime) Value() (driver.Value, error) {
if !nt.Valid {
return nil, nil
}
return nt.Time, nil
}
func parseDateTime(str string, loc *time.Location) (t time.Time, err error) {
base := "0000-00-00 00:00:00.0000000"
switch len(str) {
case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM"
if str == base[:len(str)] {
return
}
t, err = time.Parse(timeFormat[:len(str)], str)
default:
err = fmt.Errorf("invalid time string: %s", str)
return
}
// Adjust location
if err == nil && loc != time.UTC {
y, mo, d := t.Date()
h, mi, s := t.Clock()
t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil
}
return
}
// zeroDateTime is used in formatBinaryDateTime to avoid an allocation
// if the DATE or DATETIME has the zero value.
// It must never be changed.
// The current behavior depends on database/sql copying the result.
var zeroDateTime = []byte("0000-00-00 00:00:00.000000")
const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999"
/******************************************************************************
* Convert from and to bytes *
******************************************************************************/
func uint64ToBytes(n uint64) []byte {
return []byte{
byte(n),
byte(n >> 8),
byte(n >> 16),
byte(n >> 24),
byte(n >> 32),
byte(n >> 40),
byte(n >> 48),
byte(n >> 56),
}
}
func uint64ToString(n uint64) []byte {
var a [20]byte
i := 20
// U+0030 = 0
// ...
// U+0039 = 9
var q uint64
for n >= 10 {
i--
q = n / 10
a[i] = uint8(n-q*10) + 0x30
n = q
}
i--
a[i] = uint8(n) + 0x30
return a[i:]
}
// treats string value as unsigned integer representation
func stringToInt(b []byte) int {
val := 0
for i := range b {
val *= 10
val += int(b[i] - 0x30)
}
return val
}
// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize.
// If cap(buf) is not enough, reallocate new buffer.
func reserveBuffer(buf []byte, appendSize int) []byte {
newSize := len(buf) + appendSize
if cap(buf) < newSize {
// Grow buffer exponentially
newBuf := make([]byte, len(buf)*2+appendSize)
copy(newBuf, buf)
buf = newBuf
}
return buf[:newSize]
}
// escapeBytesBackslash escapes []byte with backslashes (\)
// This escapes the contents of a string (provided as []byte) by adding backslashes before special
// characters, and turning others into specific escape sequences, such as
// turning newlines into \n and null bytes into \0.
func escapeBytesBackslash(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for _, c := range v {
switch c {
case '\x00':
buf[pos] = '\\'
buf[pos+1] = '0'
pos += 2
case '\n':
buf[pos] = '\\'
buf[pos+1] = 'n'
pos += 2
case '\r':
buf[pos] = '\\'
buf[pos+1] = 'r'
pos += 2
case '\x1a':
buf[pos] = '\\'
buf[pos+1] = 'Z'
pos += 2
case '\'':
buf[pos] = '\\'
buf[pos+1] = '\''
pos += 2
case '"':
buf[pos] = '\\'
buf[pos+1] = '"'
pos += 2
case '\\':
buf[pos] = '\\'
buf[pos+1] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeStringBackslash is similar to escapeBytesBackslash but for string.
func escapeStringBackslash(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
c := v[i]
switch c {
case '\x00':
buf[pos] = '\\'
buf[pos+1] = '0'
pos += 2
case '\n':
buf[pos] = '\\'
buf[pos+1] = 'n'
pos += 2
case '\r':
buf[pos] = '\\'
buf[pos+1] = 'r'
pos += 2
case '\x1a':
buf[pos] = '\\'
buf[pos+1] = 'Z'
pos += 2
//case '\'':
// buf[pos] = '\\'
// buf[pos+1] = '\''
// pos += 2
case '"':
buf[pos] = '\\'
buf[pos+1] = '"'
pos += 2
case '\\':
buf[pos] = '\\'
buf[pos+1] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeBytesQuotes escapes apostrophes in []byte by doubling them up.
// This escapes the contents of a string by doubling up any apostrophes that
// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in
// effect on the server.
func escapeBytesQuotes(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for _, c := range v {
if c == '\'' {
buf[pos] = '\''
buf[pos+1] = '\''
pos += 2
} else {
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeStringQuotes is similar to escapeBytesQuotes but for string.
func escapeStringQuotes(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
c := v[i]
if c == '\'' {
buf[pos] = '\''
buf[pos+1] = '\''
pos += 2
} else {
buf[pos] = c
pos++
}
}
return buf[:pos]
}
/******************************************************************************
* Sync utils *
******************************************************************************/
// noCopy may be embedded into structs which must not be copied
// after the first use.
//
// See https://github.com/golang/go/issues/8005#issuecomment-190753527
// for details.
type noCopy struct{}
// Lock is a no-op used by -copylocks checker from `go vet`.
func (*noCopy) Lock() {}
// atomicBool is a wrapper around uint32 for usage as a boolean value with
// atomic access.
type atomicBool struct {
_noCopy noCopy
value uint32
}
// IsSet returns whether the current boolean value is true
func (ab *atomicBool) IsSet() bool {
return atomic.LoadUint32(&ab.value) > 0
}
// Set sets the value of the bool regardless of the previous value
func (ab *atomicBool) Set(value bool) {
if value {
atomic.StoreUint32(&ab.value, 1)
} else {
atomic.StoreUint32(&ab.value, 0)
}
}
// TrySet sets the value of the bool and returns whether the value changed
func (ab *atomicBool) TrySet(value bool) bool {
if value {
return atomic.SwapUint32(&ab.value, 1) == 0
}
return atomic.SwapUint32(&ab.value, 0) > 0
}
// atomicError is a wrapper for atomically accessed error values
type atomicError struct {
_noCopy noCopy
value atomic.Value
}
// Set sets the error value regardless of the previous value.
// The value must not be nil
func (ae *atomicError) Set(value error) {
ae.value.Store(value)
}
// Value returns the current error value
func (ae *atomicError) Value() error {
if v := ae.value.Load(); v != nil {
// this will panic if the value doesn't implement the error interface
return v.(error)
}
return nil
}
func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) {
dargs := make([]driver.Value, len(named))
for n, param := range named {
if len(param.Name) > 0 {
// TODO: support the use of Named Parameters #561
return nil, errors.New("taosSql: driver does not support the use of Named Parameters")
}
dargs[n] = param.Value
}
return dargs, nil
}
/******************************************************************************
* Utils for C memory issues debugging *
******************************************************************************/
func SetAllocMode(mode int32, path string) {
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
C.taosSetAllocMode(C.int(mode), cpath, false)
}
func DumpMemoryLeak() {
C.taosDumpMemoryLeak()
}
src/dnode/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -7,14 +7,16 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode)
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4)
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
......@@ -31,6 +33,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
TARGET_LINK_LIBRARIES(taosd balance sync)
ENDIF ()
IF (TD_MPEER)
TARGET_LINK_LIBRARIES(taosd mpeer sync)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target")
ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
......
src/dnode/inc/dnodeMClient.h
浏览文件 @ 6a504d36
......@@ -24,6 +24,8 @@ int32_t dnodeInitMClient();
void dnodeCleanupMClient();
void dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
uint32_t dnodeGetMnodeMasteIp();
void * dnodeGetMpeerInfos();
int32_t dnodeGetDnodeId();
#ifdef __cplusplus
}
......
src/dnode/inc/dnodeMgmt.h
浏览文件 @ 6a504d36
......@@ -23,7 +23,6 @@ extern "C" {
int32_t dnodeInitMgmt();
void dnodeCleanupMgmt();
void dnodeMgmt(SRpcMsg *rpcMsg);
void dnodeUpdateDnodeId(int32_t dnodeId);
void* dnodeGetVnode(int32_t vgId);
int32_t dnodeGetVnodeStatus(void *pVnode);
......
src/dnode/src/dnodeMClient.c
浏览文件 @ 6a504d36
......@@ -15,37 +15,69 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "cJSON.h"
#include "taosmsg.h"
#include "tlog.h"
#include "trpc.h"
#include "tutil.h"
#include "tsync.h"
#include "ttime.h"
#include "ttimer.h"
#include "dnode.h"
#include "dnodeMClient.h"
#include "dnodeModule.h"
#include "dnodeMgmt.h"
#include "vnode.h"
#include "mpeer.h"
#define MPEER_CONTENT_LEN 2000
static bool dnodeReadMnodeIpList();
static void dnodeSaveMnodeIpList();
static void dnodeReadDnodeInfo();
static void dnodeUpdateDnodeInfo(int32_t dnodeId);
static void dnodeProcessRspFromMnode(SRpcMsg *pMsg);
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void *tsDnodeMClientRpc = NULL;
static SRpcIpSet tsDnodeMnodeIpList = {0};
static SRpcIpSet tsMnodeIpList = {0};
static SDMNodeInfos tsMnodeInfos = {0};
static void *tsDnodeTmr = NULL;
static void *tsStatusTimer = NULL;
static uint32_t tsRebootTime;
static int32_t tsDnodeId = 0;
static char tsDnodeName[TSDB_NODE_NAME_LEN];
int32_t dnodeInitMClient() {
dnodeReadDnodeInfo();
tsRebootTime = taosGetTimestampSec();
tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
return -1;
}
if (!dnodeReadMnodeIpList()) {
dTrace("failed to read mnode iplist, set it from cfg file");
memset(&tsDnodeMnodeIpList, 0, sizeof(SRpcIpSet));
tsDnodeMnodeIpList.port = tsMnodeDnodePort;
tsDnodeMnodeIpList.numOfIps = 1;
tsDnodeMnodeIpList.ip[0] = inet_addr(tsMasterIp);
if (tsSecondIp[0]) {
tsDnodeMnodeIpList.numOfIps = 2;
tsDnodeMnodeIpList.ip[1] = inet_addr(tsSecondIp);
memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet));
memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos));
tsMnodeIpList.port = tsMnodeDnodePort;
tsMnodeIpList.numOfIps = 1;
tsMnodeIpList.ip[0] = inet_addr(tsMasterIp);
if (strcmp(tsSecondIp, tsMasterIp) != 0) {
tsMnodeIpList.numOfIps = 2;
tsMnodeIpList.ip[1] = inet_addr(tsSecondIp);
}
} else {
tsMnodeIpList.inUse = tsMnodeInfos.inUse;
tsMnodeIpList.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpList.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpList.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
}
}
tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp;
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
......@@ -67,11 +99,24 @@ int32_t dnodeInitMClient() {
return -1;
}
tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp;
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
dPrint("mnode rpc client is opened");
return 0;
}
void dnodeCleanupMClient() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
}
if (tsDnodeMClientRpc) {
rpcClose(tsDnodeMClientRpc);
tsDnodeMClientRpc = NULL;
......@@ -92,31 +137,18 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) {
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
SDMStatusRsp *pStatusRsp = pMsg->pCont;
if (pStatusRsp->ipList.numOfIps <= 0) {
dError("status msg is invalid, num of ips is %d", pStatusRsp->ipList.numOfIps);
SDMNodeInfos *mpeers = &pStatusRsp->mpeers;
if (mpeers->nodeNum <= 0) {
dError("status msg is invalid, num of ips is %d", mpeers->nodeNum);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
pStatusRsp->ipList.port = htons(pStatusRsp->ipList.port);
for (int32_t i = 0; i < pStatusRsp->ipList.numOfIps; ++i) {
pStatusRsp->ipList.ip[i] = htonl(pStatusRsp->ipList.ip[i]);
}
//dTrace("status msg is received, result:%s", tstrerror(pMsg->code));
if (memcmp(&(pStatusRsp->ipList), &tsDnodeMnodeIpList, sizeof(SRpcIpSet)) != 0) {
dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", pStatusRsp->ipList.numOfIps, pStatusRsp->ipList.inUse);
memcpy(&tsDnodeMnodeIpList, &pStatusRsp->ipList, sizeof(SRpcIpSet));
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; ++i) {
dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i]));
}
dnodeSaveMnodeIpList();
}
SDnodeState *pState = &pStatusRsp->dnodeState;
pState->numOfVnodes = htonl(pState->numOfVnodes);
pState->moduleStatus = htonl(pState->moduleStatus);
......@@ -124,75 +156,271 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
pState->dnodeId = htonl(pState->dnodeId);
dnodeProcessModuleStatus(pState->moduleStatus);
dnodeUpdateDnodeId(pState->dnodeId);
dnodeUpdateDnodeInfo(pState->dnodeId);
SRpcIpSet mgmtIpSet = {0};
mgmtIpSet.inUse = mpeers->inUse;
mgmtIpSet.numOfIps = mpeers->nodeNum;
mgmtIpSet.port = htons(mpeers->nodeInfos[0].nodePort);
for (int32_t i = 0; i < mpeers->nodeNum; i++) {
mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp);
}
if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0 || tsMnodeInfos.nodeNum == 0) {
memcpy(&tsMnodeIpList, &mgmtIpSet, sizeof(SRpcIpSet));
tsMnodeInfos.inUse = mpeers->inUse;
tsMnodeInfos.nodeNum = mpeers->nodeNum;
dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < mpeers->nodeNum; i++) {
tsMnodeInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId);
tsMnodeInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp);
tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort);
strcpy(tsMnodeInfos.nodeInfos[i].nodeName, mpeers->nodeInfos[i].nodeName);
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
}
dnodeSaveMnodeIpList();
mpeerUpdateSync();
}
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
if (tsDnodeMClientRpc) {
rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg);
rpcSendRequest(tsDnodeMClientRpc, &tsMnodeIpList, rpcMsg);
}
}
static bool dnodeReadMnodeIpList() {
char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/iplist", tsDnodeDir);
sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "r");
if (!fp) return false;
if (!fp) {
dTrace("failed to read mnode mgmtIpList.json, file not exist");
return false;
}
char option[32] = {0};
int32_t value = 0;
int32_t num = 0;
bool ret = false;
int maxLen = 2000;
char *content = calloc(1, maxLen + 1);
int len = fread(content, 1, maxLen, fp);
if (len <= 0) {
free(content);
fclose(fp);
dError("failed to read mnode mgmtIpList.json, content is null");
return false;
}
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "inUse") != 0) return false;
tsDnodeMnodeIpList.inUse = (int8_t)value;;
cJSON* root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read mnode mgmtIpList.json, invalid json format");
goto PARSE_OVER;
}
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "numOfIps") != 0) return false;
tsDnodeMnodeIpList.numOfIps = (int8_t)value;
cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
if (!inUse || inUse->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, inUse not found");
goto PARSE_OVER;
}
tsMnodeInfos.inUse = inUse->valueint;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "port") != 0) return false;
tsDnodeMnodeIpList.port = (uint16_t)value;
cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
if (!nodeNum || nodeNum->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodeNum not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeNum = nodeNum->valueint;
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strncmp(option, "ip", 2) != 0) return false;
tsDnodeMnodeIpList.ip[i] = (uint32_t)value;
cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) {
dError("failed to read mnode mgmtIpList.json, nodeInfos not found");
goto PARSE_OVER;
}
fclose(fp);
dPrint("read mnode iplist successed");
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i]));
int size = cJSON_GetArraySize(nodeInfos);
if (size != tsMnodeInfos.nodeNum) {
dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched");
goto PARSE_OVER;
}
return true;
for (int i = 0; i < size; ++i) {
cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
if (nodeInfo == NULL) continue;
cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
if (!nodeId || nodeId->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodeId not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeIp not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort");
if (!nodePort || nodePort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodePort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint;
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeName not found");
goto PARSE_OVER;
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN);
}
ret = true;
dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
}
PARSE_OVER:
free(content);
cJSON_Delete(root);
fclose(fp);
return ret;
}
static void dnodeSaveMnodeIpList() {
char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/iplist", tsDnodeDir);
sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "w");
if (!fp) return;
fprintf(fp, "inUse %d\n", tsDnodeMnodeIpList.inUse);
fprintf(fp, "numOfIps %d\n", tsDnodeMnodeIpList.numOfIps);
fprintf(fp, "port %u\n", tsDnodeMnodeIpList.port);
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
fprintf(fp, "ip%d %u\n", i, tsDnodeMnodeIpList.ip[i]);
int32_t len = 0;
int32_t maxLen = 2000;
char * content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMnodeInfos.inUse);
len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum);
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp));
len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName);
if (i < tsMnodeInfos.nodeNum -1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
len += snprintf(content + len, maxLen - len, " }]\n");
}
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
fclose(fp);
free(content);
dPrint("save mnode iplist successed");
}
uint32_t dnodeGetMnodeMasteIp() {
return tsDnodeMnodeIpList.ip[0];
return tsMnodeIpList.ip[tsMnodeIpList.inUse];
}
void* dnodeGetMpeerInfos() {
return &tsMnodeInfos;
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SDMStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
dnodeSendMsgToMnode(&rpcMsg);
}
static void dnodeReadDnodeInfo() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "r");
if (!fp) return;
char option[32] = {0};
int32_t value = 0;
int32_t num = 0;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return;
if (strcmp(option, "dnodeId") != 0) return;
tsDnodeId = value;;
fclose(fp);
dPrint("read dnodeId:%d successed", tsDnodeId);
}
static void dnodeSaveDnodeInfo() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "w");
if (!fp) return;
fprintf(fp, "dnodeId %d\n", tsDnodeId);
fclose(fp);
dPrint("save dnodeId successed");
}
void dnodeUpdateDnodeInfo(int32_t dnodeId) {
if (tsDnodeId == 0) {
dPrint("dnodeId is set to %d", dnodeId);
tsDnodeId = dnodeId;
dnodeSaveDnodeInfo();
}
}
int32_t dnodeGetDnodeId() {
return tsDnodeId;
}
\ No newline at end of file
src/dnode/src/dnodeMain.c
浏览文件 @ 6a504d36
......@@ -159,10 +159,10 @@ static int32_t dnodeInitSystem() {
dPrint("starting to initialize TDengine ...");
if (dnodeInitStorage() != 0) return -1;
if (dnodeInitModules() != 0) return -1;
if (dnodeInitRead() != 0) return -1;
if (dnodeInitWrite() != 0) return -1;
if (dnodeInitMClient() != 0) return -1;
if (dnodeInitModules() != 0) return -1;
if (dnodeInitMnode() != 0) return -1;
if (dnodeInitMgmt() != 0) return -1;
if (dnodeInitShell() != 0) return -1;
......@@ -177,7 +177,6 @@ static int32_t dnodeInitSystem() {
static void dnodeCleanUpSystem() {
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
tclearModuleStatus(TSDB_MOD_MGMT);
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
dnodeCleanupShell();
dnodeCleanupMnode();
......
src/dnode/src/dnodeMgmt.c
浏览文件 @ 6a504d36
......@@ -21,8 +21,6 @@
#include "tlog.h"
#include "trpc.h"
#include "tsdb.h"
#include "ttime.h"
#include "ttimer.h"
#include "twal.h"
#include "dnodeMClient.h"
#include "dnodeMgmt.h"
......@@ -38,52 +36,23 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void dnodeReadDnodeId();
static void *tsDnodeTmr = NULL;
static void *tsStatusTimer = NULL;
static uint32_t tsRebootTime;
static int32_t tsDnodeId = 0;
static char tsDnodeName[TSDB_DNODE_NAME_LEN];
int32_t dnodeInitMgmt() {
dnodeReadDnodeId();
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
tsRebootTime = taosGetTimestampSec();
tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
return -1;
}
int32_t code = dnodeOpenVnodes();
if (code != TSDB_CODE_SUCCESS) {
return -1;
}
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
return TSDB_CODE_SUCCESS;
}
void dnodeCleanupMgmt() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
}
dnodeCloseVnodes();
}
......@@ -213,89 +182,3 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
SMDCfgDnodeMsg *pCfg = (SMDCfgDnodeMsg *)pMsg->pCont;
return tsCfgDynamicOptions(pCfg->config);
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SDMStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
dnodeSendMsgToMnode(&rpcMsg);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
static void dnodeReadDnodeId() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "r");
if (!fp) return;
char option[32] = {0};
int32_t value = 0;
int32_t num = 0;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return;
if (strcmp(option, "dnodeId") != 0) return;
tsDnodeId = value;;
fclose(fp);
dPrint("read dnodeId:%d successed", tsDnodeId);
}
static void dnodeSaveDnodeId() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "w");
if (!fp) return;
fprintf(fp, "dnodeId %d\n", tsDnodeId);
fclose(fp);
dPrint("save dnodeId successed");
}
void dnodeUpdateDnodeId(int32_t dnodeId) {
if (tsDnodeId == 0) {
dPrint("dnodeId is set to %d", dnodeId);
tsDnodeId = dnodeId;
dnodeSaveDnodeId();
}
}
src/dnode/src/dnodeRead.c
浏览文件 @ 6a504d36
......@@ -32,27 +32,51 @@ typedef struct {
SRpcMsg rpcMsg;
} SReadMsg;
typedef struct {
pthread_t thread; // thread
int32_t workerId; // worker ID
} SReadWorker;
typedef struct {
int32_t max; // max number of workers
int32_t min; // min number of workers
int32_t num; // current number of workers
SReadWorker *readWorker;
} SReadWorkerPool;
static void *dnodeProcessReadQueue(void *param);
static void dnodeHandleIdleReadWorker();
static void dnodeHandleIdleReadWorker(SReadWorker *);
// module global variable
static SReadWorkerPool readPool;
static taos_qset readQset;
static int32_t threads; // number of query threads
static int32_t maxThreads;
static int32_t minThreads;
int32_t dnodeInitRead() {
readQset = taosOpenQset();
minThreads = 3;
maxThreads = tsNumOfCores * tsNumOfThreadsPerCore;
if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads;
readPool.min = 2;
readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
readPool.readWorker = (SReadWorker *) calloc(sizeof(SReadWorker), readPool.max);
if (readPool.readWorker == NULL) return -1;
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
pWorker->workerId = i;
}
dPrint("dnode read is opened");
return 0;
}
void dnodeCleanupRead() {
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
if (pWorker->thread)
pthread_join(pWorker->thread, NULL);
}
taosCloseQset(readQset);
dPrint("dnode read is closed");
}
......@@ -116,18 +140,25 @@ void *dnodeAllocateRqueue(void *pVnode) {
taosAddIntoQset(readQset, queue, pVnode);
// spawn a thread to process queue
if (threads < maxThreads) {
pthread_t thread;
if (readPool.num < readPool.max) {
do {
SReadWorker *pWorker = readPool.readWorker + readPool.num;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&thread, &thAttr, dnodeProcessReadQueue, readQset) != 0) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
dError("failed to create thread to process read queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
readPool.num++;
dTrace("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num);
} while (readPool.num < readPool.min);
}
dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue);
dTrace("pVnode:%p, read queue:%p is allocated", pVnode, queue);
return queue;
}
......@@ -167,14 +198,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
}
static void *dnodeProcessReadQueue(void *param) {
taos_qset qset = (taos_qset)param;
SReadWorker *pWorker = param;
SReadMsg *pReadMsg;
int type;
void *pVnode;
while (1) {
if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) {
dnodeHandleIdleReadWorker();
if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
dnodeHandleIdleReadWorker(pWorker);
continue;
}
......@@ -186,11 +217,12 @@ static void *dnodeProcessReadQueue(void *param) {
return NULL;
}
static void dnodeHandleIdleReadWorker() {
static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
int32_t num = taosGetQueueNumber(readQset);
if (num == 0 || (num <= minThreads && threads > minThreads)) {
threads--;
if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) {
readPool.num--;
dTrace("read worker:%d is released, total:%d", pWorker->workerId, readPool.num);
pthread_exit(NULL);
} else {
usleep(100);
......
src/dnode/src/dnodeWrite.c
浏览文件 @ 6a504d36
......@@ -28,6 +28,7 @@
#include "vnode.h"
typedef struct {
taos_qall qall;
taos_qset qset; // queue set
pthread_t thread; // thread
int32_t workerId; // worker ID
......@@ -40,7 +41,7 @@ typedef struct {
SRpcMsg rpcMsg;
} SWriteMsg;
typedef struct _wworker_pool {
typedef struct {
int32_t max; // max number of workers
int32_t nextId; // from 0 to max-1, cyclic
SWriteWorker *writeWorker;
......@@ -65,6 +66,14 @@ int32_t dnodeInitWrite() {
}
void dnodeCleanupWrite() {
for (int32_t i = 0; i < wWorkerPool.max; ++i) {
SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
}
}
free(wWorkerPool.writeWorker);
dPrint("dnode write is closed");
}
......@@ -113,6 +122,7 @@ void *dnodeAllocateWqueue(void *pVnode) {
if (pWorker->qset == NULL) return NULL;
taosAddIntoQset(pWorker->qset, queue, pVnode);
pWorker->qall = taosAllocateQall();
wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
pthread_attr_t thAttr;
......@@ -122,13 +132,17 @@ void *dnodeAllocateWqueue(void *pVnode) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
dError("failed to create thread to process read queue, reason:%s", strerror(errno));
taosCloseQset(pWorker->qset);
} else {
dTrace("write worker:%d is launched", pWorker->workerId);
}
pthread_attr_destroy(&thAttr);
} else {
taosAddIntoQset(pWorker->qset, queue, pVnode);
wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
}
dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue);
dTrace("pVnode:%p, write queue:%p is allocated", pVnode, queue);
return queue;
}
......@@ -160,17 +174,14 @@ void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code) {
static void *dnodeProcessWriteQueue(void *param) {
SWriteWorker *pWorker = (SWriteWorker *)param;
taos_qall qall;
SWriteMsg *pWrite;
SWalHead *pHead;
int32_t numOfMsgs;
int type;
void *pVnode, *item;
qall = taosAllocateQall();
while (1) {
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, qall, &pVnode);
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
if (numOfMsgs <=0) {
dnodeHandleIdleWorker(pWorker); // thread exit if no queues anymore
continue;
......@@ -178,7 +189,7 @@ static void *dnodeProcessWriteQueue(void *param) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
pWrite = NULL;
taosGetQitem(qall, &type, &item);
taosGetQitem(pWorker->qall, &type, &item);
if (type == TAOS_QTYPE_RPC) {
pWrite = (SWriteMsg *)item;
pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
......@@ -196,9 +207,9 @@ static void *dnodeProcessWriteQueue(void *param) {
walFsync(vnodeGetWal(pVnode));
// browse all items, and process them one by one
taosResetQitems(qall);
taosResetQitems(pWorker->qall);
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, &type, &item);
taosGetQitem(pWorker->qall, &type, &item);
if (type == TAOS_QTYPE_RPC) {
pWrite = (SWriteMsg *)item;
dnodeSendRpcWriteRsp(pVnode, item, pWrite->rpcMsg.code);
......@@ -209,8 +220,6 @@ static void *dnodeProcessWriteQueue(void *param) {
}
}
taosFreeQall(qall);
return NULL;
}
......@@ -221,8 +230,10 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
usleep(1000);
sched_yield();
} else {
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);
pWorker->qset = NULL;
dTrace("write worker:%d is released", pWorker->workerId);
pthread_exit(NULL);
}
}
src/inc/mnode.h
浏览文件 @ 6a504d36
......@@ -41,28 +41,19 @@ struct _vg_obj;
struct _db_obj;
struct _acct_obj;
struct _user_obj;
struct _mnode_obj;
typedef struct {
typedef struct _mnode_obj {
int32_t mnodeId;
uint32_t privateIp;
uint32_t publicIp;
int64_t createdTime;
int64_t lostTime;
uint64_t dbVersion;
uint32_t rack;
uint16_t idc;
uint16_t slot;
int8_t role;
int8_t status;
int8_t numOfMnodes;
int32_t numOfDnodes;
char mnodeName[TSDB_DNODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t reserved[14];
int8_t updateEnd[1];
int32_t refCount;
int syncFd;
void *hbTimer;
void *pSync;
uint32_t privateIp;
uint32_t publicIp;
uint16_t port;
int8_t role;
char mnodeName[TSDB_NODE_NAME_LEN + 1];
} SMnodeObj;
typedef struct _dnode_obj {
......@@ -81,7 +72,7 @@ typedef struct _dnode_obj {
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function
int32_t customScore; // config by user
char dnodeName[TSDB_DNODE_NAME_LEN + 1];
char dnodeName[TSDB_NODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t updateEnd[1];
int32_t refCount;
......
src/mnode/inc/mgmtMnode.h → src/inc/mpeer.h
浏览文件 @ 6a504d36
......@@ -13,35 +13,44 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MGMT_MNODE_H
#define TDENGINE_MGMT_MNODE_H
#ifndef TDENGINE_MPEER_H
#define TDENGINE_MPEER_H
#ifdef __cplusplus
extern "C" {
#endif
enum _TSDB_MN_STATUS {
TSDB_MN_STATUS_OFFLINE,
TSDB_MN_STATUS_UNSYNCED,
TSDB_MN_STATUS_SYNCING,
TSDB_MN_STATUS_SERVING
};
struct _mnode_obj;
enum _TSDB_MN_ROLE {
TSDB_MN_ROLE_UNDECIDED,
TSDB_MN_ROLE_SLAVE,
TSDB_MN_ROLE_MASTER
enum _TAOS_MN_STATUS {
TAOS_MN_STATUS_OFFLINE,
TAOS_MN_STATUS_DROPPING,
TAOS_MN_STATUS_READY
};
int32_t mgmtInitMnodes();
void mgmtCleanupMnodes();
// general implementation
int32_t mpeerInit();
void mpeerCleanup();
// special implementation
int32_t mpeerInitMnodes();
void mpeerCleanupMnodes();
int32_t mpeerAddMnode(int32_t dnodeId);
int32_t mpeerRemoveMnode(int32_t dnodeId);
void * mpeerGetMnode(int32_t mnodeId);
int32_t mpeerGetMnodesNum();
void * mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode);
void mpeerReleaseMnode(struct _mnode_obj *pMnode);
bool mpeerIsMaster();
bool mgmtInServerStatus();
bool mgmtIsMaster();
void mpeerGetPrivateIpList(SRpcIpSet *ipSet);
void mpeerGetPublicIpList(SRpcIpSet *ipSet);
void mpeerGetMpeerInfos(void *mpeers);
bool mgmtCheckRedirect(void *handle);
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet);
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet);
int32_t mpeerForwardReqToPeer(void *pHead);
void mpeerUpdateSync();
#ifdef __cplusplus
}
......
src/inc/taosdef.h
浏览文件 @ 6a504d36
......@@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_COLUMNS 256
#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns
#define TSDB_DNODE_NAME_LEN 64
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 192
#define TSDB_DB_NAME_LEN 32
#define TSDB_COL_NAME_LEN 64
......@@ -233,7 +233,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
/*
* this is defined in CMakeList.txt
*/
//#define TSDB_REPLICA_MAX_NUM 3
#define TSDB_REPLICA_MAX_NUM 3
#define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta
......
src/inc/taosmsg.h
浏览文件 @ 6a504d36
......@@ -581,7 +581,7 @@ typedef struct {
typedef struct {
uint32_t version;
int32_t dnodeId;
char dnodeName[TSDB_DNODE_NAME_LEN];
char dnodeName[TSDB_NODE_NAME_LEN + 1];
uint32_t privateIp;
uint32_t publicIp;
uint32_t lastReboot; // time stamp for last reboot
......@@ -595,7 +595,20 @@ typedef struct {
} SDMStatusMsg;
typedef struct {
SRpcIpSet ipList;
int32_t nodeId;
uint32_t nodeIp;
uint16_t nodePort;
char nodeName[TSDB_NODE_NAME_LEN + 1];
} SDMNodeInfo;
typedef struct {
int8_t inUse;
int8_t nodeNum;
SDMNodeInfo nodeInfos[TSDB_MAX_MPEERS];
} SDMNodeInfos;
typedef struct {
SDMNodeInfos mpeers;
SDnodeState dnodeState;
SVnodeAccess vnodeAccess[];
} SDMStatusRsp;
......
src/inc/tbalance.h
浏览文件 @ 6a504d36
......@@ -31,6 +31,7 @@ struct _dnode_obj;
int32_t balanceInit();
void balanceCleanUp();
void balanceNotify();
void balanceReset();
int32_t balanceAllocVnodes(struct _vg_obj *pVgroup);
int32_t balanceDropDnode(struct _dnode_obj *pDnode);
......
src/inc/tcluster.h
浏览文件 @ 6a504d36
......@@ -37,6 +37,7 @@ int32_t clusterInit();
void clusterCleanUp();
char* clusterGetDnodeStatusStr(int32_t dnodeStatus);
bool clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType);
void clusterMonitorDnodeModule();
int32_t clusterInitDnodes();
void clusterCleanupDnodes();
......
src/inc/twal.h
浏览文件 @ 6a504d36
......@@ -36,6 +36,7 @@ typedef struct {
typedef struct {
int8_t commitLog; // commitLog
int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed
} SWalCfg;
typedef void* twalh; // WAL HANDLE
......
src/mnode/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -13,5 +13,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})
TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
#TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
ENDIF ()
\ No newline at end of file
src/mnode/inc/mgmtSdb.h
浏览文件 @ 6a504d36
......@@ -21,21 +21,32 @@ extern "C" {
#endif
typedef enum {
SDB_KEY_TYPE_STRING,
SDB_KEY_TYPE_AUTO
SDB_TABLE_DNODE = 0,
SDB_TABLE_MNODE = 1,
SDB_TABLE_ACCOUNT = 2,
SDB_TABLE_USER = 3,
SDB_TABLE_DB = 4,
SDB_TABLE_VGROUP = 5,
SDB_TABLE_STABLE = 6,
SDB_TABLE_CTABLE = 7,
SDB_TABLE_MAX = 8
} ESdbTable;
typedef enum {
SDB_KEY_STRING,
SDB_KEY_INT,
SDB_KEY_AUTO
} ESdbKeyType;
typedef enum {
SDB_OPER_TYPE_GLOBAL,
SDB_OPER_TYPE_LOCAL
SDB_OPER_GLOBAL,
SDB_OPER_LOCAL
} ESdbOperType;
typedef struct {
ESdbOperType type;
void * table;
void * pObj;
int64_t version;
int32_t maxRowSize;
int32_t rowSize;
void * rowData;
} SSdbOperDesc;
......@@ -45,6 +56,7 @@ typedef struct {
int32_t hashSessions;
int32_t maxRowSize;
int32_t refCountPos;
ESdbTable tableId;
ESdbKeyType keyType;
int32_t (*insertFp)(SSdbOperDesc *pOper);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
......@@ -52,10 +64,22 @@ typedef struct {
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pDesc);
int32_t (*destroyFp)(SSdbOperDesc *pDesc);
int32_t (*restoredFp)();
} SSdbTableDesc;
typedef struct {
int64_t version;
void * wal;
pthread_mutex_t mutex;
} SSdbObject;
int32_t sdbInit();
void sdbCleanUp();
SSdbObject *sdbGetObj();
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
int sdbProcessWrite(void *param, void *data, int type);
int32_t sdbInsertRow(SSdbOperDesc *pOper);
int32_t sdbDeleteRow(SSdbOperDesc *pOper);
......@@ -66,7 +90,7 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
void sdbIncRef(void *thandle, void *pRow);
void sdbDecRef(void *thandle, void *pRow);
int64_t sdbGetNumOfRows(void *handle);
int64_t sdbGetId(void *handle);
int32_t sdbGetId(void *handle);
uint64_t sdbGetVersion();
#ifdef __cplusplus
......
src/mnode/src/mgmtDb.c
浏览文件 @ 6a504d36
......@@ -24,8 +24,8 @@
#include "mgmtDb.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mpeer.h"
#include "mgmtShell.h"
#include "mgmtMnode.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtTable.h"
......@@ -88,14 +88,9 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
if (pOper->maxRowSize < tsDbUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
......@@ -107,22 +102,28 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionRestored() {
return 0;
}
int32_t mgmtInitDbs() {
SDbObj tObj;
tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DB,
.tableName = "dbs",
.hashSessions = TSDB_MAX_DBS,
.maxRowSize = tsDbUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtDbActionInsert,
.deleteFp = mgmtDbActionDelete,
.updateFp = mgmtDbActionUpdate,
.encodeFp = mgmtDbActionEncode,
.decodeFp = mgmtDbActionDecode,
.destroyFp = mgmtDbActionDestroy,
.restoredFp = mgmtDbActionRestored
};
tsDbSdb = sdbOpenTable(&tableDesc);
......@@ -310,7 +311,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->cfg = *pCreate;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = sizeof(SDbObj)
......@@ -663,7 +664,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
pDb->status = true;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
......@@ -678,8 +679,6 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
}
static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
SCMCreateDbMsg *pCreate = pMsg->pCont;
pCreate->maxSessions = htonl(pCreate->maxSessions);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
......@@ -750,7 +749,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
......@@ -808,7 +807,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) {
mPrint("db:%s, drop db from sdb", pDb->name);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb
};
......
src/mnode/src/mgmtDnode.c
浏览文件 @ 6a504d36
......@@ -19,8 +19,8 @@
#include "tbalance.h"
#include "tcluster.h"
#include "mnode.h"
#include "mpeer.h"
#include "mgmtDClient.h"
#include "mgmtMnode.h"
#include "mgmtShell.h"
#include "mgmtDServer.h"
#include "mgmtUser.h"
......@@ -77,6 +77,7 @@ void * clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : N
void * clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; }
void clusterReleaseDnode(struct _dnode_obj *pDnode) {}
void clusterUpdateDnode(struct _dnode_obj *pDnode) {}
void clusterMonitorDnodeModule() {}
#endif
......@@ -141,8 +142,6 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
}
void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMStatusMsg *pStatus = rpcMsg->pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->privateIp = htonl(pStatus->privateIp);
......@@ -210,6 +209,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
balanceNotify();
clusterMonitorDnodeModule();
}
clusterReleaseDnode(pDnode);
......@@ -221,7 +221,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
return;
}
mgmtGetMnodePrivateIpList(&pRsp->ipList);
mpeerGetMpeerInfos(&pRsp->mpeers);
pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId);
pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus);
......
src/mnode/src/mgmtMain.c
浏览文件 @ 6a504d36
......@@ -22,11 +22,11 @@
#include "taccount.h"
#include "tbalance.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mpeer.h"
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDServer.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtVgroup.h"
#include "mgmtUser.h"
......@@ -105,20 +105,25 @@ int32_t mgmtStartSystem() {
}
if (mgmtInitTables() < 0) {
mError("failed to init meters");
mError("failed to init tables");
return -1;
}
if (mgmtInitDClient() < 0) {
if (mpeerInit() < 0) {
mError("failed to init mpeers");
return -1;
}
if (mgmtInitDServer() < 0) {
if (sdbInit() < 0) {
mError("failed to init sdb");
return -1;
}
if (mgmtInitMnodes() < 0) {
mError("failed to init mnodes");
if (mgmtInitDClient() < 0) {
return -1;
}
if (mgmtInitDServer() < 0) {
return -1;
}
......@@ -135,7 +140,7 @@ int32_t mgmtStartSystem() {
void mgmtStopSystem() {
if (mgmtIsMaster()) {
if (mpeerIsMaster()) {
mTrace("it is a master mgmt node, it could not be stopped");
return;
}
......@@ -147,7 +152,7 @@ void mgmtStopSystem() {
void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes();
mpeerCleanup();
balanceCleanUp();
mgmtCleanUpShell();
mgmtCleanupDClient();
......@@ -158,6 +163,7 @@ void mgmtCleanUpSystem() {
clusterCleanUp();
mgmtCleanUpUsers();
acctCleanUp();
sdbCleanUp();
taosTmrCleanUp(tsMgmtTmr);
mPrint("mgmt is cleaned up");
}
......
src/mnode/src/mgmtMnode.c
浏览文件 @ 6a504d36
......@@ -17,43 +17,40 @@
#include "os.h"
#include "taoserror.h"
#include "trpc.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "tsync.h"
#include "mpeer.h"
#include "mgmtShell.h"
#include "mgmtUser.h"
#ifndef _MPEER
static SMnodeObj tsMnodeObj = {0};
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
int32_t mgmtInitMnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
#ifndef _MPEER
static SMnodeObj tsMnodeObj = {0};
int32_t mpeerInitMnodes() {
tsMnodeObj.mnodeId = 1;
tsMnodeObj.privateIp = inet_addr(tsPrivateIp);
tsMnodeObj.publicIp = inet_addr(tsPublicIp);
tsMnodeObj.createdTime = taosGetTimestampMs();
tsMnodeObj.role = TSDB_MN_ROLE_MASTER;
tsMnodeObj.status = TSDB_MN_STATUS_SERVING;
tsMnodeObj.numOfMnodes = 1;
sprintf(tsMnodeObj.mnodeName, "%d", tsMnodeObj.mnodeId);
tsMnodeObj.role = TAOS_SYNC_ROLE_MASTER;
tsMnodeObj.port = tsMnodeDnodePort;
sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId);
return TSDB_CODE_SUCCESS;
}
void mgmtCleanupMnodes() {}
bool mgmtInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; }
bool mgmtIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; }
bool mgmtCheckRedirect(void *thandle) { return false; }
static int32_t mgmtGetMnodesNum() {
return 1;
}
void mpeerCleanupMnodes() {}
int32_t mpeerAddMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
int32_t mpeerRemoveMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
void * mpeerGetMnode(int32_t mnodeId) { return &tsMnodeObj; }
int32_t mpeerGetMnodesNum() { return 1; }
void mpeerReleaseMnode(struct _mnode_obj *pMnode) {}
bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; }
void mpeerUpdateSync() {}
static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) {
if (*pMnode == NULL) {
*pMnode = &tsMnodeObj;
} else {
......@@ -63,22 +60,58 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
return *pMnode;
}
char *taosGetMnodeStatusStr(int32_t mnodeStatus) {
switch (mnodeStatus) {
case TSDB_MN_STATUS_OFFLINE: return "offline";
case TSDB_MN_STATUS_UNSYNCED: return "unsynced";
case TSDB_MN_STATUS_SYNCING: return "syncing";
case TSDB_MN_STATUS_SERVING: return "serving";
default: return "undefined";
}
void mpeerGetPrivateIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->numOfIps = 1;
ipSet->port = htons(tsMnodeObj.port);
ipSet->ip[0] = htonl(tsMnodeObj.privateIp);
}
char *taosGetMnodeRoleStr(int32_t mnodeRole) {
switch (mnodeRole) {
case TSDB_MN_ROLE_UNDECIDED: return "undicided";
case TSDB_MN_ROLE_SLAVE: return "slave";
case TSDB_MN_ROLE_MASTER: return "master";
default: return "undefined";
void mpeerGetPublicIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->numOfIps = 1;
ipSet->port = htons(tsMnodeObj.port);
ipSet->ip[0] = htonl(tsMnodeObj.publicIp);
}
void mpeerGetMpeerInfos(void *param) {
SDMNodeInfos *mpeers = param;
mpeers->inUse = 0;
mpeers->nodeNum = 1;
mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId);
mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp);
mpeers->nodeInfos[0].nodePort = htons(tsMnodeObj.port);
strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName);
}
int32_t mpeerForwardReqToPeer(void *pHead) {
return TSDB_CODE_SUCCESS;
}
#endif
int32_t mpeerInit() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
return mpeerInitMnodes();
}
void mpeerCleanup() {
mpeerCleanupMnodes();
}
static char *mpeerGetMnodeRoleStr(int32_t role) {
switch (role) {
case TAOS_SYNC_ROLE_OFFLINE:
return "offline";
case TAOS_SYNC_ROLE_UNSYNCED:
return "unsynced";
case TAOS_SYNC_ROLE_SLAVE:
return "slave";
case TAOS_SYNC_ROLE_MASTER:
return "master";
default:
return "undefined";
}
}
......@@ -115,12 +148,6 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "role");
......@@ -135,7 +162,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mgmtGetMnodesNum();
pShow->numOfRows = mpeerGetMnodesNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
mgmtReleaseUser(pUser);
......@@ -151,7 +178,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextMnode(pShow->pNode, (SMnodeObj **)&pMnode);
pShow->pNode = mpeerGetNextMnode(pShow->pNode, &pMnode);
if (pMnode == NULL) break;
cols = 0;
......@@ -175,33 +202,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetMnodeStatusStr(pMnode->status));
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetMnodeRoleStr(pMnode->role));
strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role));
cols++;
numOfRows++;
mpeerReleaseMnode(pMnode);
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
\ No newline at end of file
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;
ipSet->ip[0] = htonl(tsMnodeObj.privateIp);
}
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;
ipSet->ip[0] = htonl(tsMnodeObj.publicIp);
}
#endif
\ No newline at end of file
src/mnode/src/mgmtProfile.c
浏览文件 @ 6a504d36
......@@ -19,7 +19,7 @@
#include "taccount.h"
#include "tcluster.h"
#include "mgmtDb.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtShell.h"
#include "mgmtTable.h"
......@@ -681,7 +681,6 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
......@@ -705,7 +704,6 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
......@@ -729,7 +727,6 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
......
src/mnode/src/mgmtSdb.c
浏览文件 @ 6a504d36
......@@ -15,44 +15,25 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tchecksum.h"
#include "tglobalcfg.h"
#include "tlog.h"
#include "trpc.h"
#include "tutil.h"
#include "tqueue.h"
#include "twal.h"
#include "hashint.h"
#include "hashstr.h"
#include "mpeer.h"
#include "mgmtSdb.h"
#define abs(x) (((x) < 0) ? -(x) : (x))
#define SDB_MAX_PEERS 4
#define SDB_DELIMITER 0xFFF00F00
#define SDB_ENDCOMMIT 0xAFFFAAAF
typedef struct {
uint64_t swVersion;
int16_t sdbFileVersion;
char reserved[2];
TSCKSUM checkSum;
} SSdbHeader;
typedef struct _SSdbTable {
SSdbHeader header;
char tableName[TSDB_DB_NAME_LEN];
char fileName[TSDB_FILENAME_LEN];
char tableName[TSDB_DB_NAME_LEN + 1];
ESdbTable tableId;
ESdbKeyType keyType;
int32_t tableId;
int32_t hashSessions;
int32_t maxRowSize;
int32_t refCountPos;
int32_t autoIndex;
int32_t fd;
int64_t numOfRows;
int64_t version;
int64_t fileSize;
void * iHandle;
int32_t (*insertFp)(SSdbOperDesc *pDesc);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
......@@ -60,428 +41,343 @@ typedef struct _SSdbTable {
int32_t (*decodeFp)(SSdbOperDesc *pOper);
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*destroyFp)(SSdbOperDesc *pOper);
int32_t (*restoredFp)();
pthread_mutex_t mutex;
} SSdbTable;
typedef struct {
int64_t version;
int64_t offset;
int32_t rowSize;
void * row;
} SRowMeta;
typedef struct {
int32_t delimiter;
int32_t rowSize;
int64_t version;
char data[];
} SRowHead;
typedef enum {
SDB_FORWARD_TYPE_INSERT,
SDB_FORWARD_TYPE_DELETE,
SDB_FORWARD_TYPE_UPDATE
} ESdbForwardType;
typedef struct {
ESdbForwardType type;
int32_t tableId;
int64_t version;
int32_t rowSize;
void * rowData;
} SForwardMsg;
extern char version[];
const int16_t sdbFileVersion = 2;
int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL;
static SSdbTable *sdbTableList[10] = {0};
static int32_t sdbNumOfTables = 0;
static uint64_t sdbVersion = 0;
static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash};
static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash};
static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash};
static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData};
static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash};
static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData};
uint64_t sdbGetVersion() { return sdbVersion; }
int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; }
SDB_ACTION_INSERT,
SDB_ACTION_DELETE,
SDB_ACTION_UPDATE
} ESdbActionType;
static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0};
static int32_t tsSdbNumOfTables = 0;
static SSdbObject * tsSdbObj;
static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash};
static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash};
static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash};
static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData};
static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash};
static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData};
uint64_t sdbGetVersion() { return tsSdbObj->version; }
int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; }
int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; }
static char *sdbGetActionStr(int32_t action) {
switch (action) {
case SDB_ACTION_INSERT:
return "insert";
case SDB_ACTION_DELETE:
return "delete";
case SDB_ACTION_UPDATE:
return "update";
}
return "invalid";
}
static char *sdbGetkeyStr(SSdbTable *pTable, void *row) {
static char str[16];
switch (pTable->keyType) {
case SDB_KEY_TYPE_STRING:
case SDB_KEY_STRING:
return (char *)row;
case SDB_KEY_TYPE_AUTO:
case SDB_KEY_INT:
case SDB_KEY_AUTO:
sprintf(str, "%d", *(int32_t *)row);
return str;
default:
return "unknown";
return "invalid";
}
}
static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) {
if (mpeerForwardRequestFp) {
return mpeerForwardRequestFp(forwardMsg);
} else {
return 0;
}
static void *sdbGetTableFromId(int32_t tableId) {
return tsSdbTableList[tableId];
}
static void sdbFinishCommit(SSdbTable *pTable) {
uint32_t sdbEcommit = SDB_ENDCOMMIT;
off_t offset = lseek(pTable->fd, 0, SEEK_END);
assert(offset == pTable->fileSize);
twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit));
pTable->fileSize += sizeof(sdbEcommit);
}
int32_t sdbInit() {
tsSdbObj = calloc(1, sizeof(SSdbObject));
pthread_mutex_init(&tsSdbObj->mutex, NULL);
static int32_t sdbOpenSdbFile(SSdbTable *pTable) {
struct stat fstat, ofstat;
uint64_t size;
char * dirc = NULL;
char * basec = NULL;
union {
char cversion[64];
uint64_t iversion;
} swVersion;
memcpy(swVersion.cversion, version, sizeof(uint64_t));
// check sdb.db and .sdb.db status
char fn[TSDB_FILENAME_LEN] = "\0";
dirc = strdup(pTable->fileName);
basec = strdup(pTable->fileName);
sprintf(fn, "%s/.%s", dirname(dirc), basename(basec));
tfree(dirc);
tfree(basec);
if (stat(fn, &ofstat) == 0) { // .sdb.db file exists
if (stat(pTable->fileName, &fstat) == 0) {
remove(fn);
} else {
remove(pTable->fileName);
rename(fn, pTable->fileName);
}
}
pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pTable->fd < 0) {
sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName);
SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1};
tsSdbObj->wal = walOpen(tsMnodeDir, &walCfg);
if (tsSdbObj->wal == NULL) {
sdbError("failed to open sdb in %s", tsMnodeDir);
return -1;
}
pTable->fileSize = 0;
stat(pTable->fileName, &fstat);
size = sizeof(pTable->header);
sdbTrace("open sdb file for read");
walRestore(tsSdbObj->wal, tsSdbObj, sdbProcessWrite);
if (fstat.st_size == 0) {
pTable->header.swVersion = swVersion.iversion;
pTable->header.sdbFileVersion = sdbFileVersion;
if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) {
sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
int32_t totalRows = 0;
int32_t numOfTables = 0;
for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) {
SSdbTable *pTable = sdbGetTableFromId(tableId);
if (pTable == NULL) continue;
if (pTable->restoredFp) {
(*pTable->restoredFp)();
}
twrite(pTable->fd, &(pTable->header), size);
pTable->fileSize += size;
sdbFinishCommit(pTable);
} else {
uint32_t sdbEcommit = 0;
off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END);
while (offset > 0) {
read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit));
if (sdbEcommit == SDB_ENDCOMMIT) {
ftruncate(pTable->fd, offset + sizeof(sdbEcommit));
break;
}
offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR);
}
lseek(pTable->fd, 0, SEEK_SET);
ssize_t tsize = read(pTable->fd, &(pTable->header), size);
if (tsize < size) {
sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
totalRows += pTable->numOfRows;
numOfTables++;
sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows);
}
if (pTable->header.swVersion != swVersion.iversion) {
sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName);
}
sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables);
if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) {
sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
}
mpeerUpdateSync();
pTable->fileSize += size;
// skip end commit symbol
lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR);
pTable->fileSize += sizeof(sdbEcommit);
}
return TSDB_CODE_SUCCESS;
}
pTable->numOfRows = 0;
void sdbCleanUp() {
if (tsSdbObj) {
pthread_mutex_destroy(&tsSdbObj->mutex);
walClose(tsSdbObj->wal);
free(tsSdbObj);
tsSdbObj = NULL;
}
}
return pTable->fd;
SSdbObject *sdbGetObj() {
return tsSdbObj;
}
static int32_t sdbInitTableByFile(SSdbTable *pTable) {
sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName);
if (sdbOpenSdbFile(pTable) < 0) {
sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName);
return -1;
void sdbIncRef(void *handle, void *pRow) {
if (pRow) {
SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
atomic_add_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
*pRefCount);
}
}
}
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)malloc(total_size);
if (rowHead == NULL) {
sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName);
return -1;
void sdbDecRef(void *handle, void *pRow) {
if (pRow) {
SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
int32_t refCount = atomic_sub_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
*pRefCount);
}
int8_t *updateEnd = pRow + pTable->refCountPos - 1;
if (refCount <= 0 && *updateEnd) {
sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName,
sdbGetkeyStr(pTable, pRow), *pRefCount);
SSdbOperDesc oper = {.pObj = pRow};
(*pTable->destroyFp)(&oper);
}
}
}
int32_t numOfChanged = 0;
int32_t maxAutoIndex = 0;
while (1) {
memset(rowHead, 0, total_size);
static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead));
if (bytes < 0) {
sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName);
tfree(rowHead);
return -1;
}
if (handle == NULL) return NULL;
if (bytes == 0) break;
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) {
pTable->fileSize++;
lseek(pTable->fd, -(bytes - 1), SEEK_CUR);
continue;
}
return pMeta;
}
if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) {
sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName,
pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize);
pTable->fileSize += sizeof(SRowHead);
continue;
}
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM));
if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) {
// TODO: Here may cause pTable->fileSize not end of the file
sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version,
rowHead->rowSize);
break;
}
if (handle == NULL) return NULL;
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) {
sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version);
pTable->fileSize += real_size;
continue;
}
pthread_mutex_lock(&pTable->mutex);
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
if (pMeta) sdbIncRef(pTable, pMeta->row);
pthread_mutex_unlock(&pTable->mutex);
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data);
if (pMeta == NULL) {
return NULL;
}
pTable->version = MAX(pTable->version, abs(rowHead->version));
return pMeta->row;
}
void *pMetaRow = sdbGetRow(pTable, rowHead->data);
if (pMetaRow == NULL) {
if (rowHead->version < 0) {
sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version,
sdbGetkeyStr(pTable, rowHead->data));
} else {
static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
SRowMeta rowMeta;
rowMeta.version = rowHead->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = rowHead->rowSize;
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.rowSize = rowHead->rowSize
};
int32_t code = (*pTable->decodeFp)(&oper);
if (code == TSDB_CODE_SUCCESS) {
rowMeta.row = oper.pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
pthread_mutex_lock(&pTable->mutex);
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta);
sdbIncRef(pTable, pOper->pObj);
pTable->numOfRows++;
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
}
}
} else {
if (rowHead->version < 0) {
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data);
pTable->numOfRows--;
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
SRowMeta rowMeta;
rowMeta.version = rowHead->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = rowHead->rowSize;
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.rowSize = rowHead->rowSize,
.pObj = pMetaRow
};
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data);
int32_t code = (*pTable->decodeFp)(&oper);
if (code == TSDB_CODE_SUCCESS) {
rowMeta.row = oper.pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
}
}
numOfChanged++;
}
pTable->fileSize += real_size;
pTable->fileSize += 4;
lseek(pTable->fd, 4, SEEK_CUR);
if (pTable->keyType == SDB_KEY_AUTO) {
pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj));
}
void *pNode = NULL;
while (1) {
SRowMeta * pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
pthread_mutex_unlock(&pTable->mutex);
SSdbOperDesc oper = {
.pObj = pMeta->row,
.table = pTable,
.version = pMeta->version,
};
sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
sdbIncRef(pTable, oper.pObj);
int32_t code = (*pTable->insertFp)(&oper);
if (code != TSDB_CODE_SUCCESS) {
sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data));
}
}
(*pTable->insertFp)(pOper);
return TSDB_CODE_SUCCESS;
}
sdbVersion += pTable->version;
static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
pthread_mutex_lock(&pTable->mutex);
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
pTable->autoIndex = maxAutoIndex;
}
sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
tfree(rowHead);
return 0;
(*pTable->deleteFp)(pOper);
int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
*updateEnd = 1;
sdbDecRef(pTable, pOper->pObj);
return TSDB_CODE_SUCCESS;
}
void *sdbOpenTable(SSdbTableDesc *pDesc) {
SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
if (pTable == NULL) return NULL;
static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
pTable->keyType = pDesc->keyType;
pTable->hashSessions = pDesc->hashSessions;
pTable->maxRowSize = pDesc->maxRowSize;
pTable->refCountPos = pDesc->refCountPos;
pTable->insertFp = pDesc->insertFp;
pTable->deleteFp = pDesc->deleteFp;
pTable->updateFp = pDesc->updateFp;
pTable->encodeFp = pDesc->encodeFp;
pTable->decodeFp = pDesc->decodeFp;
pTable->destroyFp = pDesc->destroyFp;
strcpy(pTable->tableName, pDesc->tableName);
sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName);
(*pTable->updateFp)(pOper);
return TSDB_CODE_SUCCESS;
}
if (sdbInitIndexFp[pTable->keyType] != NULL) {
pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta));
}
static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
int32_t code = 0;
pthread_mutex_init(&pTable->mutex, NULL);
pthread_mutex_lock(&tsSdbObj->mutex);
tsSdbObj->version++;
pHead->version = tsSdbObj->version;
if (sdbInitTableByFile(pTable) < 0) return NULL;
code = mpeerForwardReqToPeer(pHead);
if (code != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&tsSdbObj->mutex);
sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
return code;
}
pTable->tableId = sdbNumOfTables++;
sdbTableList[pTable->tableId] = pTable;
code = walWrite(tsSdbObj->wal, pHead);
pthread_mutex_unlock(&tsSdbObj->mutex);
sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d",
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables);
if (code < 0) {
sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
} else {
sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action),
sdbGetkeyStr(pTable, pHead->cont), pHead->version);
}
return pTable;
walFsync(tsSdbObj->wal);
taosFreeQitem(pHead);
return code;
}
static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
pthread_mutex_lock(&tsSdbObj->mutex);
if (pHead->version <= tsSdbObj->version) {
pthread_mutex_unlock(&tsSdbObj->mutex);
return TSDB_CODE_SUCCESS;
} else if (pHead->version != tsSdbObj->version + 1) {
pthread_mutex_unlock(&tsSdbObj->mutex);
sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64,
pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
tsSdbObj->version);
return TSDB_CODE_OTHERS;
}
if (handle == NULL) return NULL;
tsSdbObj->version = pHead->version;
sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
int32_t code = -1;
if (action == SDB_ACTION_INSERT) {
SSdbOperDesc oper = {
.rowSize = pHead->len,
.rowData = pHead->cont,
.table = pTable,
};
code = (*pTable->decodeFp)(&oper);
if (code < 0) {
sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
return pMeta;
}
code = sdbInsertLocal(pTable, &oper);
} else if (action == SDB_ACTION_DELETE) {
SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
void sdbIncRef(void *handle, void *pRow) {
if (pRow) {
SSdbTable *pTable = handle;
int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos);
atomic_add_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount);
}
}
}
SSdbOperDesc oper = {
.table = pTable,
.pObj = rowMeta->row,
};
void sdbDecRef(void *handle, void *pRow) {
if (pRow) {
SSdbTable *pTable = handle;
int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos);
int32_t refCount = atomic_sub_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount);
}
int8_t* updateEnd = pRow + pTable->refCountPos - 1;
if (refCount <= 0 && *updateEnd) {
sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount);
SSdbOperDesc oper = {.pObj = pRow};
(*pTable->destroyFp)(&oper);
code = sdbDeleteLocal(pTable, &oper);
} else if (action == SDB_ACTION_UPDATE) {
SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
SSdbOperDesc oper1 = {
.table = pTable,
.pObj = rowMeta->row,
};
sdbDeleteLocal(pTable, &oper1);
SSdbOperDesc oper2 = {
.rowSize = pHead->len,
.rowData = pHead->cont,
.table = pTable,
};
code = (*pTable->decodeFp)(&oper2);
if (code < 0) {
sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
code = sdbInsertLocal(pTable, &oper2);
}
}
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
if (handle == NULL) return NULL;
int sdbProcessWrite(void *param, void *data, int type) {
SWalHead *pHead = data;
int32_t tableId = pHead->msgType / 10;
int32_t action = pHead->msgType % 10;
pthread_mutex_lock(&pTable->mutex);
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
if (pMeta) sdbIncRef(pTable, pMeta->row);
pthread_mutex_unlock(&pTable->mutex);
SSdbTable *pTable = sdbGetTableFromId(tableId);
assert(pTable != NULL);
if (pMeta == NULL) {
return NULL;
if (pHead->version == 0) {
return sdbProcessWriteFromApp(pTable, pHead, action);
} else {
return sdbProcessWriteFromWal(pTable, pHead, action);
}
return pMeta->row;
}
int32_t sdbInsertRow(SSdbOperDesc *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) {
sdbError("sdb tables is null");
return TSDB_CODE_OTHERS;
}
if (pTable == NULL) return -1;
if (sdbGetRow(pTable, pOper->pObj)) {
sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj));
......@@ -489,87 +385,33 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) {
return TSDB_CODE_ALREADY_THERE;
}
pOper->maxRowSize = pTable->maxRowSize;
if (pTable->keyType == SDB_KEY_AUTO) {
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_INSERT,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pOper->rowSize,
.rowData = pOper->rowData,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion);
pthread_mutex_unlock(&pTable->mutex);
return TSDB_CODE_OTHERS;
}
}
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
pthread_mutex_unlock(&pTable->mutex);
sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion);
return -1;
}
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
*((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
// let vgId increase from 2
if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) {
*((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
}
}
pTable->version++;
sdbVersion++;
pOper->rowData = rowHead->data;
(*pTable->encodeFp)(pOper);
rowHead->rowSize = pOper->rowSize;
rowHead->delimiter = SDB_DELIMITER;
rowHead->version = pTable->version;
assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize);
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("table:%s, failed to get checksum while inserting", pTable->tableName);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
tfree(rowHead);
// update in SDB layer
SRowMeta rowMeta;
rowMeta.version = pTable->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta);
sdbIncRef(pTable, pOper->pObj);
if (pOper->type == SDB_OPER_GLOBAL) {
int32_t size = sizeof(SWalHead) + pTable->maxRowSize;
SWalHead *pHead = taosAllocateQitem(size);
pHead->version = 0;
pHead->len = pOper->rowSize;
pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
pTable->numOfRows++;
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
pOper->rowData = pHead->cont;
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
(*pTable->insertFp)(pOper);
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
if (code < 0) return code;
}
return 0;
return sdbInsertLocal(pTable, pOper);
}
// row here can be object or null-terminated string
......@@ -586,218 +428,137 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
void * pMetaRow = pMeta->row;
assert(pMetaRow != NULL);
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_DELETE,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pMeta->rowSize,
.rowData = pMeta->row,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to delete record", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
}
int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
pTable->version++;
sdbVersion++;
if (pOper->type == SDB_OPER_GLOBAL) {
int32_t rowSize = 0;
switch (pTable->keyType) {
case SDB_KEY_TYPE_STRING:
case SDB_KEY_STRING:
rowSize = strlen((char *)pOper->pObj) + 1;
break;
case SDB_KEY_TYPE_AUTO:
case SDB_KEY_INT:
case SDB_KEY_AUTO:
rowSize = sizeof(uint64_t);
break;
default:
return -1;
}
rowHead->delimiter = SDB_DELIMITER;
rowHead->rowSize = rowSize;
rowHead->version = -(pTable->version);
memcpy(rowHead->data, pOper->pObj, rowSize);
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
tfree(rowHead);
int32_t size = sizeof(SWalHead) + rowSize;
SWalHead *pHead = taosAllocateQitem(size);
pHead->version = 0;
pHead->len = rowSize;
pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
memcpy(pHead->cont, pOper->pObj, rowSize);
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
// Delete from current layer
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
if (code < 0) return code;
}
(*pTable->deleteFp)(pOper);
int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1;
*updateEnd = 1;
sdbDecRef(pTable, pOper->pObj);
return 0;
return sdbDeleteLocal(pTable, pOper);
}
// row here can be the object or the string info (encoded string)
int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version);
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
}
void *pMetaRow = pMeta->row;
void * pMetaRow = pMeta->row;
assert(pMetaRow != NULL);
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_GLOBAL) {
int32_t size = sizeof(SWalHead) + pTable->maxRowSize;
SWalHead *pHead = taosAllocateQitem(size);
pHead->version = 0;
pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_UPDATE,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pOper->rowSize,
.rowData = pOper->rowData,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to update record", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
}
pOper->rowData = pHead->cont;
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
pthread_mutex_unlock(&pTable->mutex);
sdbError("table:%s, failed to allocate row head memory", pTable->tableName);
return -1;
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
if (code < 0) return code;
}
if (pMetaRow != pOper->pObj) {
memcpy(rowHead->data, pOper->rowData, pOper->rowSize);
rowHead->rowSize = pOper->rowSize;
} else {
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.maxRowSize = pTable->maxRowSize,
.pObj = pOper->pObj
};
(*pTable->encodeFp)(&oper);
rowHead->rowSize = oper.rowSize;
}
return sdbUpdateLocal(pTable, pOper);
}
pTable->version++;
sdbVersion++;
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
rowHead->delimiter = SDB_DELIMITER;
rowHead->version = pTable->version;
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
*ppRow = NULL;
if (pTable == NULL) return NULL;
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) return NULL;
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
*ppRow = pMeta->row;
sdbIncRef(handle, pMeta->row);
pMeta->version = pTable->version;
pMeta->offset = pTable->fileSize;
pMeta->rowSize = rowHead->rowSize;
return pNode;
}
pthread_mutex_unlock(&pTable->mutex);
void *sdbOpenTable(SSdbTableDesc *pDesc) {
SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
if (pTable == NULL) return NULL;
strcpy(pTable->tableName, pDesc->tableName);
pTable->keyType = pDesc->keyType;
pTable->tableId = pDesc->tableId;
pTable->hashSessions = pDesc->hashSessions;
pTable->maxRowSize = pDesc->maxRowSize;
pTable->refCountPos = pDesc->refCountPos;
pTable->insertFp = pDesc->insertFp;
pTable->deleteFp = pDesc->deleteFp;
pTable->updateFp = pDesc->updateFp;
pTable->encodeFp = pDesc->encodeFp;
pTable->decodeFp = pDesc->decodeFp;
pTable->destroyFp = pDesc->destroyFp;
pTable->restoredFp = pDesc->restoredFp;
(*pTable->updateFp)(pOper); // update in upper layer
if (sdbInitIndexFp[pTable->keyType] != NULL) {
pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta));
}
tfree(rowHead);
pthread_mutex_init(&pTable->mutex, NULL);
return 0;
tsSdbNumOfTables++;
tsSdbTableList[pTable->tableId] = pTable;
return pTable;
}
void sdbCloseTable(void *handle) {
SSdbTable *pTable = (SSdbTable *)handle;
void * pNode = NULL;
if (pTable == NULL) return;
tsSdbNumOfTables--;
tsSdbTableList[pTable->tableId] = NULL;
void *pNode = NULL;
while (1) {
SRowMeta * pMeta;
SRowMeta *pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
SSdbOperDesc oper = {
.pObj = pMeta->row,
.table = pTable,
.version = pMeta->version,
};
(*pTable->destroyFp)(&oper);
}
if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle);
if (pTable->fd) tclose(pTable->fd);
if (sdbCleanUpIndexFp[pTable->keyType]) {
(*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle);
}
pthread_mutex_destroy(&pTable->mutex);
sdbNumOfTables--;
sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables);
tfree(pTable);
}
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
*ppRow = NULL;
if (pTable == NULL) return NULL;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) return NULL;
*ppRow = pMeta->row;
sdbIncRef(handle, pMeta->row);
return pNode;
sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables);
free(pTable);
}
src/mnode/src/mgmtShell.c
浏览文件 @ 6a504d36
......@@ -27,7 +27,7 @@
#include "mgmtDb.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
......@@ -42,7 +42,6 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *sec
static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg);
static void mgmtProcessMsgFromShell(SRpcMsg *pMsg);
static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg);
static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
......@@ -142,19 +141,13 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return;
}
if (mgmtCheckRedirect(rpcMsg->handle)) {
if (!mpeerIsMaster()) {
// rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect());
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (!mgmtInServerStatus()) {
mgmtProcessMsgWhileNotReady(rpcMsg);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED);
rpcFreeCont(rpcMsg->pCont);
......@@ -337,9 +330,9 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
}
if (pMsg->usePublicIp) {
mgmtGetMnodePublicIpList(&pHBRsp->ipList);
mpeerGetPublicIpList(&pHBRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pHBRsp->ipList);
mpeerGetPrivateIpList(&pHBRsp->ipList);
}
/*
......@@ -423,9 +416,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
pConnectRsp->superAuth = pUser->superAuth;
if (pMsg->usePublicIp) {
mgmtGetMnodePublicIpList(&pConnectRsp->ipList);
mpeerGetPublicIpList(&pConnectRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pConnectRsp->ipList);
mpeerGetPrivateIpList(&pConnectRsp->ipList);
}
connect_over:
......@@ -501,18 +494,6 @@ static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) {
rpcSendResponse(&rpcRsp);
}
static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg) {
mTrace("%s is ignored since SDB is not ready", taosMsg[rpcMsg->msgType]);
SRpcMsg rpcRsp = {
.msgType = 0,
.pCont = 0,
.contLen = 0,
.code = TSDB_CODE_NOT_READY,
.handle = rpcMsg->handle
};
rpcSendResponse(&rpcRsp);
}
void mgmtSendSimpleResp(void *thandle, int32_t code) {
SRpcMsg rpcRsp = {
.msgType = 0,
......
src/mnode/src/mgmtTable.c
浏览文件 @ 6a504d36
......@@ -29,7 +29,7 @@
#include "tcluster.h"
#include "mgmtDServer.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
......@@ -169,6 +169,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
}
static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SChildTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
......@@ -177,7 +178,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
pOper->rowSize = tsChildTableUpdateSize;
} else {
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) {
if (maxRowSize < tsChildTableUpdateSize + schemaSize) {
return TSDB_CODE_INVALID_MSG_LEN;
}
memcpy(pOper->rowData, pTable, tsChildTableUpdateSize);
......@@ -219,35 +220,11 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtInitChildTables() {
static int32_t mgmtChildTableActionRestored() {
void *pNode = NULL;
void *pLastNode = NULL;
SChildTableObj *pTable = NULL;
SChildTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableName = "ctables",
.hashSessions = tsMaxTables,
.maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.insertFp = mgmtChildTableActionInsert,
.deleteFp = mgmtChildTableActionDelete,
.updateFp = mgmtChildTableActionUpdate,
.encodeFp = mgmtChildTableActionEncode,
.decodeFp = mgmtChildTableActionDecode,
.destroyFp = mgmtChildTableActionDestroy,
};
tsChildTableSdb = sdbOpenTable(&tableDesc);
if (tsChildTableSdb == NULL) {
mError("failed to init child table data");
return -1;
}
pNode = NULL;
while (1) {
pLastNode = pNode;
mgmtDecTableRef(pTable);
......@@ -258,7 +235,7 @@ static int32_t mgmtInitChildTables() {
if (pDb == NULL) {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
......@@ -272,7 +249,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
......@@ -286,7 +263,7 @@ static int32_t mgmtInitChildTables() {
pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
......@@ -298,7 +275,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
......@@ -312,7 +289,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
......@@ -323,6 +300,35 @@ static int32_t mgmtInitChildTables() {
}
}
return 0;
}
static int32_t mgmtInitChildTables() {
SChildTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_CTABLE,
.tableName = "ctables",
.hashSessions = tsMaxTables,
.maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtChildTableActionInsert,
.deleteFp = mgmtChildTableActionDelete,
.updateFp = mgmtChildTableActionUpdate,
.encodeFp = mgmtChildTableActionEncode,
.decodeFp = mgmtChildTableActionDecode,
.destroyFp = mgmtChildTableActionDestroy,
.restoredFp = mgmtChildTableActionRestored
};
tsChildTableSdb = sdbOpenTable(&tableDesc);
if (tsChildTableSdb == NULL) {
mError("failed to init child table data");
return -1;
}
mTrace("child table is initialized");
return 0;
}
......@@ -369,12 +375,14 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
}
static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SSuperTableObj *pStable = pOper->pObj;
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) {
if (maxRowSize < tsSuperTableUpdateSize + schemaSize) {
return TSDB_CODE_INVALID_MSG_LEN;
}
......@@ -406,22 +414,28 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionRestored() {
return 0;
}
static int32_t mgmtInitSuperTables() {
SSuperTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_STABLE,
.tableName = "stables",
.hashSessions = TSDB_MAX_SUPER_TABLES,
.maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtSuperTableActionInsert,
.deleteFp = mgmtSuperTableActionDelete,
.updateFp = mgmtSuperTableActionUpdate,
.encodeFp = mgmtSuperTableActionEncode,
.decodeFp = mgmtSuperTableActionDecode,
.destroyFp = mgmtSuperTableActionDestroy,
.restoredFp = mgmtSuperTableActionRestored
};
tsSuperTableSdb = sdbOpenTable(&tableDesc);
......@@ -657,7 +671,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = sizeof(SSuperTableObj) + schemaSize
......@@ -681,7 +695,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable
};
......@@ -732,7 +746,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
pStable->sversion++;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
......@@ -763,7 +777,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
pStable->schema = realloc(pStable->schema, schemaSize);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
......@@ -798,7 +812,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
......@@ -857,7 +871,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
......@@ -894,7 +908,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
......@@ -1042,7 +1056,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsSuperTableSdb,
.pObj = pTable,
};
......@@ -1271,7 +1285,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
}
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_GLOBAL;
desc.type = SDB_OPER_GLOBAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
......@@ -1425,7 +1439,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
......@@ -1459,7 +1473,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
......@@ -1601,7 +1615,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
};
......@@ -1630,7 +1644,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
if (pTable->superTable == pStable) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
};
......@@ -1659,8 +1673,6 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_
}
static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont;
pCfg->dnode = htonl(pCfg->dnode);
pCfg->vnode = htonl(pCfg->vnode);
......@@ -1721,7 +1733,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
};
......@@ -1764,7 +1776,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
queueMsg->thandle, tstrerror(rpcMsg->code));
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
};
......
src/mnode/src/mgmtUser.c
浏览文件 @ 6a504d36
......@@ -20,7 +20,7 @@
#include "tutil.h"
#include "taccount.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtUser.h"
......@@ -70,14 +70,9 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
if (pOper->maxRowSize < tsUserUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
......@@ -89,22 +84,36 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionRestored() {
if (strcmp(tsMasterIp, tsPrivateIp) == 0) {
SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
acctReleaseAcct(pAcct);
}
return 0;
}
int32_t mgmtInitUsers() {
SUserObj tObj;
tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_USER,
.tableName = "users",
.hashSessions = TSDB_MAX_USERS,
.maxRowSize = tsUserUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtUserActionInsert,
.deleteFp = mgmtUserActionDelete,
.updateFp = mgmtUserActionUpdate,
.encodeFp = mgmtUserActionEncode,
.decodeFp = mgmtUserActionDecode,
.destroyFp = mgmtUserActionDestroy,
.restoredFp = mgmtUserActionRestored
};
tsUserSdb = sdbOpenTable(&tableDesc);
......@@ -113,12 +122,6 @@ int32_t mgmtInitUsers() {
return -1;
}
SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
acctReleaseAcct(pAcct);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg);
......@@ -143,7 +146,7 @@ void mgmtReleaseUser(SUserObj *pUser) {
static int32_t mgmtUpdateUser(SUserObj *pUser) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = tsUserUpdateSize
......@@ -191,7 +194,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = sizeof(SUserObj)
......@@ -208,7 +211,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
static int32_t mgmtDropUser(SUserObj *pUser) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser
};
......@@ -314,8 +317,6 @@ SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
}
static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pUser = pMsg->pUser;
......@@ -333,8 +334,6 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) {
}
static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
......@@ -427,8 +426,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
}
static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
......@@ -488,7 +485,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) {
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsUserSdb,
.pObj = pUser,
};
......
src/mnode/src/mgmtVgroup.c
浏览文件 @ 6a504d36
......@@ -24,7 +24,7 @@
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDServer.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
......@@ -138,13 +138,9 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pOper->maxRowSize < tsVgUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
......@@ -156,22 +152,28 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionRestored() {
return 0;
}
int32_t mgmtInitVgroups() {
SVgObj tObj;
tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_VGROUP,
.tableName = "vgroups",
.hashSessions = TSDB_MAX_VGROUPS,
.maxRowSize = tsVgUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_AUTO,
.keyType = SDB_KEY_AUTO,
.insertFp = mgmtVgroupActionInsert,
.deleteFp = mgmtVgroupActionDelete,
.updateFp = mgmtVgroupActionUpdate,
.encodeFp = mgmtVgroupActionEncode,
.decodeFp = mgmtVgroupActionDecode,
.destroyFp = mgmtVgroupActionDestroy,
.restoredFp = mgmtVgroupActionRestored,
};
tsVgroupSdb = sdbOpenTable(&tableDesc);
......@@ -187,6 +189,7 @@ int32_t mgmtInitVgroups() {
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
mTrace("vgroup is initialized");
return 0;
}
......@@ -200,7 +203,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
void mgmtUpdateVgroup(SVgObj *pVgroup) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = tsVgUpdateSize
......@@ -244,7 +247,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = sizeof(SVgObj)
......@@ -276,7 +279,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
......@@ -583,7 +586,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
mgmtAddToShellQueue(newMsg);
} else {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
......@@ -646,7 +649,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
if (queueMsg->received != queueMsg->expected) return;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
......@@ -663,8 +666,6 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
}
static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
......@@ -705,7 +706,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
};
......
src/plugins/http/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -7,6 +7,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
......
src/plugins/http/inc/cJSON.h 已删除 100644 → 0
浏览文件 @ 53b8867a
/*
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef cJSON__h
#define cJSON__h
#ifdef __cplusplus
extern "C"
{
#endif
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 5
#define CJSON_VERSION_PATCH 9
#include <stddef.h>
#include <stdint.h>
/* cJSON Types: */
#define cJSON_Invalid (0)
#define cJSON_False (1 << 0)
#define cJSON_True (1 << 1)
#define cJSON_NULL (1 << 2)
#define cJSON_Number (1 << 3)
#define cJSON_String (1 << 4)
#define cJSON_Array (1 << 5)
#define cJSON_Object (1 << 6)
#define cJSON_Raw (1 << 7) /* raw json */
#define cJSON_IsReference 256
#define cJSON_StringIsConst 512
/* The cJSON structure: */
typedef struct cJSON
{
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *next;
struct cJSON *prev;
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
struct cJSON *child;
/* The type of the item, as above. */
int type;
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
char *valuestring;
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
int64_t valueint;
/* The item's number, if type==cJSON_Number */
double valuedouble;
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
char *string;
} cJSON;
typedef struct cJSON_Hooks
{
void *(*malloc_fn)(size_t sz);
void (*free_fn)(void *ptr);
} cJSON_Hooks;
typedef int cJSON_bool;
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
#define __WINDOWS__
#endif
#ifdef __WINDOWS__
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options:
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
For *nix builds that support visibility attribute, you can define similar behavior by
setting default visibility to hidden by adding
-fvisibility=hidden (for gcc)
or
-xldscope=hidden (for sun cc)
to CFLAGS
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
*/
/* export symbols by default, this is necessary for copy pasting the C and header file */
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_EXPORT_SYMBOLS
#endif
#if defined(CJSON_HIDE_SYMBOLS)
#define CJSON_PUBLIC(type) type __stdcall
#elif defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall
#elif defined(CJSON_IMPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall
#endif
#else /* !WIN32 */
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
#else
#define CJSON_PUBLIC(type) type
#endif
#endif
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_NESTING_LIMIT
#define CJSON_NESTING_LIMIT 1000
#endif
/* returns the version of cJSON as a string */
CJSON_PUBLIC(const char*) cJSON_Version(void);
/* Supply malloc, realloc and free functions to cJSON */
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
/* Render a cJSON entity to text for transfer/storage. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. */
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
/* Delete a cJSON entity and all subentities. */
CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
/* Returns the number of items in an array (or object). */
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
/* Get item "string" from object. Case insensitive. */
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
/* These functions check the type of an item */
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
/* These calls create a cJSON item of the appropriate type. */
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
/* raw json */
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
/* These utilities create an Array of count items. */
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
/* Append item to the specified array/object. */
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
* writing to `item->string` */
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
/* Remove/Detatch items from Arrays/Objects. */
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
/* Update array items. */
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
/* Duplicate a cJSON item */
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
need to be released. With recurse!=0, it will duplicate any children connected to the item.
The item->next and ->prev pointers are always zero on return from Duplicate. */
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
CJSON_PUBLIC(void) cJSON_Minify(char *json);
/* Macros for creating things quickly. */
#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull())
#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue())
#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse())
#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b))
#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n))
#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s))
#define cJSON_AddRawToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateRaw(s))
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
/* helper for the cJSON_SetNumberValue macro */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
CJSON_PUBLIC(void) cJSON_free(void *object);
#ifdef __cplusplus
}
#endif
#endif
src/plugins/http/inc/tgHandle.h
浏览文件 @ 6a504d36
......@@ -33,6 +33,7 @@
#define TG_PASS_URL_POS 3
void tgInitHandle(HttpServer *pServer);
void tgCleanupHandle();
bool tgProcessRquest(struct HttpContext *pContext);
......
src/plugins/http/src/cJSON.c 已删除 100644 → 0
浏览文件 @ 53b8867a
/*
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/* cJSON */
/* JSON parser in C. */
#ifdef __GNUC__
#pragma GCC visibility push(default)
#endif
#include <string.h>
#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include <float.h>
#include <limits.h>
#include <ctype.h>
#include <locale.h>
#ifdef __GNUC__
#pragma GCC visibility pop
#endif
#include "cJSON.h"
/* define our own boolean type */
#define true ((cJSON_bool)1)
#define false ((cJSON_bool)0)
typedef struct {
const unsigned char *json;
size_t position;
} error;
static error global_error = { NULL, 0 };
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void)
{
return (const char*) (global_error.json + global_error.position);
}
/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */
#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 5) || (CJSON_VERSION_PATCH != 9)
#error cJSON.h and cJSON.c have different versions. Make sure that both have the same.
#endif
CJSON_PUBLIC(const char*) cJSON_Version(void)
{
static char version[15];
sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH);
return version;
}
/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */
static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2)
{
if ((string1 == NULL) || (string2 == NULL))
{
return 1;
}
if (string1 == string2)
{
return 0;
}
for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++)
{
if (*string1 == '\0')
{
return 0;
}
}
return tolower(*string1) - tolower(*string2);
}
typedef struct internal_hooks
{
void *(*allocate)(size_t size);
void (*deallocate)(void *pointer);
void *(*reallocate)(void *pointer, size_t size);
} internal_hooks;
static internal_hooks global_hooks = { malloc, free, realloc };
static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks)
{
size_t length = 0;
unsigned char *copy = NULL;
if (string == NULL)
{
return NULL;
}
length = strlen((const char*)string) + sizeof("");
if (!(copy = (unsigned char*)hooks->allocate(length)))
{
return NULL;
}
memcpy(copy, string, length);
return copy;
}
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks)
{
if (hooks == NULL)
{
/* Reset hooks */
global_hooks.allocate = malloc;
global_hooks.deallocate = free;
global_hooks.reallocate = realloc;
return;
}
global_hooks.allocate = malloc;
if (hooks->malloc_fn != NULL)
{
global_hooks.allocate = hooks->malloc_fn;
}
global_hooks.deallocate = free;
if (hooks->free_fn != NULL)
{
global_hooks.deallocate = hooks->free_fn;
}
/* use realloc only if both free and malloc are used */
global_hooks.reallocate = NULL;
if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free))
{
global_hooks.reallocate = realloc;
}
}
/* Internal constructor. */
static cJSON *cJSON_New_Item(const internal_hooks * const hooks)
{
cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON));
if (node)
{
memset(node, '\0', sizeof(cJSON));
}
return node;
}
/* Delete a cJSON structure. */
CJSON_PUBLIC(void) cJSON_Delete(cJSON *item)
{
cJSON *next = NULL;
while (item != NULL)
{
next = item->next;
if (!(item->type & cJSON_IsReference) && (item->child != NULL))
{
cJSON_Delete(item->child);
}
if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL))
{
global_hooks.deallocate(item->valuestring);
}
if (!(item->type & cJSON_StringIsConst) && (item->string != NULL))
{
global_hooks.deallocate(item->string);
}
global_hooks.deallocate(item);
item = next;
}
}
/* get the decimal point character of the current locale */
static unsigned char get_decimal_point(void)
{
struct lconv *lconv = localeconv();
return (unsigned char) lconv->decimal_point[0];
}
typedef struct
{
const unsigned char *content;
size_t length;
size_t offset;
size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */
internal_hooks hooks;
} parse_buffer;
/* check if the given size is left to read in a given parse buffer (starting with 1) */
#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length))
#define cannot_read(buffer, size) (!can_read(buffer, size))
/* check if the buffer can be accessed at the given index (starting with 0) */
#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length))
#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index))
/* get a pointer to the buffer at the position */
#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset)
/* Parse the input text to generate a number, and populate the result into item. */
static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer)
{
double number = 0;
unsigned char *after_end = NULL;
unsigned char number_c_string[64];
unsigned char decimal_point = get_decimal_point();
size_t i = 0;
if ((input_buffer == NULL) || (input_buffer->content == NULL))
{
return false;
}
/* copy the number into a temporary buffer and replace '.' with the decimal point
* of the current locale (for strtod)
* This also takes care of '\0' not necessarily being available for marking the end of the input */
for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++)
{
switch (buffer_at_offset(input_buffer)[i])
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '+':
case '-':
case 'e':
case 'E':
number_c_string[i] = buffer_at_offset(input_buffer)[i];
break;
case '.':
number_c_string[i] = decimal_point;
break;
default:
goto loop_end;
}
}
loop_end:
number_c_string[i] = '\0';
number = strtod((const char*)number_c_string, (char**)&after_end);
if (number_c_string == after_end)
{
return false; /* parse_error */
}
item->valuedouble = number;
/* use saturation in case of overflow */
if (number >= LLONG_MAX)
{
item->valueint = LLONG_MAX;
}
else if (number <= LLONG_MIN)
{
item->valueint = LLONG_MIN;
}
else
{
item->valueint = (int64_t)number;
}
item->type = cJSON_Number;
input_buffer->offset += (size_t)(after_end - number_c_string);
return true;
}
/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number)
{
if (number >= LLONG_MAX)
{
object->valueint = LLONG_MAX;
}
else if (number <= LLONG_MIN)
{
object->valueint = LLONG_MIN;
}
else
{
object->valueint = (int64_t)number;
}
return object->valuedouble = number;
}
typedef struct
{
unsigned char *buffer;
size_t length;
size_t offset;
size_t depth; /* current nesting depth (for formatted printing) */
cJSON_bool noalloc;
cJSON_bool format; /* is this print a formatted print */
internal_hooks hooks;
} printbuffer;
/* realloc printbuffer if necessary to have at least "needed" bytes more */
static unsigned char* ensure(printbuffer * const p, size_t needed)
{
unsigned char *newbuffer = NULL;
size_t newsize = 0;
if ((p == NULL) || (p->buffer == NULL))
{
return NULL;
}
if ((p->length > 0) && (p->offset >= p->length))
{
/* make sure that offset is valid */
return NULL;
}
if (needed > LLONG_MAX)
{
/* sizes bigger than LLONG_MAX are currently not supported */
return NULL;
}
needed += p->offset + 1;
if (needed <= p->length)
{
return p->buffer + p->offset;
}
if (p->noalloc) {
return NULL;
}
/* calculate new buffer size */
if (needed > (LLONG_MAX / 2))
{
/* overflow of int, use LLONG_MAX if possible */
if (needed <= LLONG_MAX)
{
newsize = LLONG_MAX;
}
else
{
return NULL;
}
}
else
{
newsize = needed * 2;
}
if (p->hooks.reallocate != NULL)
{
/* reallocate with realloc if available */
newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize);
if (newbuffer == NULL)
{
p->hooks.deallocate(p->buffer);
p->length = 0;
p->buffer = NULL;
return NULL;
}
}
else
{
/* otherwise reallocate manually */
newbuffer = (unsigned char*)p->hooks.allocate(newsize);
if (!newbuffer)
{
p->hooks.deallocate(p->buffer);
p->length = 0;
p->buffer = NULL;
return NULL;
}
if (newbuffer)
{
memcpy(newbuffer, p->buffer, p->offset + 1);
}
p->hooks.deallocate(p->buffer);
}
p->length = newsize;
p->buffer = newbuffer;
return newbuffer + p->offset;
}
/* calculate the new length of the string in a printbuffer and update the offset */
static void update_offset(printbuffer * const buffer)
{
const unsigned char *buffer_pointer = NULL;
if ((buffer == NULL) || (buffer->buffer == NULL))
{
return;
}
buffer_pointer = buffer->buffer + buffer->offset;
buffer->offset += strlen((const char*)buffer_pointer);
}
/* Render the number nicely from the given item into a string. */
static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer)
{
unsigned char *output_pointer = NULL;
double d = item->valuedouble;
int length = 0;
size_t i = 0;
unsigned char number_buffer[26]; /* temporary buffer to print the number into */
unsigned char decimal_point = get_decimal_point();
double test;
if (output_buffer == NULL)
{
return false;
}
/* This checks for NaN and Infinity */
if ((d * 0) != 0)
{
length = sprintf((char*)number_buffer, "null");
}
else
{
/* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */
length = sprintf((char*)number_buffer, "%1.15g", d);
/* Check whether the original double can be recovered */
if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || ((double)test != d))
{
/* If not, print with 17 decimal places of precision */
length = sprintf((char*)number_buffer, "%1.17g", d);
}
}
/* sprintf failed or buffer overrun occured */
if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1)))
{
return false;
}
/* reserve appropriate space in the output */
output_pointer = ensure(output_buffer, (size_t)length);
if (output_pointer == NULL)
{
return false;
}
/* copy the printed number to the output and replace locale
* dependent decimal point with '.' */
for (i = 0; i < ((size_t)length); i++)
{
if (number_buffer[i] == decimal_point)
{
output_pointer[i] = '.';
continue;
}
output_pointer[i] = number_buffer[i];
}
output_pointer[i] = '\0';
output_buffer->offset += (size_t)length;
return true;
}
/* parse 4 digit hexadecimal number */
static unsigned parse_hex4(const unsigned char * const input)
{
unsigned int h = 0;
size_t i = 0;
for (i = 0; i < 4; i++)
{
/* parse digit */
if ((input[i] >= '0') && (input[i] <= '9'))
{
h += (unsigned int) input[i] - '0';
}
else if ((input[i] >= 'A') && (input[i] <= 'F'))
{
h += (unsigned int) 10 + input[i] - 'A';
}
else if ((input[i] >= 'a') && (input[i] <= 'f'))
{
h += (unsigned int) 10 + input[i] - 'a';
}
else /* invalid */
{
return 0;
}
if (i < 3)
{
/* shift left to make place for the next nibble */
h = h << 4;
}
}
return h;
}
/* converts a UTF-16 literal to UTF-8
* A literal can be one or two sequences of the form \uXXXX */
static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer)
{
uint64_t codepoint = 0;
unsigned int first_code = 0;
const unsigned char *first_sequence = input_pointer;
unsigned char utf8_length = 0;
unsigned char utf8_position = 0;
unsigned char sequence_length = 0;
unsigned char first_byte_mark = 0;
if ((input_end - first_sequence) < 6)
{
/* input ends unexpectedly */
goto fail;
}
/* get the first utf16 sequence */
first_code = parse_hex4(first_sequence + 2);
/* check that the code is valid */
if (((first_code >= 0xDC00) && (first_code <= 0xDFFF)))
{
goto fail;
}
/* UTF16 surrogate pair */
if ((first_code >= 0xD800) && (first_code <= 0xDBFF))
{
const unsigned char *second_sequence = first_sequence + 6;
unsigned int second_code = 0;
sequence_length = 12; /* \uXXXX\uXXXX */
if ((input_end - second_sequence) < 6)
{
/* input ends unexpectedly */
goto fail;
}
if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u'))
{
/* missing second half of the surrogate pair */
goto fail;
}
/* get the second utf16 sequence */
second_code = parse_hex4(second_sequence + 2);
/* check that the code is valid */
if ((second_code < 0xDC00) || (second_code > 0xDFFF))
{
/* invalid second half of the surrogate pair */
goto fail;
}
/* calculate the unicode codepoint from the surrogate pair */
codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF));
}
else
{
sequence_length = 6; /* \uXXXX */
codepoint = first_code;
}
/* encode as UTF-8
* takes at maximum 4 bytes to encode:
* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
if (codepoint < 0x80)
{
/* normal ascii, encoding 0xxxxxxx */
utf8_length = 1;
}
else if (codepoint < 0x800)
{
/* two bytes, encoding 110xxxxx 10xxxxxx */
utf8_length = 2;
first_byte_mark = 0xC0; /* 11000000 */
}
else if (codepoint < 0x10000)
{
/* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */
utf8_length = 3;
first_byte_mark = 0xE0; /* 11100000 */
}
else if (codepoint <= 0x10FFFF)
{
/* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */
utf8_length = 4;
first_byte_mark = 0xF0; /* 11110000 */
}
else
{
/* invalid unicode codepoint */
goto fail;
}
/* encode as utf8 */
for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--)
{
/* 10xxxxxx */
(*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF);
codepoint >>= 6;
}
/* encode first byte */
if (utf8_length > 1)
{
(*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF);
}
else
{
(*output_pointer)[0] = (unsigned char)(codepoint & 0x7F);
}
*output_pointer += utf8_length;
return sequence_length;
fail:
return 0;
}
/* Parse the input text into an unescaped cinput, and populate item. */
static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer)
{
const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1;
const unsigned char *input_end = buffer_at_offset(input_buffer) + 1;
unsigned char *output_pointer = NULL;
unsigned char *output = NULL;
/* not a string */
if (buffer_at_offset(input_buffer)[0] != '\"')
{
goto fail;
}
{
/* calculate approximate size of the output (overestimate) */
size_t allocation_length = 0;
size_t skipped_bytes = 0;
while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"'))
{
/* is escape sequence */
if (input_end[0] == '\\')
{
if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length)
{
/* prevent buffer overflow when last input character is a backslash */
goto fail;
}
skipped_bytes++;
input_end++;
}
input_end++;
}
if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"'))
{
goto fail; /* string ended unexpectedly */
}
/* This is at most how much we need for the output */
allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes;
output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof(""));
if (output == NULL)
{
goto fail; /* allocation failure */
}
}
output_pointer = output;
/* loop through the string literal */
while (input_pointer < input_end)
{
if (*input_pointer != '\\')
{
*output_pointer++ = *input_pointer++;
}
/* escape sequence */
else
{
unsigned char sequence_length = 2;
if ((input_end - input_pointer) < 1)
{
goto fail;
}
switch (input_pointer[1])
{
case 'b':
*output_pointer++ = '\b';
break;
case 'f':
*output_pointer++ = '\f';
break;
case 'n':
*output_pointer++ = '\n';
break;
case 'r':
*output_pointer++ = '\r';
break;
case 't':
*output_pointer++ = '\t';
break;
case '\"':
case '\\':
case '/':
*output_pointer++ = input_pointer[1];
break;
/* UTF-16 literal */
case 'u':
sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer);
if (sequence_length == 0)
{
/* failed to convert UTF16-literal to UTF-8 */
goto fail;
}
break;
default:
goto fail;
}
input_pointer += sequence_length;
}
}
/* zero terminate the output */
*output_pointer = '\0';
item->type = cJSON_String;
item->valuestring = (char*)output;
input_buffer->offset = (size_t) (input_end - input_buffer->content);
input_buffer->offset++;
return true;
fail:
if (output != NULL)
{
input_buffer->hooks.deallocate(output);
}
if (input_pointer != NULL)
{
input_buffer->offset = (size_t)(input_pointer - input_buffer->content);
}
return false;
}
/* Render the cstring provided to an escaped version that can be printed. */
static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer)
{
const unsigned char *input_pointer = NULL;
unsigned char *output = NULL;
unsigned char *output_pointer = NULL;
size_t output_length = 0;
/* numbers of additional characters needed for escaping */
size_t escape_characters = 0;
if (output_buffer == NULL)
{
return false;
}
/* empty string */
if (input == NULL)
{
output = ensure(output_buffer, sizeof("\"\""));
if (output == NULL)
{
return false;
}
strcpy((char*)output, "\"\"");
return true;
}
/* set "flag" to 1 if something needs to be escaped */
for (input_pointer = input; *input_pointer; input_pointer++)
{
switch (*input_pointer)
{
case '\"':
case '\\':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
/* one character escape sequence */
escape_characters++;
break;
default:
if (*input_pointer < 32)
{
/* UTF-16 escape sequence uXXXX */
escape_characters += 5;
}
break;
}
}
output_length = (size_t)(input_pointer - input) + escape_characters;
output = ensure(output_buffer, output_length + sizeof("\"\""));
if (output == NULL)
{
return false;
}
/* no characters have to be escaped */
if (escape_characters == 0)
{
output[0] = '\"';
memcpy(output + 1, input, output_length);
output[output_length + 1] = '\"';
output[output_length + 2] = '\0';
return true;
}
output[0] = '\"';
output_pointer = output + 1;
/* copy the string */
for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++)
{
if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\'))
{
/* normal character, copy */
*output_pointer = *input_pointer;
}
else
{
/* character needs to be escaped */
*output_pointer++ = '\\';
switch (*input_pointer)
{
case '\\':
*output_pointer = '\\';
break;
case '\"':
*output_pointer = '\"';
break;
case '\b':
*output_pointer = 'b';
break;
case '\f':
*output_pointer = 'f';
break;
case '\n':
*output_pointer = 'n';
break;
case '\r':
*output_pointer = 'r';
break;
case '\t':
*output_pointer = 't';
break;
default:
/* escape and print as unicode codepoint */
sprintf((char*)output_pointer, "u%04x", *input_pointer);
output_pointer += 4;
break;
}
}
}
output[output_length + 1] = '\"';
output[output_length + 2] = '\0';
return true;
}
/* Invoke print_string_ptr (which is useful) on an item. */
static cJSON_bool print_string(const cJSON * const item, printbuffer * const p)
{
return print_string_ptr((unsigned char*)item->valuestring, p);
}
/* Predeclare these prototypes. */
static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer);
static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer);
static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer);
static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer);
static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer);
static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer);
/* Utility to jump whitespace and cr/lf */
static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer)
{
if ((buffer == NULL) || (buffer->content == NULL))
{
return NULL;
}
while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32))
{
buffer->offset++;
}
if (buffer->offset == buffer->length)
{
buffer->offset--;
}
return buffer;
}
/* Parse an object - create a new root, and populate. */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated)
{
parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } };
cJSON *item = NULL;
/* reset error position */
global_error.json = NULL;
global_error.position = 0;
if (value == NULL)
{
goto fail;
}
buffer.content = (const unsigned char*)value;
buffer.length = strlen((const char*)value) + sizeof("");
buffer.offset = 0;
buffer.hooks = global_hooks;
item = cJSON_New_Item(&global_hooks);
if (item == NULL) /* memory fail */
{
goto fail;
}
if (!parse_value(item, buffer_skip_whitespace(&buffer)))
{
/* parse failure. ep is set. */
goto fail;
}
/* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */
if (require_null_terminated)
{
buffer_skip_whitespace(&buffer);
if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0')
{
goto fail;
}
}
if (return_parse_end)
{
*return_parse_end = (const char*)buffer_at_offset(&buffer);
}
return item;
fail:
if (item != NULL)
{
cJSON_Delete(item);
}
if (value != NULL)
{
error local_error;
local_error.json = (const unsigned char*)value;
local_error.position = 0;
if (buffer.offset < buffer.length)
{
local_error.position = buffer.offset;
}
else if (buffer.length > 0)
{
local_error.position = buffer.length - 1;
}
if (return_parse_end != NULL)
{
*return_parse_end = (const char*)local_error.json + local_error.position;
}
global_error = local_error;
}
return NULL;
}
/* Default options for cJSON_Parse */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value)
{
return cJSON_ParseWithOpts(value, 0, 0);
}
#define cjson_min(a, b) ((a < b) ? a : b)
static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks)
{
printbuffer buffer[1];
unsigned char *printed = NULL;
memset(buffer, 0, sizeof(buffer));
/* create buffer */
buffer->buffer = (unsigned char*) hooks->allocate(256);
buffer->format = format;
buffer->hooks = *hooks;
if (buffer->buffer == NULL)
{
goto fail;
}
/* print the value */
if (!print_value(item, buffer))
{
goto fail;
}
update_offset(buffer);
/* check if reallocate is available */
if (hooks->reallocate != NULL)
{
printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->length);
buffer->buffer = NULL;
if (printed == NULL) {
goto fail;
}
}
else /* otherwise copy the JSON over to a new buffer */
{
printed = (unsigned char*) hooks->allocate(buffer->offset + 1);
if (printed == NULL)
{
goto fail;
}
memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1));
printed[buffer->offset] = '\0'; /* just to be sure */
/* free the buffer */
hooks->deallocate(buffer->buffer);
}
return printed;
fail:
if (buffer->buffer != NULL)
{
hooks->deallocate(buffer->buffer);
}
if (printed != NULL)
{
hooks->deallocate(printed);
}
return NULL;
}
/* Render a cJSON item/entity/structure to text. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item)
{
return (char*)print(item, true, &global_hooks);
}
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item)
{
return (char*)print(item, false, &global_hooks);
}
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt)
{
printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } };
if (prebuffer < 0)
{
return NULL;
}
p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer);
if (!p.buffer)
{
return NULL;
}
p.length = (size_t)prebuffer;
p.offset = 0;
p.noalloc = false;
p.format = fmt;
p.hooks = global_hooks;
if (!print_value(item, &p))
{
global_hooks.deallocate(p.buffer);
return NULL;
}
return (char*)p.buffer;
}
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const cJSON_bool fmt)
{
printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } };
if ((len < 0) || (buf == NULL))
{
return false;
}
p.buffer = (unsigned char*)buf;
p.length = (size_t)len;
p.offset = 0;
p.noalloc = true;
p.format = fmt;
p.hooks = global_hooks;
return print_value(item, &p);
}
/* Parser core - when encountering text, process appropriately. */
static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer)
{
if ((input_buffer == NULL) || (input_buffer->content == NULL))
{
return false; /* no input */
}
/* parse the different types of values */
/* null */
if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0))
{
item->type = cJSON_NULL;
input_buffer->offset += 4;
return true;
}
/* false */
if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0))
{
item->type = cJSON_False;
input_buffer->offset += 5;
return true;
}
/* true */
if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0))
{
item->type = cJSON_True;
item->valueint = 1;
input_buffer->offset += 4;
return true;
}
/* string */
if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"'))
{
return parse_string(item, input_buffer);
}
/* number */
if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9'))))
{
return parse_number(item, input_buffer);
}
/* array */
if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '['))
{
return parse_array(item, input_buffer);
}
/* object */
if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{'))
{
return parse_object(item, input_buffer);
}
return false;
}
/* Render a value to text. */
static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer)
{
unsigned char *output = NULL;
if ((item == NULL) || (output_buffer == NULL))
{
return false;
}
switch ((item->type) & 0xFF)
{
case cJSON_NULL:
output = ensure(output_buffer, 5);
if (output == NULL)
{
return false;
}
strcpy((char*)output, "null");
return true;
case cJSON_False:
output = ensure(output_buffer, 6);
if (output == NULL)
{
return false;
}
strcpy((char*)output, "false");
return true;
case cJSON_True:
output = ensure(output_buffer, 5);
if (output == NULL)
{
return false;
}
strcpy((char*)output, "true");
return true;
case cJSON_Number:
return print_number(item, output_buffer);
case cJSON_Raw:
{
size_t raw_length = 0;
if (item->valuestring == NULL)
{
if (!output_buffer->noalloc)
{
output_buffer->hooks.deallocate(output_buffer->buffer);
}
return false;
}
raw_length = strlen(item->valuestring) + sizeof("");
output = ensure(output_buffer, raw_length);
if (output == NULL)
{
return false;
}
memcpy(output, item->valuestring, raw_length);
return true;
}
case cJSON_String:
return print_string(item, output_buffer);
case cJSON_Array:
return print_array(item, output_buffer);
case cJSON_Object:
return print_object(item, output_buffer);
default:
return false;
}
}
/* Build an array from input text. */
static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer)
{
cJSON *head = NULL; /* head of the linked list */
cJSON *current_item = NULL;
if (input_buffer->depth >= CJSON_NESTING_LIMIT)
{
return false; /* to deeply nested */
}
input_buffer->depth++;
if (buffer_at_offset(input_buffer)[0] != '[')
{
/* not an array */
goto fail;
}
input_buffer->offset++;
buffer_skip_whitespace(input_buffer);
if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']'))
{
/* empty array */
goto success;
}
/* check if we skipped to the end of the buffer */
if (cannot_access_at_index(input_buffer, 0))
{
input_buffer->offset--;
goto fail;
}
/* step back to character in front of the first element */
input_buffer->offset--;
/* loop through the comma separated array elements */
do
{
/* allocate next item */
cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
if (new_item == NULL)
{
goto fail; /* allocation failure */
}
/* attach next item to list */
if (head == NULL)
{
/* start the linked list */
current_item = head = new_item;
}
else
{
/* add to the end and advance */
current_item->next = new_item;
new_item->prev = current_item;
current_item = new_item;
}
/* parse next value */
input_buffer->offset++;
buffer_skip_whitespace(input_buffer);
if (!parse_value(current_item, input_buffer))
{
goto fail; /* failed to parse value */
}
buffer_skip_whitespace(input_buffer);
}
while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ','));
if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']')
{
goto fail; /* expected end of array */
}
success:
input_buffer->depth--;
item->type = cJSON_Array;
item->child = head;
input_buffer->offset++;
return true;
fail:
if (head != NULL)
{
cJSON_Delete(head);
}
return false;
}
/* Render an array to text */
static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer)
{
unsigned char *output_pointer = NULL;
size_t length = 0;
cJSON *current_element = item->child;
if (output_buffer == NULL)
{
return false;
}
/* Compose the output array. */
/* opening square bracket */
output_pointer = ensure(output_buffer, 1);
if (output_pointer == NULL)
{
return false;
}
*output_pointer = '[';
output_buffer->offset++;
output_buffer->depth++;
while (current_element != NULL)
{
if (!print_value(current_element, output_buffer))
{
return false;
}
update_offset(output_buffer);
if (current_element->next)
{
length = (size_t) (output_buffer->format ? 2 : 1);
output_pointer = ensure(output_buffer, length + 1);
if (output_pointer == NULL)
{
return false;
}
*output_pointer++ = ',';
if(output_buffer->format)
{
*output_pointer++ = ' ';
}
*output_pointer = '\0';
output_buffer->offset += length;
}
current_element = current_element->next;
}
output_pointer = ensure(output_buffer, 2);
if (output_pointer == NULL)
{
return false;
}
*output_pointer++ = ']';
*output_pointer = '\0';
output_buffer->depth--;
return true;
}
/* Build an object from the text. */
static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer)
{
cJSON *head = NULL; /* linked list head */
cJSON *current_item = NULL;
if (input_buffer->depth >= CJSON_NESTING_LIMIT)
{
return false; /* to deeply nested */
}
input_buffer->depth++;
if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{'))
{
goto fail; /* not an object */
}
input_buffer->offset++;
buffer_skip_whitespace(input_buffer);
if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}'))
{
goto success; /* empty object */
}
/* check if we skipped to the end of the buffer */
if (cannot_access_at_index(input_buffer, 0))
{
input_buffer->offset--;
goto fail;
}
/* step back to character in front of the first element */
input_buffer->offset--;
/* loop through the comma separated array elements */
do
{
/* allocate next item */
cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
if (new_item == NULL)
{
goto fail; /* allocation failure */
}
/* attach next item to list */
if (head == NULL)
{
/* start the linked list */
current_item = head = new_item;
}
else
{
/* add to the end and advance */
current_item->next = new_item;
new_item->prev = current_item;
current_item = new_item;
}
/* parse the name of the child */
input_buffer->offset++;
buffer_skip_whitespace(input_buffer);
if (!parse_string(current_item, input_buffer))
{
goto fail; /* faile to parse name */
}
buffer_skip_whitespace(input_buffer);
/* swap valuestring and string, because we parsed the name */
current_item->string = current_item->valuestring;
current_item->valuestring = NULL;
if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':'))
{
goto fail; /* invalid object */
}
/* parse the value */
input_buffer->offset++;
buffer_skip_whitespace(input_buffer);
if (!parse_value(current_item, input_buffer))
{
goto fail; /* failed to parse value */
}
buffer_skip_whitespace(input_buffer);
}
while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ','));
if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}'))
{
goto fail; /* expected end of object */
}
success:
input_buffer->depth--;
item->type = cJSON_Object;
item->child = head;
input_buffer->offset++;
return true;
fail:
if (head != NULL)
{
cJSON_Delete(head);
}
return false;
}
/* Render an object to text. */
static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer)
{
unsigned char *output_pointer = NULL;
size_t length = 0;
cJSON *current_item = item->child;
if (output_buffer == NULL)
{
return false;
}
/* Compose the output: */
length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */
output_pointer = ensure(output_buffer, length + 1);
if (output_pointer == NULL)
{
return false;
}
*output_pointer++ = '{';
output_buffer->depth++;
if (output_buffer->format)
{
*output_pointer++ = '\n';
}
output_buffer->offset += length;
while (current_item)
{
if (output_buffer->format)
{
size_t i;
output_pointer = ensure(output_buffer, output_buffer->depth);
if (output_pointer == NULL)
{
return false;
}
for (i = 0; i < output_buffer->depth; i++)
{
*output_pointer++ = '\t';
}
output_buffer->offset += output_buffer->depth;
}
/* print key */
if (!print_string_ptr((unsigned char*)current_item->string, output_buffer))
{
return false;
}
update_offset(output_buffer);
length = (size_t) (output_buffer->format ? 2 : 1);
output_pointer = ensure(output_buffer, length);
if (output_pointer == NULL)
{
return false;
}
*output_pointer++ = ':';
if (output_buffer->format)
{
*output_pointer++ = '\t';
}
output_buffer->offset += length;
/* print value */
if (!print_value(current_item, output_buffer))
{
return false;
}
update_offset(output_buffer);
/* print comma if not last */
length = (size_t) ((output_buffer->format ? 1 : 0) + (current_item->next ? 1 : 0));
output_pointer = ensure(output_buffer, length + 1);
if (output_pointer == NULL)
{
return false;
}
if (current_item->next)
{
*output_pointer++ = ',';
}
if (output_buffer->format)
{
*output_pointer++ = '\n';
}
*output_pointer = '\0';
output_buffer->offset += length;
current_item = current_item->next;
}
output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2);
if (output_pointer == NULL)
{
return false;
}
if (output_buffer->format)
{
size_t i;
for (i = 0; i < (output_buffer->depth - 1); i++)
{
*output_pointer++ = '\t';
}
}
*output_pointer++ = '}';
*output_pointer = '\0';
output_buffer->depth--;
return true;
}
/* Get Array size/item / object item. */
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array)
{
cJSON *child = NULL;
size_t size = 0;
if (array == NULL)
{
return 0;
}
child = array->child;
while(child != NULL)
{
size++;
child = child->next;
}
/* FIXME: Can overflow here. Cannot be fixed without breaking the API */
return (int)size;
}
static cJSON* get_array_item(const cJSON *array, size_t index)
{
cJSON *current_child = NULL;
if (array == NULL)
{
return NULL;
}
current_child = array->child;
while ((current_child != NULL) && (index > 0))
{
index--;
current_child = current_child->next;
}
return current_child;
}
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index)
{
if (index < 0)
{
return NULL;
}
return get_array_item(array, (size_t)index);
}
static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive)
{
cJSON *current_element = NULL;
if ((object == NULL) || (name == NULL))
{
return NULL;
}
current_element = object->child;
if (case_sensitive)
{
while ((current_element != NULL) && (strcmp(name, current_element->string) != 0))
{
current_element = current_element->next;
}
}
else
{
while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0))
{
current_element = current_element->next;
}
}
return current_element;
}
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string)
{
return get_object_item(object, string, false);
}
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string)
{
return get_object_item(object, string, true);
}
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string)
{
return cJSON_GetObjectItem(object, string) ? 1 : 0;
}
/* Utility for array list handling. */
static void suffix_object(cJSON *prev, cJSON *item)
{
prev->next = item;
item->prev = prev;
}
/* Utility for handling references. */
static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks)
{
cJSON *reference = NULL;
if (item == NULL)
{
return NULL;
}
reference = cJSON_New_Item(hooks);
if (reference == NULL)
{
return NULL;
}
memcpy(reference, item, sizeof(cJSON));
reference->string = NULL;
reference->type |= cJSON_IsReference;
reference->next = reference->prev = NULL;
return reference;
}
/* Add item to array/object. */
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item)
{
cJSON *child = NULL;
if ((item == NULL) || (array == NULL))
{
return;
}
child = array->child;
if (child == NULL)
{
/* list is empty, start new one */
array->child = item;
}
else
{
/* append to the end */
while (child->next)
{
child = child->next;
}
suffix_object(child, item);
}
}
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item)
{
if (item == NULL)
{
return;
}
/* call cJSON_AddItemToObjectCS for code reuse */
cJSON_AddItemToObjectCS(object, (char*)cJSON_strdup((const unsigned char*)string, &global_hooks), item);
/* remove cJSON_StringIsConst flag */
item->type &= ~cJSON_StringIsConst;
}
#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5))))
#pragma GCC diagnostic push
#endif
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wcast-qual"
#endif
/* Add an item to an object with constant string as key */
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item)
{
if ((item == NULL) || (string == NULL))
{
return;
}
if (!(item->type & cJSON_StringIsConst) && item->string)
{
global_hooks.deallocate(item->string);
}
item->string = (char*)string;
item->type |= cJSON_StringIsConst;
cJSON_AddItemToArray(object, item);
}
#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5))))
#pragma GCC diagnostic pop
#endif
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item)
{
if (array == NULL)
{
return;
}
cJSON_AddItemToArray(array, create_reference(item, &global_hooks));
}
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item)
{
if ((object == NULL) || (string == NULL))
{
return;
}
cJSON_AddItemToObject(object, string, create_reference(item, &global_hooks));
}
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item)
{
if ((parent == NULL) || (item == NULL))
{
return NULL;
}
if (item->prev != NULL)
{
/* not the first element */
item->prev->next = item->next;
}
if (item->next != NULL)
{
/* not the last element */
item->next->prev = item->prev;
}
if (item == parent->child)
{
/* first element */
parent->child = item->next;
}
/* make sure the detached item doesn't point anywhere anymore */
item->prev = NULL;
item->next = NULL;
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which)
{
if (which < 0)
{
return NULL;
}
return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which));
}
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which)
{
cJSON_Delete(cJSON_DetachItemFromArray(array, which));
}
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string)
{
cJSON *to_detach = cJSON_GetObjectItem(object, string);
return cJSON_DetachItemViaPointer(object, to_detach);
}
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string)
{
cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string);
return cJSON_DetachItemViaPointer(object, to_detach);
}
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string)
{
cJSON_Delete(cJSON_DetachItemFromObject(object, string));
}
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string)
{
cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string));
}
/* Replace array/object items with new ones. */
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem)
{
cJSON *after_inserted = NULL;
if (which < 0)
{
return;
}
after_inserted = get_array_item(array, (size_t)which);
if (after_inserted == NULL)
{
cJSON_AddItemToArray(array, newitem);
return;
}
newitem->next = after_inserted;
newitem->prev = after_inserted->prev;
after_inserted->prev = newitem;
if (after_inserted == array->child)
{
array->child = newitem;
}
else
{
newitem->prev->next = newitem;
}
}
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement)
{
if ((parent == NULL) || (replacement == NULL) || (item == NULL))
{
return false;
}
if (replacement == item)
{
return true;
}
replacement->next = item->next;
replacement->prev = item->prev;
if (replacement->next != NULL)
{
replacement->next->prev = replacement;
}
if (replacement->prev != NULL)
{
replacement->prev->next = replacement;
}
if (parent->child == item)
{
parent->child = replacement;
}
item->next = NULL;
item->prev = NULL;
cJSON_Delete(item);
return true;
}
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem)
{
if (which < 0)
{
return;
}
cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem);
}
static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive)
{
if ((replacement == NULL) || (string == NULL))
{
return false;
}
/* replace the name in the replacement */
if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL))
{
cJSON_free(replacement->string);
}
replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks);
replacement->type &= ~cJSON_StringIsConst;
cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement);
return true;
}
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem)
{
replace_item_in_object(object, string, newitem, false);
}
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem)
{
replace_item_in_object(object, string, newitem, true);
}
/* Create basic types: */
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_NULL;
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_True;
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_False;
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool b)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = b ? cJSON_True : cJSON_False;
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_Number;
item->valuedouble = num;
/* use saturation in case of overflow */
if (num >= LLONG_MAX)
{
item->valueint = LLONG_MAX;
}
else if (num <= LLONG_MIN)
{
item->valueint = LLONG_MIN;
}
else
{
item->valueint = (int64_t)num;
}
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_String;
item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks);
if(!item->valuestring)
{
cJSON_Delete(item);
return NULL;
}
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type = cJSON_Raw;
item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks);
if(!item->valuestring)
{
cJSON_Delete(item);
return NULL;
}
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if(item)
{
item->type=cJSON_Array;
}
return item;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void)
{
cJSON *item = cJSON_New_Item(&global_hooks);
if (item)
{
item->type = cJSON_Object;
}
return item;
}
/* Create Arrays: */
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count)
{
size_t i = 0;
cJSON *n = NULL;
cJSON *p = NULL;
cJSON *a = NULL;
if ((count < 0) || (numbers == NULL))
{
return NULL;
}
a = cJSON_CreateArray();
for(i = 0; a && (i < (size_t)count); i++)
{
n = cJSON_CreateNumber(numbers[i]);
if (!n)
{
cJSON_Delete(a);
return NULL;
}
if(!i)
{
a->child = n;
}
else
{
suffix_object(p, n);
}
p = n;
}
return a;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count)
{
size_t i = 0;
cJSON *n = NULL;
cJSON *p = NULL;
cJSON *a = NULL;
if ((count < 0) || (numbers == NULL))
{
return NULL;
}
a = cJSON_CreateArray();
for(i = 0; a && (i < (size_t)count); i++)
{
n = cJSON_CreateNumber((double)numbers[i]);
if(!n)
{
cJSON_Delete(a);
return NULL;
}
if(!i)
{
a->child = n;
}
else
{
suffix_object(p, n);
}
p = n;
}
return a;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count)
{
size_t i = 0;
cJSON *n = NULL;
cJSON *p = NULL;
cJSON *a = NULL;
if ((count < 0) || (numbers == NULL))
{
return NULL;
}
a = cJSON_CreateArray();
for(i = 0;a && (i < (size_t)count); i++)
{
n = cJSON_CreateNumber(numbers[i]);
if(!n)
{
cJSON_Delete(a);
return NULL;
}
if(!i)
{
a->child = n;
}
else
{
suffix_object(p, n);
}
p = n;
}
return a;
}
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count)
{
size_t i = 0;
cJSON *n = NULL;
cJSON *p = NULL;
cJSON *a = NULL;
if ((count < 0) || (strings == NULL))
{
return NULL;
}
a = cJSON_CreateArray();
for (i = 0; a && (i < (size_t)count); i++)
{
n = cJSON_CreateString(strings[i]);
if(!n)
{
cJSON_Delete(a);
return NULL;
}
if(!i)
{
a->child = n;
}
else
{
suffix_object(p,n);
}
p = n;
}
return a;
}
/* Duplication */
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse)
{
cJSON *newitem = NULL;
cJSON *child = NULL;
cJSON *next = NULL;
cJSON *newchild = NULL;
/* Bail on bad ptr */
if (!item)
{
goto fail;
}
/* Create new item */
newitem = cJSON_New_Item(&global_hooks);
if (!newitem)
{
goto fail;
}
/* Copy over all vars */
newitem->type = item->type & (~cJSON_IsReference);
newitem->valueint = item->valueint;
newitem->valuedouble = item->valuedouble;
if (item->valuestring)
{
newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks);
if (!newitem->valuestring)
{
goto fail;
}
}
if (item->string)
{
newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks);
if (!newitem->string)
{
goto fail;
}
}
/* If non-recursive, then we're done! */
if (!recurse)
{
return newitem;
}
/* Walk the ->next chain for the child. */
child = item->child;
while (child != NULL)
{
newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */
if (!newchild)
{
goto fail;
}
if (next != NULL)
{
/* If newitem->child already set, then crosswire ->prev and ->next and move on */
next->next = newchild;
newchild->prev = next;
next = newchild;
}
else
{
/* Set newitem->child and move to it */
newitem->child = newchild;
next = newchild;
}
child = child->next;
}
return newitem;
fail:
if (newitem != NULL)
{
cJSON_Delete(newitem);
}
return NULL;
}
CJSON_PUBLIC(void) cJSON_Minify(char *json)
{
unsigned char *into = (unsigned char*)json;
if (json == NULL)
{
return;
}
while (*json)
{
if (*json == ' ')
{
json++;
}
else if (*json == '\t')
{
/* Whitespace characters. */
json++;
}
else if (*json == '\r')
{
json++;
}
else if (*json=='\n')
{
json++;
}
else if ((*json == '/') && (json[1] == '/'))
{
/* double-slash comments, to end of line. */
while (*json && (*json != '\n'))
{
json++;
}
}
else if ((*json == '/') && (json[1] == '*'))
{
/* multiline comments. */
while (*json && !((*json == '*') && (json[1] == '/')))
{
json++;
}
json += 2;
}
else if (*json == '\"')
{
/* string literals, which are \" sensitive. */
*into++ = (unsigned char)*json++;
while (*json && (*json != '\"'))
{
if (*json == '\\')
{
*into++ = (unsigned char)*json++;
}
*into++ = (unsigned char)*json++;
}
*into++ = (unsigned char)*json++;
}
else
{
/* All other characters. */
*into++ = (unsigned char)*json++;
}
}
/* and null-terminate. */
*into = '\0';
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_Invalid;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_False;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xff) == cJSON_True;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & (cJSON_True | cJSON_False)) != 0;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_NULL;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_Number;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_String;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_Array;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_Object;
}
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item)
{
if (item == NULL)
{
return false;
}
return (item->type & 0xFF) == cJSON_Raw;
}
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive)
{
if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a))
{
return false;
}
/* check if type is valid */
switch (a->type & 0xFF)
{
case cJSON_False:
case cJSON_True:
case cJSON_NULL:
case cJSON_Number:
case cJSON_String:
case cJSON_Raw:
case cJSON_Array:
case cJSON_Object:
break;
default:
return false;
}
/* identical objects are equal */
if (a == b)
{
return true;
}
switch (a->type & 0xFF)
{
/* in these cases and equal type is enough */
case cJSON_False:
case cJSON_True:
case cJSON_NULL:
return true;
case cJSON_Number:
if (a->valuedouble == b->valuedouble)
{
return true;
}
return false;
case cJSON_String:
case cJSON_Raw:
if ((a->valuestring == NULL) || (b->valuestring == NULL))
{
return false;
}
if (strcmp(a->valuestring, b->valuestring) == 0)
{
return true;
}
return false;
case cJSON_Array:
{
cJSON *a_element = a->child;
cJSON *b_element = b->child;
for (; (a_element != NULL) && (b_element != NULL);)
{
if (!cJSON_Compare(a_element, b_element, case_sensitive))
{
return false;
}
a_element = a_element->next;
b_element = b_element->next;
}
/* one of the arrays is longer than the other */
if (a_element != b_element) {
return false;
}
return true;
}
case cJSON_Object:
{
cJSON *a_element = NULL;
cJSON *b_element = NULL;
cJSON_ArrayForEach(a_element, a)
{
/* TODO This has O(n^2) runtime, which is horrible! */
b_element = get_object_item(b, a_element->string, case_sensitive);
if (b_element == NULL)
{
return false;
}
if (!cJSON_Compare(a_element, b_element, case_sensitive))
{
return false;
}
}
/* doing this twice, once on a and b to prevent true comparison if a subset of b
* TODO: Do this the proper way, this is just a fix for now */
cJSON_ArrayForEach(b_element, b)
{
a_element = get_object_item(a, b_element->string, case_sensitive);
if (a_element == NULL)
{
return false;
}
if (!cJSON_Compare(b_element, a_element, case_sensitive))
{
return false;
}
}
return true;
}
default:
return false;
}
}
CJSON_PUBLIC(void *) cJSON_malloc(size_t size)
{
return global_hooks.allocate(size);
}
CJSON_PUBLIC(void) cJSON_free(void *object)
{
global_hooks.deallocate(object);
}
src/plugins/http/src/httpServer.c
浏览文件 @ 6a504d36
......@@ -270,7 +270,7 @@ void httpCleanUpConnect(HttpServer *pServer) {
for (i = 0; i < pServer->numOfThreads; ++i) {
pThread = pServer->pThreads + i;
taosCloseSocket(pThread->pollFd);
//taosCloseSocket(pThread->pollFd);
while (pThread->pHead) {
httpCleanUpContext(pThread->pHead, 0);
......@@ -591,7 +591,6 @@ void httpAcceptHttpConnection(void *arg) {
bool httpInitConnect(HttpServer *pServer) {
int i;
pthread_attr_t thattr;
HttpThread * pThread;
pServer->pThreads = (HttpThread *)malloc(sizeof(HttpThread) * (size_t)pServer->numOfThreads);
......@@ -601,8 +600,6 @@ bool httpInitConnect(HttpServer *pServer) {
}
memset(pServer->pThreads, 0, sizeof(HttpThread) * (size_t)pServer->numOfThreads);
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pThread = pServer->pThreads;
for (i = 0; i < pServer->numOfThreads; ++i) {
sprintf(pThread->label, "%s%d", pServer->label, i);
......@@ -626,21 +623,27 @@ bool httpInitConnect(HttpServer *pServer) {
return false;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) {
httpError("http thread:%s, failed to create HTTP process data thread, reason:%s",
pThread->label, strerror(errno));
return false;
}
pthread_attr_destroy(&thattr);
httpTrace("http thread:%p:%s, initialized", pThread, pThread->label);
pThread++;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pServer->thread), &thattr, (void *)httpAcceptHttpConnection, (void *)(pServer)) != 0) {
httpError("http server:%s, failed to create Http accept thread, reason:%s", pServer->label, strerror(errno));
return false;
}
pthread_attr_destroy(&thattr);
httpTrace("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, pServer->serverIp,
......
src/plugins/http/src/httpSystem.c
浏览文件 @ 6a504d36
......@@ -54,7 +54,7 @@ static HttpServer *httpServer = NULL;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
int httpInitSystem() {
taos_init();
// taos_init();
httpServer = (HttpServer *)malloc(sizeof(HttpServer));
memset(httpServer, 0, sizeof(HttpServer));
......@@ -122,12 +122,14 @@ void httpStopSystem() {
if (httpServer != NULL) {
httpServer->online = false;
}
tgCleanupHandle();
}
void httpCleanUpSystem() {
httpPrint("http service cleanup");
httpStopSystem();
#if 0
#if 1
if (httpServer == NULL) {
return;
}
......
src/plugins/http/src/tgHandle.c
浏览文件 @ 6a504d36
......@@ -116,6 +116,7 @@ void tgFreeSchemas() {
}
free(tgSchemas.schemas);
tgSchemas.size = 0;
tgSchemas.schemas = NULL;
}
}
......@@ -290,6 +291,10 @@ void tgInitHandle(HttpServer *pServer) {
httpAddMethod(pServer, &tgDecodeMethod);
}
void tgCleanupHandle() {
tgFreeSchemas();
}
bool tgGetUserFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[TG_USER_URL_POS].len <= 0) {
......
src/query/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
......
src/rpc/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -4,6 +4,8 @@ PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
......@@ -22,7 +24,7 @@ ELSEIF (TD_WINDOWS_64)
ENDIF ()
ADD_LIBRARY(trpc ${SRC})
TARGET_LINK_LIBRARIES(trpc tutil)
TARGET_LINK_LIBRARIES(trpc tutil lz4)
ADD_SUBDIRECTORY(test)
src/rpc/inc/rpcServer.h 已删除 100644 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _rpc_server_header_
#define _rpc_server_header_
#ifdef __cplusplus
extern "C" {
#endif
#include "taosdef.h"
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void taosCloseTcpServerConnection(void *param);
int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
#ifdef __cplusplus
}
#endif
#endif
src/rpc/inc/rpcClient.h → src/rpc/inc/rpcTcp.h
浏览文件 @ 6a504d36
......@@ -13,20 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _rpc_client_header_
#define _rpc_client_header_
#ifndef _rpc_tcp_header_
#define _rpc_tcp_header_
#ifdef __cplusplus
extern "C" {
#endif
#include "taosdef.h"
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void taosCloseTcpClientConnection(void *chandle);
int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
void taosCloseTcpConnection(void *chandle);
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
#ifdef __cplusplus
}
......
src/rpc/src/rpcCache.c
浏览文件 @ 6a504d36
......@@ -22,18 +22,18 @@
#include "tutil.h"
#include "rpcCache.h"
typedef struct _c_hash_t {
typedef struct SConnHash {
uint32_t ip;
uint16_t port;
char connType;
struct _c_hash_t *prev;
struct _c_hash_t *next;
void * data;
struct SConnHash *prev;
struct SConnHash *next;
void *data;
uint64_t time;
} SConnHash;
typedef struct {
SConnHash ** connHashList;
SConnHash **connHashList;
mpool_h connHashMemPool;
int maxSessions;
int total;
......@@ -103,7 +103,8 @@ void rpcCloseConnCache(void *handle) {
if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool);
tfree(pCache->connHashList);
tfree(pCache->count)
tfree(pCache->count);
tfree(pCache->lockedBy);
pthread_mutex_unlock(&pCache->mutex);
......
src/rpc/src/rpcClient.c 已删除 100644 → 0
浏览文件 @ 53b8867a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosmsg.h"
#include "tlog.h"
#include "tsocket.h"
#include "tutil.h"
#include "rpcClient.h"
#include "rpcHead.h"
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
typedef struct _tcp_fd {
void *signature;
int fd; // TCP socket FD
void * thandle;
uint32_t ip;
char ipstr[20];
uint16_t port;
struct _tcp_client *pTcp;
struct _tcp_fd * prev, *next;
} STcpFd;
typedef struct _tcp_client {
pthread_t thread;
STcpFd * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
int pollFd;
int numOfFds;
char label[12];
char ipstr[20];
void *shandle; // handle passed by upper layer during server initialization
void *(*processData)(SRecvInfo *pRecv);
} STcpClient;
#define maxTcpEvents 100
static void taosCleanUpTcpFdObj(STcpFd *pFdObj);
static void *taosReadTcpData(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
STcpClient *pTcp;
pthread_attr_t thattr;
pTcp = (STcpClient *)malloc(sizeof(STcpClient));
memset(pTcp, 0, sizeof(STcpClient));
strcpy(pTcp->label, label);
strcpy(pTcp->ipstr, ip);
pTcp->shandle = shandle;
if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) {
tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno));
return NULL;
}
if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
return NULL;
}
pTcp->pollFd = epoll_create(10); // size does not matter
if (pTcp->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
return NULL;
}
pTcp->processData = fp;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) {
tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno));
return NULL;
}
tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port);
return pTcp;
}
void taosCleanUpTcpClient(void *chandle) {
STcpClient *pTcp = (STcpClient *)chandle;
if (pTcp == NULL) return;
while (pTcp->pHead) {
taosCleanUpTcpFdObj(pTcp->pHead);
pTcp->pHead = pTcp->pHead->next;
}
close(pTcp->pollFd);
pthread_cancel(pTcp->thread);
pthread_join(pTcp->thread, NULL);
// tTrace (":%s, all connections are cleaned up", pTcp->label);
tfree(pTcp);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
STcpClient * pTcp = (STcpClient *)shandle;
STcpFd * pFdObj;
struct epoll_event event;
struct in_addr destIp;
int fd;
fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr);
if (fd <= 0) return NULL;
pFdObj = (STcpFd *)malloc(sizeof(STcpFd));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pTcp->label);
tclose(fd);
return NULL;
}
memset(pFdObj, 0, sizeof(STcpFd));
pFdObj->fd = fd;
strcpy(pFdObj->ipstr, ip);
inet_aton(ip, &destIp);
pFdObj->ip = destIp.s_addr;
pFdObj->port = port;
pFdObj->pTcp = pTcp;
pFdObj->thandle = thandle;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno));
tfree(pFdObj);
tclose(fd);
return NULL;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pTcp->mutex));
pFdObj->next = pTcp->pHead;
if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj;
pTcp->pHead = pFdObj;
pTcp->numOfFds++;
pthread_cond_signal(&pTcp->fdReady);
pthread_mutex_unlock(&(pTcp->mutex));
tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds);
return pFdObj;
}
void taosCloseTcpClientConnection(void *chandle) {
STcpFd *pFdObj = (STcpFd *)chandle;
if (pFdObj == NULL) return;
taosCleanUpTcpFdObj(pFdObj);
}
int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
STcpFd *pFdObj = (STcpFd *)chandle;
if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0);
}
static void taosCleanUpTcpFdObj(STcpFd *pFdObj) {
STcpClient *pTcp;
SRecvInfo recvInfo;
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
pTcp = pFdObj->pTcp;
if (pTcp == NULL) {
tError("double free TcpFdObj!!!!");
return;
}
epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
close(pFdObj->fd);
pthread_mutex_lock(&pTcp->mutex);
pTcp->numOfFds--;
if (pTcp->numOfFds < 0)
tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj);
if (pFdObj->prev) {
(pFdObj->prev)->next = pFdObj->next;
} else {
pTcp->pHead = pFdObj->next;
}
if (pFdObj->next) {
(pFdObj->next)->prev = pFdObj->prev;
}
pthread_mutex_unlock(&pTcp->mutex);
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pTcp->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo);
tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds);
memset(pFdObj, 0, sizeof(STcpFd));
tfree(pFdObj);
}
static void *taosReadTcpData(void *param) {
STcpClient *pTcp = (STcpClient *)param;
int i, fdNum;
STcpFd *pFdObj;
struct epoll_event events[maxTcpEvents];
SRecvInfo recvInfo;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pTcp->mutex);
if (pTcp->numOfFds < 1) pthread_cond_wait(&pTcp->fdReady, &pTcp->mutex);
pthread_mutex_unlock(&pTcp->mutex);
fdNum = epoll_wait(pTcp->pollFd, events, maxTcpEvents, -1);
if (fdNum < 0) continue;
for (i = 0; i < fdNum; ++i) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s TCP error happened on FD\n", pTcp->label);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s TCP FD hang up\n", pTcp->label);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tError("%s read error, headLen:%d", pTcp->label, headLen);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead);
if (NULL == buffer) {
tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, msgLen);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
char *msg = buffer + tsRpcOverhead;
int32_t leftLen = msgLen - headLen;
int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen);
tfree(buffer);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;
recvInfo.msgLen = msgLen;
recvInfo.ip = pFdObj->ip;
recvInfo.port = pFdObj->port;
recvInfo.shandle = pTcp->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = pFdObj;
recvInfo.connType = RPC_CONN_TCP;
pFdObj->thandle = (*(pTcp->processData))(&recvInfo);
if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj);
}
}
return NULL;
}
src/rpc/src/rpcHaship.c
浏览文件 @ 6a504d36
......@@ -17,13 +17,13 @@
#include "tlog.h"
#include "tmempool.h"
typedef struct _ip_hash_t {
typedef struct SIpHash {
uint32_t ip;
uint16_t port;
int hash;
struct _ip_hash_t *prev;
struct _ip_hash_t *next;
void * data;
struct SIpHash *prev;
struct SIpHash *next;
void *data;
} SIpHash;
typedef struct {
......@@ -47,7 +47,7 @@ int rpcHashIp(void *handle, uint32_t ip, uint16_t port) {
void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;
......@@ -70,7 +70,7 @@ void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) {
void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;
......@@ -102,7 +102,7 @@ void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) {
void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;
......
src/rpc/src/rpcMain.c 100755 → 100644
浏览文件 @ 6a504d36
......@@ -27,8 +27,7 @@
#include "taosmsg.h"
#include "rpcUdp.h"
#include "rpcCache.h"
#include "rpcClient.h"
#include "rpcServer.h"
#include "rpcTcp.h"
#include "rpcHead.h"
#include "trpc.h"
#include "hash.h"
......@@ -67,7 +66,7 @@ typedef struct {
void *udphandle;// returned handle from UDP initialization
void *pCache; // connection cache
pthread_mutex_t mutex;
struct _RpcConn *connList; // connection list
struct SRpcConn *connList; // connection list
} SRpcInfo;
typedef struct {
......@@ -88,7 +87,7 @@ typedef struct {
char msg[0]; // RpcHead starts from here
} SRpcReqContext;
typedef struct _RpcConn {
typedef struct SRpcConn {
int sid; // session ID
uint32_t ownId; // own link ID
uint32_t peerId; // peer link ID
......@@ -156,8 +155,8 @@ void (*taosCleanUpConn[])(void *thandle) = {
int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
taosSendUdpData,
taosSendUdpData,
taosSendTcpServerData,
taosSendTcpClientData
taosSendTcpData,
taosSendTcpData
};
void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = {
......@@ -170,8 +169,8 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) =
void (*taosCloseConn[])(void *chandle) = {
NULL,
NULL,
taosCloseTcpServerConnection,
taosCloseTcpClientConnection
taosCloseTcpConnection,
taosCloseTcpConnection
};
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType);
......@@ -287,15 +286,15 @@ void *rpcOpen(const SRpcInit *pInit) {
void rpcClose(void *param) {
SRpcInfo *pRpc = (SRpcInfo *)param;
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
for (int i = 0; i < pRpc->sessions; ++i) {
if (pRpc->connList && pRpc->connList[i].user[0]) {
rpcCloseConn((void *)(pRpc->connList + i));
}
}
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
taosHashCleanup(pRpc->hash);
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
......@@ -522,11 +521,15 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort,
static void rpcCloseConn(void *thandle) {
SRpcConn *pConn = (SRpcConn *)thandle;
SRpcInfo *pRpc = pConn->pRpc;
if (pConn->user[0] == 0) return;
rpcLockConn(pConn);
if (pConn->user[0] == 0) {
rpcUnlockConn(pConn);
return;
}
pConn->user[0] = 0;
if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle);
......@@ -817,7 +820,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
SRpcInfo *pRpc = pConn->pRpc;
tTrace("%s %p, link is broken", pRpc->label, pConn);
pConn->chandle = NULL;
// pConn->chandle = NULL;
if (pConn->outType) {
SRpcReqContext *pContext = pConn->pContext;
......
src/rpc/src/rpcServer.c → src/rpc/src/rpcTcp.c
浏览文件 @ 6a504d36
......@@ -18,30 +18,30 @@
#include "tlog.h"
#include "tsocket.h"
#include "tutil.h"
#include "rpcServer.h"
#include "rpcHead.h"
#include "rpcTcp.h"
#define TAOS_IPv4ADDR_LEN 16
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
typedef struct _fd_obj {
typedef struct SFdObj {
void *signature;
int fd; // TCP socket FD
void * thandle; // handle from upper layer, like TAOS
char ipstr[TAOS_IPv4ADDR_LEN];
unsigned int ip;
void *thandle; // handle from upper layer, like TAOS
uint32_t ip;
uint16_t port;
struct _thread_obj *pThreadObj;
struct _fd_obj * prev, *next;
struct SThreadObj *pThreadObj;
struct SFdObj *prev;
struct SFdObj *next;
} SFdObj;
typedef struct _thread_obj {
typedef struct SThreadObj {
pthread_t thread;
SFdObj * pHead;
pthread_mutex_t threadMutex;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
char ipstr[TSDB_IPv4ADDR_LEN];
int pollFd;
int numOfFds;
int threadId;
......@@ -51,7 +51,7 @@ typedef struct _thread_obj {
} SThreadObj;
typedef struct {
char ip[40];
char ip[TSDB_IPv4ADDR_LEN];
uint16_t port;
char label[12];
int numOfThreads;
......@@ -60,178 +60,335 @@ typedef struct {
pthread_t thread;
} SServerObj;
static void taosCleanUpFdObj(SFdObj *pFdObj);
static void taosProcessTcpData(void *param);
static void *taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd);
static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
int i;
SServerObj *pServerObj;
pthread_attr_t thattr;
SThreadObj *pThreadObj;
pServerObj = (SServerObj *)malloc(sizeof(SServerObj));
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
strcpy(pServerObj->ip, ip);
pServerObj->port = port;
strcpy(pServerObj->label, label);
pServerObj->numOfThreads = numOfThreads;
pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads);
pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
if (pServerObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
free(pServerObj);
return NULL;
}
memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads);
int code = 0;
pThreadObj = pServerObj->pThreadObj;
for (i = 0; i < numOfThreads; ++i) {
for (int i = 0; i < numOfThreads; ++i) {
pThreadObj->processData = fp;
strcpy(pThreadObj->label, label);
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) {
tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno));
return NULL;
code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
if (code < 0) {
tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
break;;
}
if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
return NULL;
code = pthread_cond_init(&(pThreadObj->fdReady), NULL);
if (code != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
break;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
return NULL;
code = -1;
break;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) {
tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno));
return NULL;
code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP process data thread(%s)", label, strerror(errno));
break;
}
pThreadObj->threadId = i;
pThreadObj++;
}
if (code == 0) {
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) {
tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno));
return NULL;
code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP accept thread(%s)", label, strerror(errno));
}
/*
if ( pthread_create(&(pServerObj->thread), &thattr,
(void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) {
tError("%s failed to create UD accept thread, reason:%s", label,
strerror(errno));
return NULL;
}
*/
pthread_attr_destroy(&thattr);
if (code != 0) {
free(pServerObj->pThreadObj);
free(pServerObj);
pServerObj = NULL;
} else {
tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
}
return (void *)pServerObj;
}
void taosCleanUpTcpServer(void *handle) {
int i;
SServerObj *pServerObj = handle;
SThreadObj *pThreadObj;
SServerObj *pServerObj = (SServerObj *)handle;
if (pServerObj == NULL) return;
pthread_cancel(pServerObj->thread);
pthread_join(pServerObj->thread, NULL);
for (i = 0; i < pServerObj->numOfThreads; ++i) {
for (int i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
while (pThreadObj->pHead) {
taosCleanUpFdObj(pThreadObj->pHead);
pThreadObj->pHead = pThreadObj->pHead;
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
pthread_cond_destroy(&(pThreadObj->fdReady));
pthread_mutex_destroy(&(pThreadObj->threadMutex));
pthread_mutex_destroy(&(pThreadObj->mutex));
}
tfree(pServerObj->pThreadObj);
tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
tfree(pServerObj->pThreadObj);
tfree(pServerObj);
}
void taosCloseTcpServerConnection(void *chandle) {
SFdObj *pFdObj = (SFdObj *)chandle;
static void taosAcceptTcpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in caddr;
int sockFd;
int threadId = 0;
SThreadObj *pThreadObj;
SServerObj *pServerObj;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) return;
tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
while (1) {
socklen_t addrlen = sizeof(caddr);
connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen);
if (connFd < 0) {
tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno));
continue;
}
tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port);
taosKeepTcpAlive(connFd);
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
pFdObj->port = caddr.sin_port;
tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
} else {
close(connFd);
tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno));
}
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
}
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
SThreadObj *pThreadObj;
pthread_attr_t thattr;
pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
memset(pThreadObj, 0, sizeof(SThreadObj));
strcpy(pThreadObj->label, label);
strcpy(pThreadObj->ipstr, ip);
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) {
tError("%s failed to init TCP client mutex(%s)", label, strerror(errno));
return NULL;
}
if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
return NULL;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP client epoll", label);
return NULL;
}
pThreadObj->processData = fp;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP read data thread(%s)", label, strerror(errno));
return NULL;
}
tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port);
return pThreadObj;
}
void taosCleanUpTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
tTrace (":%s, all connections are cleaned up", pThreadObj->label);
tfree(pThreadObj);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
SThreadObj * pThreadObj = shandle;
struct in_addr destIp;
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr);
if (fd <= 0) return NULL;
inet_aton(ip, &destIp);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = destIp.s_addr;
tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
}
return pFdObj;
}
void taosCloseTcpConnection(void *chandle) {
SFdObj *pFdObj = chandle;
if (pFdObj == NULL) return;
taosCleanUpFdObj(pFdObj);
taosFreeFdObj(pFdObj);
}
int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
SFdObj *pFdObj = (SFdObj *)chandle;
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
SFdObj *pFdObj = chandle;
if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0);
}
static void taosReportBrokenLink(SFdObj *pFdObj) {
SThreadObj *pThreadObj = pFdObj->pThreadObj;
// notify the upper layer, so it will clean the associated context
if (pFdObj->thandle) {
SRecvInfo recvInfo;
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
(*(pThreadObj->processData))(&recvInfo);
}
}
#define maxEvents 10
static void taosProcessTcpData(void *param) {
SThreadObj * pThreadObj;
int i, fdNum;
SFdObj * pFdObj;
static void *taosProcessTcpData(void *param) {
SThreadObj *pThreadObj = param;
SFdObj *pFdObj;
struct epoll_event events[maxEvents];
SRecvInfo recvInfo;
pThreadObj = (SThreadObj *)param;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pThreadObj->threadMutex);
pthread_mutex_lock(&pThreadObj->mutex);
if (pThreadObj->numOfFds < 1) {
pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->threadMutex);
pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex);
}
pthread_mutex_unlock(&pThreadObj->threadMutex);
pthread_mutex_unlock(&pThreadObj->mutex);
fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
if (fdNum < 0) continue;
for (i = 0; i < fdNum; ++i) {
for (int i = 0; i < fdNum; ++i) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s TCP thread:%d, error happened on FD", pThreadObj->label, pThreadObj->threadId);
taosCleanUpFdObj(pFdObj);
tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s TCP thread:%d, FD hang up", pThreadObj->label, pThreadObj->threadId);
taosCleanUpFdObj(pFdObj);
tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle);
taosReportBrokenLink(pFdObj);
continue;
}
int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno);
taosCleanUpFdObj(pFdObj);
tError("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
taosReportBrokenLink(pFdObj);
continue;
}
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
char *buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
tError("%s TCP malloc(size:%d) fail\n", pThreadObj->label, msgLen);
taosCleanUpFdObj(pFdObj);
tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
taosReportBrokenLink(pFdObj);
continue;
}
......@@ -240,13 +397,14 @@ static void taosProcessTcpData(void *param) {
int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s read error, leftLen:%d retLen:%d", pThreadObj->label, leftLen, retLen);
taosCleanUpFdObj(pFdObj);
tError("%s %p, read error, leftLen:%d retLen:%d",
pThreadObj->label, pFdObj->thandle, leftLen, retLen);
taosReportBrokenLink(pFdObj);
tfree(buffer);
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen);
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;
......@@ -259,111 +417,64 @@ static void taosProcessTcpData(void *param) {
recvInfo.connType = RPC_CONN_TCP;
pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj);
if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj);
}
}
return NULL;
}
static void taosAcceptTcpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in clientAddr;
int sockFd;
int threadId = 0;
SThreadObj * pThreadObj;
SServerObj * pServerObj;
SFdObj * pFdObj;
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) {
struct epoll_event event;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) {
tError("%s failed to open TCP socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
return;
} else {
tTrace("%s TCP server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
}
while (1) {
socklen_t addrlen = sizeof(clientAddr);
connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
tError("%s TCP accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno));
continue;
}
tTrace("%s TCP connection from ip:%s port:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr),
htons(clientAddr.sin_port));
taosKeepTcpAlive(connFd);
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1);
if (pFdObj == NULL) return NULL;
pFdObj = (SFdObj *)malloc(sizeof(SFdObj));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label);
close(connFd);
continue;
}
memset(pFdObj, 0, sizeof(SFdObj));
pFdObj->fd = connFd;
strcpy(pFdObj->ipstr, inet_ntoa(clientAddr.sin_addr));
pFdObj->ip = clientAddr.sin_addr.s_addr;
pFdObj->port = htons(clientAddr.sin_port);
pFdObj->fd = fd;
pFdObj->pThreadObj = pThreadObj;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
tError("%s failed to add TCP FD for epoll, error:%s", pServerObj->label, strerror(errno));
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tfree(pFdObj);
close(connFd);
continue;
return NULL;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pThreadObj->threadMutex));
pthread_mutex_lock(&(pThreadObj->mutex));
pFdObj->next = pThreadObj->pHead;
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->threadMutex));
pthread_mutex_unlock(&(pThreadObj->mutex));
tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds);
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
return pFdObj;
}
static void taosCleanUpFdObj(SFdObj *pFdObj) {
SThreadObj *pThreadObj;
static void taosFreeFdObj(SFdObj *pFdObj) {
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
pThreadObj = pFdObj->pThreadObj;
if (pThreadObj == NULL) {
tError("FdObj double clean up!!!");
SThreadObj *pThreadObj = pFdObj->pThreadObj;
pthread_mutex_lock(&pThreadObj->mutex);
if (pFdObj->signature == NULL) {
pthread_mutex_unlock(&pThreadObj->mutex);
return;
}
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
pFdObj->signature = NULL;
close(pFdObj->fd);
pthread_mutex_lock(&pThreadObj->threadMutex);
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId);
tError("%s %p, TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
// remove from the FdObject list
......@@ -377,139 +488,12 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) {
(pFdObj->next)->prev = pFdObj->prev;
}
pthread_mutex_unlock(&pThreadObj->threadMutex);
// notify the upper layer, so it will clean the associated context
SRecvInfo recvInfo;
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo);
tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId,
pFdObj, pThreadObj->numOfFds);
memset(pFdObj, 0, sizeof(SFdObj));
tfree(pFdObj);
}
#if 0
static void taosAcceptUDConnection(void *arg) {
int connFd = -1;
int sockFd;
int threadId = 0;
SThreadObj * pThreadObj;
SServerObj * pServerObj;
SFdObj * pFdObj;
struct epoll_event event;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenUDServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) {
tError("%s failed to open UD socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
return;
} else {
tTrace("%s UD server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
}
while (1) {
connFd = accept(sockFd, NULL, NULL);
if (connFd < 0) {
tError("%s UD accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno));
continue;
}
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
pFdObj = (SFdObj *)malloc(sizeof(SFdObj));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label);
close(connFd);
continue;
}
pthread_mutex_unlock(&pThreadObj->mutex);
memset(pFdObj, 0, sizeof(SFdObj));
pFdObj->fd = connFd;
pFdObj->pThreadObj = pThreadObj;
tTrace("%s %p, FD:%p is cleaned, numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
tError("%s failed to add UD FD for epoll, error:%s", pServerObj->label, strerror(errno));
tfree(pFdObj);
close(connFd);
continue;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pThreadObj->threadMutex));
pFdObj->next = pThreadObj->pHead;
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->threadMutex));
tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds);
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
}
#endif
#if 0
void taosListTcpConnection(void *handle, char *buffer) {
SServerObj *pServerObj;
SThreadObj *pThreadObj;
SFdObj * pFdObj;
int i, numOfFds, numOfConns;
char * msg;
pServerObj = (SServerObj *)handle;
buffer[0] = 0;
msg = buffer;
numOfConns = 0;
pThreadObj = pServerObj->pThreadObj;
for (i = 0; i < pServerObj->numOfThreads; ++i) {
numOfFds = 0;
sprintf(msg, "TCP:%s Thread:%d number of connections:%d\n", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds);
msg = msg + strlen(msg);
pFdObj = pThreadObj->pHead;
while (pFdObj) {
sprintf(msg, " ip:%s port:%hu\n", pFdObj->ipstr, pFdObj->port);
msg = msg + strlen(msg);
numOfFds++;
numOfConns++;
pFdObj = pFdObj->next;
}
if (numOfFds != pThreadObj->numOfFds)
tError("TCP:%s thread:%d BIG error, numOfFds:%d actual numOfFds:%d", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds, numOfFds);
pThreadObj++;
}
sprintf(msg, "TCP:%s total connections:%d\n", pServerObj->label, numOfConns);
return;
}
#endif
src/rpc/src/rpcUdp.c
浏览文件 @ 6a504d36
......@@ -32,7 +32,7 @@
int tsUdpDelay = 0;
typedef struct {
void * signature;
void *signature;
int index;
int fd;
uint16_t port; // peer port
......@@ -53,23 +53,23 @@ typedef struct {
int server;
char ip[16]; // local IP
uint16_t port; // local Port
void * shandle; // handle passed by upper layer during server initialization
void *shandle; // handle passed by upper layer during server initialization
int threads;
char label[12];
void * tmrCtrl;
void *tmrCtrl;
void *(*fp)(SRecvInfo *pPacket);
SUdpConn udpConn[];
} SUdpConnSet;
typedef struct {
void * signature;
void *signature;
uint32_t ip; // dest IP
uint16_t port; // dest Port
SUdpConn * pConn;
SUdpConn *pConn;
struct sockaddr_in destAdd;
void * msgHdr;
void *msgHdr;
int totalLen;
void * timer;
void *timer;
int emptyNum;
} SUdpBuf;
......@@ -78,9 +78,8 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port);
static void taosProcessUdpBufTimer(void *param, void *tmrId);
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
pthread_attr_t thAttr;
SUdpConn * pConn;
SUdpConnSet * pSet;
SUdpConn *pConn;
SUdpConnSet *pSet;
int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn);
pSet = (SUdpConnSet *)malloc((size_t)size);
......@@ -106,9 +105,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
}
}
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
uint16_t ownPort;
for (int i = 0; i < threads; ++i) {
pConn = pSet->udpConn + i;
......@@ -146,7 +142,12 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
pConn->tmrCtrl = pSet->tmrCtrl;
}
if (pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn) != 0) {
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
pthread_attr_destroy(&thAttr);
if (code != 0) {
tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno));
taosCloseSocket(pConn->fd);
taosCleanUpUdpConnection(pSet);
......@@ -156,7 +157,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
++pSet->threads;
}
pthread_attr_destroy(&thAttr);
tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads);
return pSet;
......@@ -164,7 +164,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
void taosCleanUpUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn * pConn;
SUdpConn *pConn;
if (pSet == NULL) return;
......@@ -205,10 +205,10 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por
}
static void *taosRecvUdpData(void *param) {
SUdpConn *pConn = param;
struct sockaddr_in sourceAdd;
int dataLen;
unsigned int addLen;
SUdpConn * pConn = (SUdpConn *)param;
uint16_t port;
int minSize = sizeof(SRpcHead);
SRecvInfo recvInfo;
......@@ -274,7 +274,7 @@ static void *taosRecvUdpData(void *param) {
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
SUdpConn *pConn = (SUdpConn *)chandle;
SUdpBuf * pBuf;
SUdpBuf *pBuf;
if (pConn == NULL || pConn->signature != pConn) return -1;
......
src/rpc/test/rclient.c
浏览文件 @ 6a504d36
......@@ -204,6 +204,8 @@ int main(int argc, char *argv[]) {
tPrint("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
tPrint("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
getchar();
taosCloseLogger();
return 0;
......
src/vnode/tsdb/src/tsdbMain.c → src/tsdb/src/tsdbMain.c
浏览文件 @ 6a504d36
......@@ -814,6 +814,13 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
return iters;
}
static void tsdbFreeMemTable(SMemTable *pMemTable) {
if (pMemTable) {
tSkipListDestroy(pMemTable->pData);
free(pMemTable);
}
}
// Commit to file
static void *tsdbCommitData(void *arg) {
// TODO
......@@ -860,7 +867,8 @@ static void *tsdbCommitData(void *arg) {
// TODO: free the skiplist
for (int i = 0; i < pCfg->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->imem) { // Here has memory leak
if (pTable && pTable->imem) {
tsdbFreeMemTable(pTable->imem);
pTable->imem = NULL;
}
}
......
src/vnode/tsdb/src/tsdbMeta.c → src/tsdb/src/tsdbMeta.c
浏览文件 @ 6a504d36
......@@ -34,7 +34,7 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) {
*contLen = tsdbEstimateTableEncodeSize(pTable);
if (*contLen < 0) return NULL;
void *ret = malloc(*contLen);
void *ret = calloc(1, *contLen);
if (ret == NULL) return NULL;
void *ptr = ret;
......
src/util/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -6,9 +6,11 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt)
TARGET_LINK_LIBRARIES(tutil pthread os m rt)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV)
......@@ -65,7 +67,7 @@ ELSEIF (TD_WINDOWS_64)
LIST(APPEND SRC ./src/tutil.c)
LIST(APPEND SRC ./src/version.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty iconv regex pthread os winmm IPHLPAPI ws2_32)
TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32)
ELSEIF(TD_DARWIN_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
LIST(APPEND SRC ./src/hash.c)
......@@ -102,7 +104,7 @@ ELSEIF(TD_DARWIN_64)
LIST(APPEND SRC ./src/version.c)
LIST(APPEND SRC ./src/hash.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty iconv pthread os)
TARGET_LINK_LIBRARIES(tutil iconv pthread os)
ENDIF()
#IF (TD_CLUSTER)
......
src/util/src/hash.c
浏览文件 @ 6a504d36
......@@ -298,7 +298,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) {
*/
static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize,
uint32_t hashVal) {
size_t totalSize = dataSize + sizeof(SHashNode) + keyLen;
size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null
SHashNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {
......
src/util/src/ihash.c
浏览文件 @ 6a504d36
......@@ -189,7 +189,6 @@ void taosCleanUpIntHash(void *handle) {
free(pObj->hashList);
}
memset(pObj, 0, sizeof(IHashObj));
free(pObj->lockedBy);
free(pObj);
}
......
src/util/src/tqueue.c
浏览文件 @ 6a504d36
......@@ -117,7 +117,7 @@ int taosWriteQitem(taos_queue param, int type, void *item) {
queue->numOfItems++;
if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1);
pTrace("item:%p is put into queue, type:%d items:%d", item, type, queue->numOfItems);
pTrace("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems);
pthread_mutex_unlock(&queue->mutex);
......@@ -298,13 +298,15 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
STaosQnode *pNode = NULL;
int code = 0;
for(int i=0; i<qset->numOfQueues; ++i) {
pthread_mutex_lock(&qset->mutex);
for(int i=0; i<qset->numOfQueues; ++i) {
//pthread_mutex_lock(&qset->mutex);
if (qset->current == NULL)
qset->current = qset->head;
STaosQueue *queue = qset->current;
if (queue) qset->current = queue->next;
pthread_mutex_unlock(&qset->mutex);
//pthread_mutex_unlock(&qset->mutex);
if (queue == NULL) break;
pthread_mutex_lock(&queue->mutex);
......@@ -326,6 +328,8 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
if (pNode) break;
}
pthread_mutex_unlock(&qset->mutex);
return code;
}
......@@ -335,13 +339,15 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) {
STaosQall *qall = (STaosQall *)p2;
int code = 0;
for(int i=0; i<qset->numOfQueues; ++i) {
pthread_mutex_lock(&qset->mutex);
for(int i=0; i<qset->numOfQueues; ++i) {
// pthread_mutex_lock(&qset->mutex);
if (qset->current == NULL)
qset->current = qset->head;
queue = qset->current;
if (queue) qset->current = queue->next;
pthread_mutex_unlock(&qset->mutex);
// pthread_mutex_unlock(&qset->mutex);
if (queue == NULL) break;
pthread_mutex_lock(&queue->mutex);
......@@ -365,6 +371,7 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) {
if (code != 0) break;
}
pthread_mutex_unlock(&qset->mutex);
return code;
}
......
src/util/src/tsched.c
浏览文件 @ 6a504d36
......@@ -40,66 +40,68 @@ static void *taosProcessSchedQueue(void *param);
static void taosDumpSchedulerStatus(void *qhandle, void *tmrId);
void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) {
pthread_attr_t attr;
SSchedQueue * pSched = (SSchedQueue *)malloc(sizeof(SSchedQueue));
SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1);
if (pSched == NULL) {
pError("%s: no enough memory for pSched, reason: %s", label, strerror(errno));
goto _error;
pError("%s: no enough memory for pSched", label);
return NULL;
}
pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize);
if (pSched->queue == NULL) {
pError("%s: no enough memory for queue", label);
taosCleanUpScheduler(pSched);
return NULL;
}
pSched->qthread = calloc(sizeof(pthread_t), numOfThreads);
if (pSched->qthread == NULL) {
pError("%s: no enough memory for qthread", label);
taosCleanUpScheduler(pSched);
return NULL;
}
memset(pSched, 0, sizeof(SSchedQueue));
pSched->queueSize = queueSize;
strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
pSched->label[sizeof(pSched->label)-1] = '\0';
pSched->fullSlot = 0;
pSched->emptySlot = 0;
if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) {
pError("init %s:queueMutex failed, reason:%s", pSched->label, strerror(errno));
goto _error;
pError("init %s:queueMutex failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) {
pError("init %s:empty semaphore failed, reason:%s", pSched->label, strerror(errno));
goto _error;
pError("init %s:empty semaphore failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
if (tsem_init(&pSched->fullSem, 0, 0) != 0) {
pError("init %s:full semaphore failed, reason:%s", pSched->label, strerror(errno));
goto _error;
}
if ((pSched->queue = (SSchedMsg *)malloc((size_t)pSched->queueSize * sizeof(SSchedMsg))) == NULL) {
pError("%s: no enough memory for queue, reason:%s", pSched->label, strerror(errno));
goto _error;
}
memset(pSched->queue, 0, (size_t)pSched->queueSize * sizeof(SSchedMsg));
pSched->fullSlot = 0;
pSched->emptySlot = 0;
pSched->qthread = malloc(sizeof(pthread_t) * (size_t)numOfThreads);
if (pSched->qthread == NULL) {
pError("%s: no enough memory for qthread, reason: %s", pSched->label, strerror(errno));
goto _error;
pError("init %s:full semaphore failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
for (int i = 0; i < numOfThreads; ++i) {
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for (int i = 0; i < numOfThreads; ++i) {
if (pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched) != 0) {
pError("%s: failed to create rpc thread, reason:%s", pSched->label, strerror(errno));
goto _error;
int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched);
pthread_attr_destroy(&attr);
if (code != 0) {
pError("%s: failed to create rpc thread(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
++pSched->numOfThreads;
}
pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads);
pTrace("%s scheduler is initialized, numOfThreads:%d", label, pSched->numOfThreads);
return (void *)pSched;
_error:
taosCleanUpScheduler(pSched);
return NULL;
}
void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) {
......@@ -124,21 +126,21 @@ void *taosProcessSchedQueue(void *param) {
pTrace("wait %s fullSem was interrupted", pSched->label);
continue;
}
pError("wait %s fullSem failed, errno:%d, reason:%s", pSched->label, errno, strerror(errno));
pError("wait %s fullSem failed(%s)", pSched->label, strerror(errno));
}
if (pthread_mutex_lock(&pSched->queueMutex) != 0)
pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno));
msg = pSched->queue[pSched->fullSlot];
memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg));
pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize;
if (pthread_mutex_unlock(&pSched->queueMutex) != 0)
pError("unlock %s queueMutex failed, reason:%s\n", pSched->label, strerror(errno));
pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno));
if (tsem_post(&pSched->emptySem) != 0)
pError("post %s emptySem failed, reason:%s\n", pSched->label, strerror(errno));
pError("post %s emptySem failed(%s)", pSched->label, strerror(errno));
if (msg.fp)
(*(msg.fp))(&msg);
......@@ -158,22 +160,23 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) {
while (tsem_wait(&pSched->emptySem) != 0) {
if (errno != EINTR) {
pError("wait %s emptySem failed, reason:%s", pSched->label, strerror(errno));
pError("wait %s emptySem failed(%s)", pSched->label, strerror(errno));
break;
}
pTrace("wait %s emptySem was interrupted", pSched->label);
}
if (pthread_mutex_lock(&pSched->queueMutex) != 0)
pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno));
pSched->queue[pSched->emptySlot] = *pMsg;
pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize;
if (pthread_mutex_unlock(&pSched->queueMutex) != 0)
pError("unlock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno));
if (tsem_post(&pSched->fullSem) != 0) pError("post %s fullSem failed, reason:%s", pSched->label, strerror(errno));
if (tsem_post(&pSched->fullSem) != 0)
pError("post %s fullSem failed(%s)", pSched->label, strerror(errno));
return 0;
}
......@@ -183,9 +186,11 @@ void taosCleanUpScheduler(void *param) {
if (pSched == NULL) return;
for (int i = 0; i < pSched->numOfThreads; ++i) {
if (pSched->qthread[i])
pthread_cancel(pSched->qthread[i]);
}
for (int i = 0; i < pSched->numOfThreads; ++i) {
if (pSched->qthread[i])
pthread_join(pSched->qthread[i], NULL);
}
......@@ -197,8 +202,8 @@ void taosCleanUpScheduler(void *param) {
taosTmrStopA(&pSched->pTimer);
}
free(pSched->queue);
free(pSched->qthread);
if (pSched->queue) free(pSched->queue);
if (pSched->qthread) free(pSched->qthread);
free(pSched); // fix memory leak
}
......
src/vnode/CMakeLists.txt
浏览文件 @ 6a504d36
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
ADD_SUBDIRECTORY(wal)
ADD_SUBDIRECTORY(tsdb)
ADD_SUBDIRECTORY(main)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(vnode ${SRC})
TARGET_LINK_LIBRARIES(vnode tsdb)
ENDIF ()
\ No newline at end of file
src/vnode/cache/inc/cache.h 已删除 100644 → 0
浏览文件 @ 53b8867a
#if !defined(_TD_CACHE_H_)
#define _TD_CACHE_H_
#define TD_MIN_CACHE_BLOCK_SIZE 1024*1024 /* 1M */
#define TD_MAX_CACHE_BLOCK_SIZE 64*1024*1024 /* 64M */
typedef void cache_pool_t;
typedef struct SCacheBlock
{
int32_t blockId;
char data[];
} SCacheBlock;
#endif // _TD_CACHE_H_
src/vnode/main/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ 53b8867a
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(vnode ${SRC})
TARGET_LINK_LIBRARIES(vnode tsdb)
ENDIF ()
\ No newline at end of file
src/vnode/main/src/vnodeMain.c → src/vnode/src/vnodeMain.c
浏览文件 @ 6a504d36
......@@ -160,7 +160,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
syncInfo.writeToCache = vnodeWriteToQueue;
syncInfo.confirmForward = dnodeSendRpcWriteRsp;
syncInfo.notifyRole = vnodeNotifyRole;
pVnode->sync = syncStart(&syncInfo);;
pVnode->sync = syncStart(&syncInfo);
pVnode->events = NULL;
pVnode->cq = NULL;
......@@ -224,10 +224,11 @@ void vnodeRelease(void *pVnodeRaw) {
// remove the whole directory
}
dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId);
free(pVnode);
int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1);
dTrace("pVnode:%p vgId:%d, vnode is released, vnodes:%d", pVnode, vgId, count);
if (count <= 0) {
taosCleanUpIntHash(tsDnodeVnodesHash);
vnodeModuleInit = PTHREAD_ONCE_INIT;
......@@ -368,6 +369,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
pVnode->walCfg.wals = (int8_t)wals;
pVnode->walCfg.keep = 0;
int32_t arbitratorIp = -1;
num = fscanf(fp, "%s %u", option[0], &arbitratorIp);
......
src/vnode/wal/src/walMain.c → src/wal/src/walMain.c
浏览文件 @ 6a504d36
......@@ -36,6 +36,7 @@
typedef struct {
int fd;
int keep;
int level;
int max; // maximum number of wal files
uint32_t id; // increase continuously
......@@ -61,6 +62,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
pWal->id = 0;
pWal->num = 0;
pWal->level = pCfg->commitLog;
pWal->keep = pCfg->keep;
strcpy(pWal->path, path);
pthread_mutex_init(&pWal->mutex, NULL);
......@@ -83,9 +85,9 @@ void walClose(void *handle) {
if (handle == NULL) return;
SWal *pWal = handle;
close(pWal->fd);
if (pWal->keep == 0) {
// remove all files in the directory
for (int i=0; i<pWal->num; ++i) {
sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
......@@ -95,6 +97,9 @@ void walClose(void *handle) {
wTrace("wal:%s, it is removed", pWal->name);
}
}
} else {
wTrace("wal:%s, it is closed and kept", pWal->name);
}
pthread_mutex_destroy(&pWal->mutex);
......
tests/pytest/insert/basic.py 0 → 100644
浏览文件 @ 6a504d36
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
tdSql.prepare()
tdSql.execute('show databases')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db')
tdSql.execute('use db')
tdSql.execute('create table tb (ts timestamp, speed int)')
insertRows = 10
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
tdSql.execute('insert into tb values (now + %dm, %d)' % (i, i))
# tdLog.info("insert earlier data")
# tdSql.execute('insert into tb values (now - 5m , 10)')
# tdSql.execute('insert into tb values (now - 6m , 10)')
# tdSql.execute('insert into tb values (now - 7m , 10)')
# tdSql.execute('insert into tb values (now - 8m , 10)')
# tdSql.query("select * from tb")
# tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
tests/pytest/simpletest.sh 0 → 100755
浏览文件 @ 6a504d36
sudo python3 ./test.py -f insert/basic.py
tests/pytest/test.py 0 → 100644
浏览文件 @ 6a504d36
#!/usr/bin/python
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# install pip
# pip install src/connector/python/linux/python2/
# -*- coding: utf-8 -*-
import sys
import getopt
from util.log import *
from util.dnodes import *
from util.cases import *
import taos
# add testcase here:
from insert.basic import *
if __name__ == "__main__":
fileName = "all"
deployPath = ""
masterIp = ""
testCluster = False
opts, args = getopt.getopt(sys.argv[1:], 'f:p:m:sch', [
'file=', 'path=', 'master', 'stop', 'cluster', 'help'])
for key, value in opts:
if key in ['-h', '--help']:
tdLog.printNoPrefix(
'A collection of test cases written using Python')
tdLog.printNoPrefix('-f Name of test case file written by Python')
tdLog.printNoPrefix('-p Deploy Path for Simulator')
tdLog.printNoPrefix('-m Master Ip for Simulator')
tdLog.printNoPrefix('-c Test Cluster Flag')
tdLog.printNoPrefix('-s stop All dnodes')
sys.exit(0)
if key in ['-f', '--file']:
fileName = value
if key in ['-p', '--path']:
deployPath = value
if key in ['-m', '--master']:
masterIp = value
if key in ['-c', '--cluster']:
testCluster = True
if key in ['-s', '--stop']:
cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -9 taosd"
os.system(cmd)
tdLog.exit('stop All dnodes')
if masterIp == "":
tdDnodes.init(deployPath)
if testCluster:
tdLog.notice("Procedures for testing cluster")
if fileName == "all":
tdCases.runAllCluster()
else:
tdCases.runOneCluster(fileName)
else:
tdLog.notice("Procedures for testing self-deployment")
tdDnodes.stopAll()
tdDnodes.deploy(1)
tdDnodes.start(1)
conn = taos.connect(
host='192.168.0.1',
config=tdDnodes.getSimCfgPath())
if fileName == "all":
tdCases.runAllLinux(conn)
else:
tdCases.runOneLinux(conn, fileName)
conn.close()
else:
tdLog.notice("Procedures for tdengine deployed in %s" % (masterIp))
conn = taos.connect(host=masterIp, config=tdDnodes.getSimCfgPath())
if fileName == "all":
tdCases.runAllWindows(conn)
else:
tdCases.runOneWindows(conn, fileName)
conn.close()
tests/pytest/util/cases.py 0 → 100644
浏览文件 @ 6a504d36
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import time
import datetime
from util.log import *
class TDCase:
def __init__(self, name, case):
self.name = name
self.case = case
class TDCases:
def __init__(self):
self.linuxCases = []
self.windowsCases = []
self.clusterCases = []
def addWindows(self, name, case):
self.windowsCases.append(TDCase(name, case))
def addLinux(self, name, case):
self.linuxCases.append(TDCase(name, case))
def addCluster(self, name, case):
self.clusterCases.append(TDCase(name, case))
def runAllLinux(self, conn):
tdLog.notice("run total %d cases" % (len(self.linuxCases)))
for case in self.linuxCases:
case.case.init(conn)
case.case.run()
case.case.stop()
tdLog.notice("total %d cases executed" % (len(self.linuxCases)))
def runOneLinux(self, conn, fileName):
tdLog.notice("run cases like %s" % (fileName))
runNum = 0
for case in self.linuxCases:
if case.name.find(fileName) != -1:
case.case.init(conn)
case.case.run()
case.case.stop()
time.sleep(5)
runNum += 1
tdLog.notice("total %d cases executed" % (runNum))
def runAllWindows(self, conn):
tdLog.notice("run total %d cases" % (len(self.windowsCases)))
for case in self.windowsCases:
case.case.init(conn)
case.case.run()
case.case.stop()
tdLog.notice("total %d cases executed" % (len(self.windowsCases)))
def runOneWindows(self, conn, fileName):
tdLog.notice("run cases like %s" % (fileName))
runNum = 0
for case in self.windowsCases:
if case.name.find(fileName) != -1:
case.case.init(conn)
case.case.run()
case.case.stop()
time.sleep(2)
runNum += 1
tdLog.notice("total %d cases executed" % (runNum))
def runAllCluster(self):
tdLog.notice("run total %d cases" % (len(self.clusterCases)))
for case in self.clusterCases:
case.case.init()
case.case.run()
case.case.stop()
tdLog.notice("total %d cases executed" % (len(self.clusterCases)))
def runOneCluster(self, fileName):
tdLog.notice("run cases like %s" % (fileName))
runNum = 0
for case in self.clusterCases:
if case.name.find(fileName) != -1:
case.case.init()
case.case.run()
case.case.stop()
time.sleep(2)
runNum += 1
tdLog.notice("total %d cases executed" % (runNum))
tdCases = TDCases()
tests/pytest/util/dnodes.py 0 → 100644
浏览文件 @ 6a504d36
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import os.path
from util.log import *
class TDSimClient:
def init(self, path):
self.path = path
def getCfgDir(self):
return self.cfgDir
def cfg(self, option, value):
cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def deploy(self):
self.logDir = "%s/sim/psim/log" % (self.path,)
self.cfgDir = "%s/sim/psim/cfg" % (self.path)
self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path)
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("logDir", self.logDir)
self.cfg("numOfLogLines", "100000000")
self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("locale", "en_US.UTF-8")
self.cfg("charset", "GBK")
self.cfg("asyncLog", "0")
self.cfg("anyIp", "0")
self.cfg("sdbDebugFlag", "135")
self.cfg("rpcDebugFlag", "135")
self.cfg("tmrDebugFlag", "131")
self.cfg("cDebugFlag", "135")
self.cfg("udebugFlag", "135")
self.cfg("jnidebugFlag", "135")
self.cfg("qdebugFlag", "135")
tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
class TDDnode:
def __init__(self, index):
self.index = index
self.running = 0
self.deployed = 0
def init(self, path):
self.path = path
def deploy(self):
self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index)
self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index)
self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index)
cmd = "rm -rf " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.startIP()
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("publicIp", "192.168.0.%d" % (self.index))
self.cfg("internalIp", "192.168.0.%d" % (self.index))
self.cfg("privateIp", "192.168.0.%d" % (self.index))
self.cfg("dataDir", self.dataDir)
self.cfg("logDir", self.logDir)
self.cfg("numOfLogLines", "100000000")
self.cfg("mgmtEqualVnodeNum", "0")
self.cfg("clog", "1")
self.cfg("statusInterval", "1")
self.cfg("numOfTotalVnodes", "64")
self.cfg("numOfMPeers", "3")
self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0")
self.cfg("maxVnodeConnections", "30000")
self.cfg("maxMgmtConnections", "30000")
self.cfg("maxMeterConnections", "30000")
self.cfg("maxShellConns", "30000")
self.cfg("locale", "en_US.UTF-8")
self.cfg("charset", "UTF-8")
self.cfg("asyncLog", "0")
self.cfg("anyIp", "0")
self.cfg("dDebugFlag", "135")
self.cfg("mDebugFlag", "135")
self.cfg("sdbDebugFlag", "135")
self.cfg("rpcDebugFlag", "135")
self.cfg("tmrDebugFlag", "131")
self.cfg("cDebugFlag", "135")
self.cfg("httpDebugFlag", "135")
self.cfg("monitorDebugFlag", "135")
self.cfg("udebugFlag", "135")
self.cfg("jnidebugFlag", "135")
self.cfg("qdebugFlag", "135")
self.deployed = 1
tdLog.debug(
"dnode:%d is deployed and configured by %s" %
(self.index, self.cfgPath))
def start(self):
binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/"
binPath = os.path.realpath(binPath)
binPath += "/build/bin/"
if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index))
cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % (
binPath, self.cfgDir)
print(cmd)
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
tdLog.debug("wait 2 seconds for the dnode:%d to start." % (self.index))
time.sleep(2)
def stop(self):
if self.running != 0:
cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigint taosd" % (
self.cfgDir)
if os.system(cmd) != 0:
tdLog.exit(cmd)
tdLog.debug("dnode:%d is stopped by kill -SIGINT" % (self.index))
tdLog.debug(
"wait 2 seconds for the dnode:%d to stop." %
(self.index))
time.sleep(2)
def forcestop(self):
if self.running != 0:
cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" % (
self.cfgDir)
if os.system(cmd) != 0:
tdLog.exit(cmd)
tdLog.debug("dnode:%d is stopped by kill -9" % (self.index))
tdLog.debug(
"wait 2 seconds for the dnode:%d to stop." %
(self.index))
time.sleep(2)
def startIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def stopIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def cfg(self, option, value):
cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def getDnodeRootDir(self, index):
dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index)
return dnodeRootDir
def getDnodesRootDir(self):
dnodesRootDir = "%s/sim/psim" % (self.path)
return dnodesRootDir
class TDDnodes:
def __init__(self):
self.dnodes = []
self.dnodes.append(TDDnode(1))
self.dnodes.append(TDDnode(2))
self.dnodes.append(TDDnode(3))
self.dnodes.append(TDDnode(4))
self.dnodes.append(TDDnode(5))
self.dnodes.append(TDDnode(6))
self.dnodes.append(TDDnode(7))
self.dnodes.append(TDDnode(8))
self.dnodes.append(TDDnode(9))
self.dnodes.append(TDDnode(10))
def init(self, path):
cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd"
os.system(cmd)
binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/"
tdLog.debug("binPath %s" % (binPath))
binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath))
# cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
# tdLog.debug(cmd)
# os.system(cmd)
# cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
# cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
if path == "":
# self.path = os.path.expanduser('~')
self.path = os.path.abspath(binPath + "../../")
else:
self.path = os.path.realpath(path)
for i in range(len(self.dnodes)):
self.dnodes[i].init(self.path)
self.sim = TDSimClient()
self.sim.init(self.path)
self.sim.deploy()
def deploy(self, index):
self.check(index)
self.dnodes[index - 1].deploy()
def cfg(self, index, option, value):
self.check(index)
self.dnodes[index - 1].cfg(option, value)
def start(self, index):
self.check(index)
self.dnodes[index - 1].start()
def stop(self, index):
self.check(index)
self.dnodes[index - 1].stop()
def forcestop(self, index):
self.check(index)
self.dnodes[index - 1].forcestop()
def startIP(self, index):
self.check(index)
self.dnodes[index - 1].startIP()
def stopIP(self, index):
self.check(index)
self.dnodes[index - 1].stopIP()
def check(self, index):
if index < 1 or index > 10:
tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
def stopAll(self):
tdLog.debug("stop all dnodes")
for i in range(len(self.dnodes)):
self.dnodes[i].stop()
cmd = "sudo systemctl stop taosd"
os.system(cmd)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
cmd = "ps -ef | grep -w taosd | grep 'dnode' | grep -v grep | awk '{print $2}' && sudo pkill -sigkill taosd"
os.system(cmd)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
def getDnodesRootDir(self):
dnodesRootDir = "%s/sim" % (self.path)
return dnodesRootDir
def getSimCfgPath(self):
return self.sim.getCfgDir()
tdDnodes = TDDnodes()
tests/pytest/util/log.py 0 → 100644
浏览文件 @ 6a504d36
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import time
import datetime
from distutils.log import warn as printf
class TDLog:
def __init__(self):
self.path = ""
def info(self, info):
printf("%s %s" % (datetime.datetime.now(), info))
def sleep(self, sec):
printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
time.sleep(sec)
def debug(self, err):
printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
def success(self, info):
printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
def notice(self, err):
printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err))
def exit(self, err):
printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err))
sys.exit(1)
def printfNoPrefix(self, info):
printf("\033[1;36m%s\033[0m" % (info))
tdLog = TDLog()
tests/pytest/util/sql.py 0 → 100644
浏览文件 @ 6a504d36
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import time
import datetime
from util.log import *
class TDSql:
def __init__(self):
self.queryRows = 0
self.queryCols = 0
self.affectedRows = 0
def init(self, cursor):
self.cursor = cursor
def close(self):
self.cursor.close()
def prepare(self):
tdLog.info("prepare database:db")
self.cursor.execute('reset query cache')
self.cursor.execute('drop database if exists db')
self.cursor.execute('create database db')
self.cursor.execute('use db')
def error(self, sql):
expectErrNotOccured = True
try:
self.cursor.execute(sql)
except BaseException:
expectErrNotOccured = False
if expectErrNotOccured:
tdLog.exit("sql:%.40s, expect error not occured" % (sql))
else:
tdLog.info("sql:%.40s, expect error occured" % (sql))
def query(self, sql):
self.sql = sql
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self.queryRows = len(self.queryResult)
self.queryCols = len(self.cursor.description)
# if self.queryRows == 1 and self.queryCols == 1:
# tdLog.info("sql:%s, rows:%d cols:%d data:%s" % (self.sql, self.queryRows, self.queryCols, self.queryResult[0][0]))
# else:
# tdLog.info("sql:%s, rows:%d cols:%d" % (self.sql, self.queryRows, self.queryCols))
return self.queryRows
def checkRows(self, expectRows):
if self.queryRows != expectRows:
tdLog.exit(
"sql:%.40s, queryRows:%d != expect:%d" %
(self.sql, self.queryRows, expectRows))
tdLog.info("sql:%.40s, queryRows:%d == expect:%d" %
(self.sql, self.queryRows, expectRows))
def checkData(self, row, col, data):
if row < 0:
tdLog.exit(
"sql:%.40s, row:%d is smaller than zero" %
(self.sql, row))
if col < 0:
tdLog.exit(
"sql:%.40s, col:%d is smaller than zero" %
(self.sql, col))
if row >= self.queryRows:
tdLog.exit(
"sql:%.40s, row:%d is larger than queryRows:%d" %
(self.sql, row, self.queryRows))
if col >= self.queryCols:
tdLog.exit(
"sql:%.40s, col:%d is larger than queryRows:%d" %
(self.sql, col, self.queryCols))
if self.queryResult[row][col] != data:
tdLog.exit(
"sql:%.40s row:%d col:%d data:%s != expect:%s" %
(self.sql, row, col, self.queryResult[row][col], data))
tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" %
(self.sql, row, col, self.queryResult[row][col], data))
def getData(self, row, col):
if row < 0:
tdLog.exit(
"sql:%.40s, row:%d is smaller than zero" %
(self.sql, row))
if col < 0:
tdLog.exit(
"sql:%.40s, col:%d is smaller than zero" %
(self.sql, col))
if row >= self.queryRows:
tdLog.exit(
"sql:%.40s, row:%d is larger than queryRows:%d" %
(self.sql, row, self.queryRows))
if col >= self.queryCols:
tdLog.exit(
"sql:%.40s, col:%d is larger than queryRows:%d" %
(self.sql, col, self.queryCols))
return self.queryResult[row][col]
def executeTimes(self, sql, times):
for i in range(times):
try:
return self.cursor.execute(sql)
except BaseException:
time.sleep(1)
continue
def execute(self, sql):
self.sql = sql
self.affectedRows = self.cursor.execute(sql)
return self.affectedRows
def checkAffectedRows(self, expectAffectedRows):
if self.affectedRows != expectAffectedRows:
tdLog.exit("sql:%.40s, affectedRows:%d != expect:%d" %
(self.sql, self.affectedRows, expectAffectedRows))
tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" %
(self.sql, self.affectedRows, expectAffectedRows))
tdSql = TDSql()
tests/script/basicSuite.sim
浏览文件 @ 6a504d36
......@@ -10,4 +10,6 @@ run general/db/basic3.sim
run general/db/basic4.sim
run general/db/basic5.sim
run general/user/basic1.sim
##################################
tests/script/tmp/dnode2.sim → tests/script/tmp/mnodes.sim
浏览文件 @ 6a504d36
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2
system sh/exec_up.sh -n dnode1 -s start
system sh/exec_up.sh -n dnode2 -s start
sql connect
\ No newline at end of file
system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3
system sh/cfg.sh -n dnode1 -c numOfMPeers -v 3
system sh/cfg.sh -n dnode2 -c numOfMPeers -v 3
system sh/cfg.sh -n dnode3 -c numOfMPeers -v 3
tests/test/c/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -6,6 +6,12 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
add_executable(benchmarkPerTable benchmarkPerTable.c)
target_link_libraries(benchmarkPerTable taos_static pthread)
add_executable(insertPerTable insertPerTable.c)
target_link_libraries(insertPerTable taos_static pthread)
add_executable(insertPerRow insertPerRow.c)
target_link_libraries(insertPerRow taos_static pthread)
add_executable(importOneRow importOneRow.c)
target_link_libraries(importOneRow taos_static pthread)
ENDIF()
tests/test/c/importOneRow.c 0 → 100644
浏览文件 @ 6a504d36
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "tlog.h"
#include "ttimer.h"
#include "tutil.h"
#define MAX_RANDOM_POINTS 20000
#define GREEN "\033[1;32m"
#define NC "\033[0m"
void taos_error(TAOS *taos);
void* taos_execute(void *param);
typedef struct {
pthread_t pid;
int index;
} ThreadObj;
int threadNum = 1;
int rowNum = 1000;
int replica = 1;
void printHelp() {
char indent[10] = " ";
printf("Used to test the performance of TDengine\n After writing one row of data to all tables, write the next row\n");
printf("%s%s\n", indent, "-r");
printf("%s%s%s%d\n", indent, indent, "Number of records to write table, default is ", rowNum);
printf("%s%s\n", indent, "-t");
printf("%s%s%s%d\n", indent, indent, "Number of threads to be used, default is ", threadNum);
printf("%s%s\n", indent, "-replica");
printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica);
exit(EXIT_SUCCESS);
}
void shellParseArgument(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
printHelp();
exit(0);
} else if (strcmp(argv[i], "-r") == 0) {
rowNum = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0) {
threadNum = atoi(argv[++i]);
} else if (strcmp(argv[i], "-replica") == 0) {
replica = atoi(argv[++i]);
} else {
}
}
dPrint("%s rowNum:%d %s", GREEN, rowNum, NC);
dPrint("%s threadNum:%d %s", GREEN, threadNum, NC);
dPrint("%s replica:%d %s", GREEN, replica, NC);
}
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
taos_init();
ThreadObj *threads = calloc(threadNum, sizeof(ThreadObj));
for (int i = 0; i < threadNum; ++i) {
ThreadObj * pthread = threads + i;
pthread_attr_t thattr;
pthread->index = i;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pthread->pid, &thattr, taos_execute, pthread);
}
for (int i = 0; i < threadNum; i++) {
pthread_join(threads[i].pid, NULL);
}
printf("all finished\n");
return 0;
}
void taos_error(TAOS *con) {
fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
taos_close(con);
exit(1);
}
void* taos_execute(void *param) {
ThreadObj *pThread = (ThreadObj *)param;
void *taos = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
if (taos == NULL) taos_error(taos);
char sql[1024] = {0};
sprintf(sql, "create database if not exists db replica %d", replica);
taos_query(taos, sql);
sprintf(sql, "create table if not exists db.t%d (ts timestamp, i int, j float, k double)", pThread->index);
taos_query(taos, sql);
int64_t timestamp = 1530374400000L;
sprintf(sql, "insert into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, 0, 0, 0);
int code = taos_query(taos, sql);
if (code != 0) printf("error code:%d, sql:%s\n", code, sql);
int affectrows = taos_affected_rows(taos);
if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql);
timestamp -= 1000;
int total_affect_rows = affectrows;
for (int i = 1; i < rowNum; ++i) {
sprintf(sql, "import into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, i, i, i);
code = taos_query(taos, sql);
if (code != 0) printf("error code:%d, sql:%s\n", code, sql);
int affectrows = taos_affected_rows(taos);
if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql);
total_affect_rows += affectrows;
timestamp -= 1000;
}
printf("thread:%d run finished total_affect_rows:%d\n", pThread->index, total_affect_rows);
return NULL;
}
tests/test/c/insertPerRow.c 0 → 100644
浏览文件 @ 6a504d36
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "tlog.h"
#include "ttimer.h"
#include "tutil.h"
#define MAX_RANDOM_POINTS 20000
#define GREEN "\033[1;32m"
#define NC "\033[0m"
typedef struct {
int64_t rowsPerTable;
int64_t pointsPerTable;
int64_t tableBeginIndex;
int64_t tableEndIndex;
int threadIndex;
char dbName[32];
char stableName[64];
pthread_t thread;
} SInfo;
void *syncTest(void *param);
void generateRandomPoints();
void shellParseArgument(int argc, char *argv[]);
void createDbAndTable();
void insertData();
int32_t randomData[MAX_RANDOM_POINTS];
int64_t rowsPerTable = 10000;
int64_t pointsPerTable = 1;
int64_t numOfThreads = 1;
int64_t numOfTablesPerThread = 200;
char dbName[32] = "db";
char stableName[64] = "st";
int32_t cache = 16384;
int32_t tables = 1000;
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
generateRandomPoints();
taos_init();
createDbAndTable();
insertData();
}
void createDbAndTable() {
dPrint("start to create table");
TAOS * con;
struct timeval systemTime;
int64_t st, et;
char qstr[64000];
con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
if (con == NULL) {
dError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(1);
}
sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
if (taos_query(con, qstr)) {
dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
exit(0);
}
sprintf(qstr, "use %s", dbName);
if (taos_query(con, qstr)) {
dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t totalTables = numOfTablesPerThread * numOfThreads;
if (strcmp(stableName, "no") != 0) {
int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
for (int64_t f = 0; f < pointsPerTable; ++f) {
len += sprintf(qstr + len, ", f%ld double", f);
}
sprintf(qstr + len, ") tags(t int)");
if (taos_query(con, qstr)) {
dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
for (int64_t t = 0; t < totalTables; ++t) {
sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
if (taos_query(con, qstr)) {
dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
}
} else {
for (int64_t t = 0; t < totalTables; ++t) {
int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
for (int64_t f = 0; f < pointsPerTable; ++f) {
len += sprintf(qstr + len, ", f%ld double", f);
}
sprintf(qstr + len, ")");
if (taos_query(con, qstr)) {
dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
}
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
}
void insertData() {
struct timeval systemTime;
int64_t st, et;
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
dPrint("%d threads are spawned to insert data", numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads);
// Start threads to write
for (int i = 0; i < numOfThreads; ++i) {
pInfo[i].rowsPerTable = rowsPerTable;
pInfo[i].pointsPerTable = pointsPerTable;
pInfo[i].tableBeginIndex = i * numOfTablesPerThread;
pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread;
pInfo[i].threadIndex = i;
strcpy(pInfo[i].dbName, dbName);
strcpy(pInfo[i].stableName, stableName);
pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i));
}
taosMsleep(300);
for (int i = 0; i < numOfThreads; i++) {
pthread_join(pInfo[i].thread, NULL);
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
double seconds = (et - st) / 1000.0 / 1000.0;
int64_t totalTables = numOfTablesPerThread * numOfThreads;
int64_t totalRows = totalTables * rowsPerTable;
int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable;
double speedOfRows = totalRows / seconds;
double speedOfPoints = totalPoints / seconds;
dPrint(
"%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf "
"PointsPerSecond:%.1lf%s",
GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC);
dPrint("threads exit");
pthread_attr_destroy(&thattr);
free(pInfo);
}
void *syncTest(void *param) {
TAOS * con;
SInfo * pInfo = (SInfo *)param;
struct timeval systemTime;
int64_t st, et;
char qstr[65000];
int maxBytes = 60000;
dPrint("thread:%d, start to run", pInfo->threadIndex);
con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
if (con == NULL) {
dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
exit(1);
}
sprintf(qstr, "use %s", pInfo->dbName);
taos_query(con, qstr);
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t start = 1430000000000;
int64_t interval = 1000; // 1000 ms
char *sql = qstr;
char inserStr[] = "insert into";
int len = sprintf(sql, "%s", inserStr);
for (int64_t row = 0; row < pInfo->rowsPerTable; row++) {
for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) {
len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
len += sprintf(sql + len, "(%ld", start + row * interval);
for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) {
len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]);
// len += sprintf(sql + len, ",%ld", row);
}
len += sprintf(sql + len, ")");
if (len > maxBytes) {
if (taos_query(con, qstr)) {
dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
table, row, taos_errstr(con));
}
// "insert into"
len = sprintf(sql, "%s", inserStr);
}
}
}
if (len != strlen(inserStr)) {
taos_query(con, qstr);
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
int64_t totalRows = totalTables * pInfo->rowsPerTable;
int64_t totalPoints = totalRows * pInfo->pointsPerTable;
dPrint("thread:%d, insert finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex,
(et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints);
return NULL;
}
void generateRandomPoints() {
for (int r = 0; r < MAX_RANDOM_POINTS; ++r) {
randomData[r] = rand() % 1000;
}
}
void printHelp() {
char indent[10] = " ";
printf("Used to test the performance of TDengine\n After writing one row of data to all tables, write the next row\n");
printf("%s%s\n", indent, "-d");
printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
printf("%s%s\n", indent, "-s");
printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table");
printf("%s%s\n", indent, "-c");
printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
printf("%s%s\n", indent, "-r");
printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable);
printf("%s%s\n", indent, "-p");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable);
printf("%s%s\n", indent, "-t");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads);
printf("%s%s\n", indent, "-n");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread);
printf("%s%s\n", indent, "-tables");
printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables);
printf("%s%s\n", indent, "-cache");
printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache);
exit(EXIT_SUCCESS);
}
void shellParseArgument(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
printHelp();
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
strcpy(dbName, argv[++i]);
} else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-s") == 0) {
strcpy(stableName, argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
rowsPerTable = atoi(argv[++i]);
} else if (strcmp(argv[i], "-p") == 0) {
pointsPerTable = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0) {
numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-n") == 0) {
numOfTablesPerThread = atoi(argv[++i]);
} else if (strcmp(argv[i], "-tables") == 0) {
tables = atoi(argv[++i]);
} else if (strcmp(argv[i], "-cache") == 0) {
cache = atoi(argv[++i]);
} else {
}
}
dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC);
dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
dPrint("%sdbName:%s%s", GREEN, dbName, NC);
dPrint("%stableName:%s%s", GREEN, stableName, NC);
dPrint("%sstart to run%s", GREEN, NC);
}
tests/test/c/benchmarkPerTable.c → tests/test/c/insertPerTable.c
浏览文件 @ 6a504d36
......@@ -258,7 +258,7 @@ void generateRandomPoints() {
void printHelp() {
char indent[10] = " ";
printf("Used to test the performance of TDengine, the insert method is table-by-table\n");
printf("Used to test the performance of TDengine\n After writing all the data in one table, start the next table\n");
printf("%s%s\n", indent, "-d");
printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
......@@ -289,6 +289,8 @@ void shellParseArgument(int argc, char *argv[]) {
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
strcpy(dbName, argv[++i]);
} else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-s") == 0) {
strcpy(stableName, argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
......
tests/tsim/CMakeLists.txt
浏览文件 @ 6a504d36
......@@ -3,6 +3,7 @@ PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
......@@ -12,4 +13,4 @@ ENDIF ()
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(tsim ${SRC})
TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread )
TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson)
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或