提交 3c244a4a 编写于 作者: S Shengliang Guan

Merge branch 'develop' into feature/os

......@@ -23,14 +23,18 @@
客户端遇到链接故障,请按照下面的步骤进行检查:
1. 确保客户端与服务端版本号是完全一致的,开源社区版和企业版也不能混用
2. 在服务器,执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行,启动*taosd*
3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)
4. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,或客户端所在计算机的系统hosts文件
5. 检查防火墙设置,确认TCP/UDP 端口6030-6039 是打开的
6. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*
7. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
8. 如果仍不能排除连接故障,请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
1. 检查网络环境
* 云服务器:检查云服务器的安全组是否打开TCP/UDP 端口6030-6039的访问权限
* 本地虚拟机:检查网络能否ping通,尽量避免使用`localhost` 作为hostname
* 公司服务器:如果为NAT网络环境,请务必检查服务器能否将消息返回值客户端
2. 确保客户端与服务端版本号是完全一致的,开源社区版和企业版也不能混用
3. 在服务器,执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行,启动*taosd*
4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)
5. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,或客户端所在计算机的系统hosts文件
6. 检查防火墙设置,确认TCP/UDP 端口6030-6039 是打开的
7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*
8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
9. 如果仍不能排除连接故障,请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
检查UDP端口连接是否工作:`nc -vuz {hostIP} {port} `
检查服务器侧TCP端口连接是否工作:`nc -l {port}`
检查客户端侧TCP端口链接是否工作:`nc {hostIP} {port}`
......
......@@ -377,6 +377,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
}
}
if (pe->num == 0) {
assert(pe->next == NULL);
} else {
assert(pe->next != NULL);
}
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
}
......@@ -390,22 +396,8 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
if (pRes != NULL) {
atomic_sub_fetch_64(&pHashObj->size, 1);
FREE_HASH_NODE(pHashObj, pRes);
if (pe->num == 0) {
assert(pe->next == NULL);
} else {
assert(pe->next != NULL);
}
return 0;
} else {
if (pe->num == 0) {
assert(pe->next == NULL);
} else {
assert(pe->next != NULL);
}
return -1;
}
}
......
......@@ -161,6 +161,21 @@ class WorkerThread:
logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
break
# Before we fetch the task and run it, let's ensure we properly "use" the database
try:
if (gConfig.per_thread_db_connection): # most likely TRUE
if not self._dbConn.isOpen: # might have been closed during server auto-restart
self._dbConn.open()
self.useDb() # might encounter exceptions. TODO: catch
except taos.error.ProgrammingError as err:
errno = Helper.convertErrno(err.errno)
if errno in [0x383, 0x386, 0x00B, 0x014] : # invalid database, dropping, Unable to establish connection, Database not ready
# ignore
dummy = 0
else:
print("\nCaught programming error. errno=0x{:X}, msg={} ".format(errno, err.msg))
raise
# Fetch a task from the Thread Coordinator
logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid))
task = tc.fetchTask()
......@@ -324,10 +339,12 @@ class ThreadCoordinator:
logger.debug("[STT] transition ended")
# Due to limitation (or maybe not) of the Python library,
# we cannot share connections across threads
if sm.hasDatabase():
for t in self._pool.threadList:
logger.debug("[DB] use db for all worker threads")
t.useDb()
# Here we are in main thread, we cannot operate the connections created in workers
# Moving below to task loop
# if sm.hasDatabase():
# for t in self._pool.threadList:
# logger.debug("[DB] use db for all worker threads")
# t.useDb()
# t.execSql("use db") # main thread executing "use
# db" on behalf of every worker thread
except taos.error.ProgrammingError as err:
......@@ -387,7 +404,7 @@ class ThreadCoordinator:
transitionFailed = self._doTransition() # To start, we end step -1 first
except taos.error.ProgrammingError as err:
transitionFailed = True
errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme
errno2 = Helper.convertErrno(err.errno) # correct error scheme
errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err)
logger.info(errMsg)
self._execStats.registerFailure(errMsg)
......@@ -468,6 +485,10 @@ class ThreadCoordinator:
# We define a class to run a number of threads in locking steps.
class Helper:
@classmethod
def convertErrno(cls, errno):
return errno if (errno > 0) else 0x80000000 + errno
class ThreadPool:
def __init__(self, numThreads, maxSteps):
......@@ -613,8 +634,7 @@ class DbConn:
def resetDb(self): # reset the whole database, etc.
if (not self.isOpen):
raise RuntimeError(
"Cannot reset database until connection is open")
raise RuntimeError("Cannot reset database until connection is open")
# self._tdSql.prepare() # Recreate database, etc.
self.execute('drop database if exists db')
......@@ -681,8 +701,7 @@ class DbConnRest(DbConn):
def close(self):
if (not self.isOpen):
raise RuntimeError(
"Cannot clean up database until connection is open")
raise RuntimeError("Cannot clean up database until connection is open")
# Do nothing for REST
logger.debug("[DB] REST Database connection closed")
self.isOpen = False
......@@ -747,27 +766,32 @@ class DbConnRest(DbConn):
class MyTDSql:
def __init__(self):
def __init__(self, hostAddr, cfgPath):
# Make the DB connection
self._conn = taos.connect(host=hostAddr, config=cfgPath)
self._cursor = self._conn.cursor()
self.queryRows = 0
self.queryCols = 0
self.affectedRows = 0
def init(self, cursor, log=True):
self.cursor = cursor
# def init(self, cursor, log=True):
# self.cursor = cursor
# if (log):
# caller = inspect.getframeinfo(inspect.stack()[1][0])
# self.cursor.log(caller.filename + ".sql")
def close(self):
self.cursor.close()
self._conn.close() # TODO: very important, cursor close does NOT close DB connection!
self._cursor.close()
def query(self, sql):
self.sql = sql
try:
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self._cursor.execute(sql)
self.queryResult = self._cursor.fetchall()
self.queryRows = len(self.queryResult)
self.queryCols = len(self.cursor.description)
self.queryCols = len(self._cursor.description)
except Exception as e:
# caller = inspect.getframeinfo(inspect.stack()[1][0])
# args = (caller.filename, caller.lineno, sql, repr(e))
......@@ -778,7 +802,7 @@ class MyTDSql:
def execute(self, sql):
self.sql = sql
try:
self.affectedRows = self.cursor.execute(sql)
self.affectedRows = self._cursor.execute(sql)
except Exception as e:
# caller = inspect.getframeinfo(inspect.stack()[1][0])
# args = (caller.filename, caller.lineno, sql, repr(e))
......@@ -791,13 +815,13 @@ class DbConnNative(DbConn):
# Class variables
_lock = threading.Lock()
_connInfoDisplayed = False
totalConnections = 0 # Not private
def __init__(self):
super().__init__()
self._type = self.TYPE_NATIVE
self._conn = None
self._cursor = None
# self._cursor = None
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
......@@ -814,7 +838,8 @@ class DbConnNative(DbConn):
buildPath = root[:len(root) - len("/build/bin")]
break
if buildPath == None:
raise RuntimeError("Failed to determine buildPath, selfPath={}".format(selfPath))
raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}"
.format(selfPath, projPath))
return buildPath
......@@ -822,33 +847,40 @@ class DbConnNative(DbConn):
cfgPath = self.getBuildPath() + "/test/cfg"
hostAddr = "127.0.0.1"
with self._lock: # force single threading for opening DB connections
if not self._connInfoDisplayed:
self.__class__._connInfoDisplayed = True # updating CLASS variable
logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable
self._cursor = self._conn.cursor()
cls = self.__class__ # Get the class, to access class variables
with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!!
if not cls._connInfoDisplayed:
cls._connInfoDisplayed = True # updating CLASS variable
logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
# Make the connection
# self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable
# self._cursor = self._conn.cursor()
# Record the count in the class
self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection
cls.totalConnections += 1
self._cursor.execute('reset query cache')
self._tdSql.execute('reset query cache')
# self._cursor.execute('use db') # do this at the beginning of every
# Open connection
self._tdSql = MyTDSql()
self._tdSql.init(self._cursor)
# self._tdSql = MyTDSql()
# self._tdSql.init(self._cursor)
def close(self):
if (not self.isOpen):
raise RuntimeError(
"Cannot clean up database until connection is open")
raise RuntimeError("Cannot clean up database until connection is open")
self._tdSql.close()
# Decrement the class wide counter
cls = self.__class__ # Get the class, to access class variables
with cls._lock:
cls.totalConnections -= 1
logger.debug("[DB] Database connection closed")
self.isOpen = False
def execute(self, sql):
if (not self.isOpen):
raise RuntimeError(
"Cannot execute database commands until connection is open")
raise RuntimeError("Cannot execute database commands until connection is open")
logger.debug("[SQL] Executing SQL: {}".format(sql))
self._lastSql = sql
nRows = self._tdSql.execute(sql)
......@@ -1528,7 +1560,7 @@ class Task():
try:
self._executeInternal(te, wt) # TODO: no return value?
except taos.error.ProgrammingError as err:
errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme
errno2 = Helper.convertErrno(err.errno)
if (gConfig.continue_on_exception): # user choose to continue
self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
errno2, err, wt.getDbConn().getLastSql()))
......@@ -1678,9 +1710,8 @@ class ExecutionStats:
logger.info(
"| Total Elapsed Time (from wall clock): {:.3f} seconds".format(
self._elapsedTime))
logger.info(
"| Top numbers written: {}".format(
TaskExecutor.getBoundedList()))
logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList()))
logger.info("| Total Number of Active DB Native Connections: {}".format(DbConnNative.totalConnections))
logger.info(
"----------------------------------------------------------------------")
......@@ -1789,7 +1820,7 @@ class TdSuperTable:
try:
dbc.query("select TBNAME from db.{}".format(self._stName)) # TODO: analyze result set later
except taos.error.ProgrammingError as err:
errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
errno2 = Helper.convertErrno(err.errno)
logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
raise
......@@ -1891,7 +1922,7 @@ class TaskReadData(StateTransitionTask):
if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?!
dbc.execute("select {} from db.{}".format(aggExpr, sTable.getName()))
except taos.error.ProgrammingError as err:
errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
errno2 = Helper.convertErrno(err.errno)
logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql()))
raise
......@@ -1920,9 +1951,8 @@ class TaskDropSuperTable(StateTransitionTask):
self.execWtSql(wt, "drop table {}".format(
regTableName)) # nRows always 0, like MySQL
except taos.error.ProgrammingError as err:
# correcting for strange error number scheme
errno2 = err.errno if (
err.errno > 0) else 0x80000000 + err.errno
# correcting for strange error number scheme
errno2 = Helper.convertErrno(err.errno)
if (errno2 in [0x362]): # mnode invalid table name
isSuccess = False
logger.debug(
......@@ -2429,7 +2459,11 @@ class ServiceManagerThread:
for line in iter(out.readline, b''):
# print("Finished reading a line: {}".format(line))
# print("Adding item to queue...")
line = line.decode("utf-8").rstrip()
try:
line = line.decode("utf-8").rstrip()
except UnicodeError:
print("\nNon-UTF8 server output: {}\n".format(line))
# This might block, and then causing "out" buffer to block
queue.put(line)
self._printProgress("_i")
......@@ -2455,7 +2489,7 @@ class ServiceManagerThread:
def svcErrorReader(self, err: IO, queue):
for line in iter(err.readline, b''):
print("\nTD Svc STDERR: {}".format(line))
print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line))
class TdeSubProcess:
......
......@@ -35,6 +35,8 @@ class TDTestCase:
tdSql.checkRows(2)
tdSql.checkData(1, 1, '涛思数据')
tdSql.error("insert into tb values (now, 'taosdata001')")
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
......@@ -575,6 +575,20 @@ class TDTestCase:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev bigint)")
tdSql.execute(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 63) - 1))
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 63) + 1))
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
......@@ -579,6 +579,20 @@ class TDTestCase:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev binary(5))")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags("dev_001")')
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags("dev")')
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
......@@ -575,6 +575,23 @@ class TDTestCase:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev float)")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%f)' % -3.4E38 - 1)
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38 + 1)
tdSql.execute(
'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38)
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags(%f)' % -3.4E38)
tdSql.query("show tables")
tdSql.checkRows(2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
......@@ -574,6 +574,24 @@ class TDTestCase:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev int)")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 31))
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 31)))
tdSql.execute(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 31) - 1))
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 31) + 1))
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(5))")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags("dev_001")')
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags("dev")')
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
......@@ -574,6 +574,23 @@ class TDTestCase:
# TSIM:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev smallint)")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 15))
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 15)))
tdSql.execute(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 15) - 1))
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 15) + 1))
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(2)
def stop(self):
tdSql.close()
......
......@@ -575,6 +575,24 @@ class TDTestCase:
# TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
# convert end
tdSql.execute("create database db")
tdSql.execute("use db")
tdSql.execute(
"create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint)")
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 7))
tdSql.error(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 7)))
tdSql.execute(
'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 7) - 1))
tdSql.execute(
'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 7) + 1))
print("==============step2")
tdSql.query("show tables")
tdSql.checkRows(2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
system sh/cfg.sh -n dnode3 -c http -v 1
system sh/cfg.sh -n dnode4 -c http -v 1
system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030
system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030
system sh/cfg.sh -n dnode1 -c serverPort -v 6030
system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1
#system sh/exec.sh -n dnode1 -s start
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册