From 9570328908be602153fb1fe45d76db6b6a97fe06 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 16 May 2020 11:03:27 +0800 Subject: [PATCH] TD-333: boundary check --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/inc/taosdef.h | 10 +- src/rpc/src/rpcMain.c | 8 +- tests/pytest/table/boundary.py | 161 +++++++++++++++++++++++ tests/pytest/table/tablename-boundary.py | 46 ------- 6 files changed, 173 insertions(+), 56 deletions(-) create mode 100644 tests/pytest/table/boundary.py delete mode 100644 tests/pytest/table/tablename-boundary.py diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 1654f76f1c..dbbec01329 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -43,7 +43,7 @@ extern "C" { typedef struct SParsedColElem { int16_t colIndex; - int16_t offset; + uint16_t offset; } SParsedColElem; typedef struct SParsedDataColInfo { diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index a7eec31388..e6a37a2745 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -49,7 +49,7 @@ typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; int16_t numOfColumns; - int16_t rowSize; + int32_t rowSize; } STableComInfo; typedef struct STableMeta { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5dad1bdb53..949f60c37b 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -193,20 +193,20 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_ACCT_LEN TSDB_UNI_LEN #define TSDB_PASSWORD_LEN TSDB_UNI_LEN -#define TSDB_MAX_COLUMNS 256 +#define TSDB_MAX_COLUMNS 1024 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 16 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 6mb -#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 16 -#define TSDB_MAX_TAGS_LEN 512 -#define TSDB_MAX_TAGS 32 +#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_TAGS_LEN 65536 +#define TSDB_MAX_TAGS 128 #define TSDB_AUTH_LEN 16 #define TSDB_KEY_LEN 16 diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7e80f0d282..ff96306bea 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -361,9 +361,10 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; - if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH || - type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || - type == TSDB_MSG_TYPE_CM_SHOW ) + if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP + || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META + || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; rpcSendReqToServer(pRpc, pContext); @@ -1109,6 +1110,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } + tTrace("connection type is: %d", pConn->connType); writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); if (writtenLen != msgLen) { diff --git a/tests/pytest/table/boundary.py b/tests/pytest/table/boundary.py new file mode 100644 index 0000000000..faa222231b --- /dev/null +++ b/tests/pytest/table/boundary.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +import random +import string +import subprocess +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init( self, conn ): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + + def getLimitFromSourceCode( self, name ): + cmd = "grep -w '#define %s' ../../src/inc/taosdef.h|awk '{print $3}'" % name + return int(subprocess.check_output(cmd, shell=True)) + + + def generateString( self, length ): + chars = string.ascii_uppercase + string.ascii_lowercase + v = "" + for i in range( length ): + v += random.choice( chars ) + return v + + + def checkTagBoundaries( self ): + tdLog.debug( "checking tag boundaries" ) + tdSql.prepare() + + maxTags = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS' ) + totalTagsLen = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS_LEN' ) + tdLog.notice( "max tags is %d" % maxTags ) + tdLog.notice( "max total tag length is %d" % totalTagsLen ) + + # for binary tags, 2 bytes are used for length + tagLen = (totalTagsLen - maxTags * 2) // maxTags + firstTagLen = totalTagsLen - 2 * maxTags - tagLen * (maxTags - 1) + + sql = "create table cars(ts timestamp, f int) tags(t0 binary(%d)" % firstTagLen + for i in range( 1, maxTags ): + sql += ", t%d binary(%d)" % (i, tagLen) + sql += ");" + + tdLog.debug( "creating super table: " + sql ) + tdSql.execute( sql ) + tdSql.query( 'show stables' ) + tdSql.checkRows( 1 ) + + for i in range( 10 ): + sql = "create table car%d using cars tags('%d'" % (i, i) + sql += ", '0'" * (maxTags - 1) + ");" + tdLog.debug( "creating table: " + sql ) + tdSql.execute( sql ) + + sql = "insert into car%d values(now, 0);" % i + tdLog.debug( "inserting data: " + sql ) + tdSql.execute( sql ) + + tdSql.query( 'show tables' ) + tdLog.info( 'tdSql.checkRow(10)' ) + tdSql.checkRows( 10 ) + + tdSql.query( 'select * from cars;' ) + tdSql.checkRows( 10 ) + + + def checkColumnBoundaries( self ): + tdLog.debug( "checking column boundaries" ) + tdSql.prepare() + + # one column is for timestamp + maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1 + + sql = "create table cars (ts timestamp" + for i in range( maxCols ): + sql += ", c%d int" % i + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + sql = "insert into cars values (now" + for i in range( maxCols ): + sql += ", %d" % i + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'select * from cars' ) + tdSql.checkRows( 1 ) + + + def checkTableNameBoundaries( self ): + tdLog.debug( "checking table name boundaries" ) + tdSql.prepare() + + maxTableNameLen = self.getLimitFromSourceCode( 'TSDB_TABLE_NAME_LEN' ) + tdLog.notice( "table name max length is %d" % maxTableNameLen ) + + name = self.generateString( maxTableNameLen - 1) + tdLog.info( "table name is '%s'" % name ) + + tdSql.execute( "create table %s (ts timestamp, value int)" % name ) + tdSql.execute( "insert into %s values(now, 0)" % name ) + + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + tdSql.query( 'select * from %s' % name ) + tdSql.checkRows( 1 ) + + + def checkRowBoundaries( self ): + tdLog.debug( "checking row boundaries" ) + tdSql.prepare() + + # 8 bytes for timestamp + maxRowSize = 65536 - 8 + maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1 + + # for binary cols, 2 bytes are used for length + colLen = (maxRowSize - maxCols * 2) // maxCols + firstColLen = maxRowSize - 2 * maxCols - colLen * (maxCols - 1) + + sql = "create table cars (ts timestamp, c0 binary(%d)" % firstColLen + for i in range( 1, maxCols ): + sql += ", c%d binary(%d)" % (i, colLen) + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + col = self.generateString( firstColLen ) + sql = "insert into cars values (now, '%s'" % col + col = self.generateString( colLen ) + for i in range( 1, maxCols ): + sql += ", '%s'" % col + sql += ");" + tdLog.info( sql ); + tdSql.execute( sql ) + tdSql.query( "select * from cars" ) + tdSql.checkRows( 1 ) + + + def run(self): + self.checkTagBoundaries() + self.checkColumnBoundaries() + self.checkTableNameBoundaries() + self.checkRowBoundaries() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/tablename-boundary.py b/tests/pytest/table/tablename-boundary.py deleted file mode 100644 index fba1b2cf3c..0000000000 --- a/tests/pytest/table/tablename-boundary.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import string -import random -import subprocess -from util.log import * -from util.cases import * -from util.sql import * - - -class TDTestCase: - def init(self, conn): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) - - def run(self): - tdSql.prepare() - - getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" - tableNameMaxLen = int( - subprocess.check_output( - getTableNameLen, shell=True)) - tdLog.notice("table name max length is %d" % tableNameMaxLen) - chars = string.ascii_uppercase + string.ascii_lowercase - tb_name = ''.join(random.choices(chars, k=tableNameMaxLen)) - tdLog.info('tb_name length %d' % len(tb_name)) - tdLog.info('create table %s (ts timestamp, value int)' % tb_name) - tdSql.error( - 'create table %s (ts timestamp, speed binary(4089))' % - tb_name) - - tb_name = ''.join(random.choices(chars, k=191)) - tdLog.info('tb_name length %d' % len(tb_name)) - tdLog.info('create table %s (ts timestamp, value int)' % tb_name) - tdSql.execute( - 'create table %s (ts timestamp, speed binary(4089))' % - tb_name) - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) -- GitLab