Merge pull request #1932 from taosdata/feature/boundary-check

TD-333: boundary check

src/client/inc/tscUtil.h

@@ -43,7 +43,7 @@ extern "C" {
 
 typedef struct SParsedColElem {
   int16_t colIndex;
-  int16_t offset;
+  uint16_t offset;
 } SParsedColElem;
 
 typedef struct SParsedDataColInfo {

src/client/inc/tsclient.h

@@ -49,7 +49,7 @@ typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
   int16_t numOfColumns;
-  int16_t rowSize;
+  int32_t rowSize;
 } STableComInfo;
 
 typedef struct STableMeta {

src/inc/taosdef.h

@@ -193,20 +193,20 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_ACCT_LEN             TSDB_UNI_LEN
 #define TSDB_PASSWORD_LEN         TSDB_UNI_LEN
 
-#define TSDB_MAX_COLUMNS          256
+#define TSDB_MAX_COLUMNS          1024
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       192
 #define TSDB_DB_NAME_LEN          32
 #define TSDB_COL_NAME_LEN         64
-#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 16
+#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 6mb
 
-#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 16
-#define TSDB_MAX_TAGS_LEN         512
-#define TSDB_MAX_TAGS             32
+#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 64
+#define TSDB_MAX_TAGS_LEN         65536
+#define TSDB_MAX_TAGS             128
 
 #define TSDB_AUTH_LEN             16
 #define TSDB_KEY_LEN              16

src/rpc/src/rpcMain.c

@@ -361,9 +361,10 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   // connection type is application specific. 
   // for TDengine, all the query, show commands shall have TCP connection
   char type = pMsg->msgType;
-  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH ||
-      type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META ||
-      type == TSDB_MSG_TYPE_CM_SHOW )
+  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE
+    || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP
+    || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META
+    || type == TSDB_MSG_TYPE_CM_SHOW )
     pContext->connType = RPC_CONN_TCPC;
   
   rpcSendReqToServer(pRpc, pContext);
@@ -1109,6 +1110,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
+  tTrace("connection type is: %d", pConn->connType);
   writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
 
   if (writtenLen != msgLen) {

tests/pytest/table/boundary.py

+# -*- coding: utf-8 -*-
+
+import random
+import string
+import subprocess
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init( self, conn ):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+
+    def getLimitFromSourceCode( self, name ):
+        cmd = "grep -w '#define %s' ../../src/inc/taosdef.h|awk '{print $3}'" % name
+        return int(subprocess.check_output(cmd, shell=True))
+
+
+    def generateString( self, length ):
+        chars = string.ascii_uppercase + string.ascii_lowercase
+        v = ""
+        for i in range( length ):
+            v += random.choice( chars )
+        return v
+
+
+    def checkTagBoundaries( self ):
+        tdLog.debug( "checking tag boundaries" )
+        tdSql.prepare()
+
+        maxTags = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS' )
+        totalTagsLen = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS_LEN' )
+        tdLog.notice( "max tags is %d" % maxTags )
+        tdLog.notice( "max total tag length is %d" % totalTagsLen )
+
+        # for binary tags, 2 bytes are used for length
+        tagLen = (totalTagsLen - maxTags * 2) // maxTags
+        firstTagLen = totalTagsLen - 2 * maxTags - tagLen * (maxTags - 1)
+
+        sql = "create table cars(ts timestamp, f int) tags(t0 binary(%d)" % firstTagLen
+        for i in range( 1, maxTags ):
+            sql += ", t%d binary(%d)" % (i, tagLen)
+        sql += ");"
+
+        tdLog.debug( "creating super table: " + sql )
+        tdSql.execute( sql )
+        tdSql.query( 'show stables' )
+        tdSql.checkRows( 1 )
+
+        for i in range( 10 ):
+            sql = "create table car%d using cars tags('%d'" % (i, i)
+            sql += ", '0'" * (maxTags - 1) + ");"
+            tdLog.debug( "creating table: " + sql )
+            tdSql.execute( sql )
+
+            sql = "insert into car%d values(now, 0);" % i
+            tdLog.debug( "inserting data: " + sql )
+            tdSql.execute( sql )
+
+        tdSql.query( 'show tables' )
+        tdLog.info( 'tdSql.checkRow(10)' )
+        tdSql.checkRows( 10 )
+
+        tdSql.query( 'select * from cars;' )
+        tdSql.checkRows( 10 )
+
+
+    def checkColumnBoundaries( self ):
+        tdLog.debug( "checking column boundaries" )
+        tdSql.prepare()
+
+        # one column is for timestamp
+        maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1
+
+        sql = "create table cars (ts timestamp"
+        for i in range( maxCols ):
+            sql += ", c%d int" % i
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        sql = "insert into cars values (now"
+        for i in range( maxCols ):
+            sql += ", %d" % i
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'select * from cars' )
+        tdSql.checkRows( 1 )
+
+
+    def checkTableNameBoundaries( self ):
+        tdLog.debug( "checking table name boundaries" )
+        tdSql.prepare()
+
+        maxTableNameLen = self.getLimitFromSourceCode( 'TSDB_TABLE_NAME_LEN' )
+        tdLog.notice( "table name max length is %d" % maxTableNameLen )
+
+        name = self.generateString( maxTableNameLen - 1)
+        tdLog.info( "table name is '%s'" % name )
+
+        tdSql.execute( "create table %s (ts timestamp, value int)" % name )
+        tdSql.execute( "insert into %s values(now, 0)" % name )
+
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        tdSql.query( 'select * from %s' % name )
+        tdSql.checkRows( 1 )
+
+
+    def checkRowBoundaries( self ):
+        tdLog.debug( "checking row boundaries" )
+        tdSql.prepare()
+
+        # 8 bytes for timestamp
+        maxRowSize = 65536 - 8
+        maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1
+
+        # for binary cols, 2 bytes are used for length
+        colLen = (maxRowSize - maxCols * 2) // maxCols
+        firstColLen = maxRowSize - 2 * maxCols - colLen * (maxCols - 1)
+
+        sql = "create table cars (ts timestamp, c0 binary(%d)" % firstColLen
+        for i in range( 1, maxCols ):
+            sql += ", c%d binary(%d)" % (i, colLen)
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        col = self.generateString( firstColLen )
+        sql = "insert into cars values (now, '%s'" % col
+        col = self.generateString( colLen )
+        for i in range( 1, maxCols ):
+            sql += ", '%s'" % col 
+        sql += ");"
+        tdLog.info( sql );
+        tdSql.execute( sql )
+        tdSql.query( "select * from cars" )
+        tdSql.checkRows( 1 )
+
+
+    def run(self):
+        self.checkTagBoundaries()
+        self.checkColumnBoundaries()
+        self.checkTableNameBoundaries()
+        self.checkRowBoundaries()
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/table/tablename-boundary.py

-# -*- coding: utf-8 -*-
-
-import sys
-import string
-import random
-import subprocess
-from util.log import *
-from util.cases import *
-from util.sql import *
-
-
-class TDTestCase:
-    def init(self, conn):
-        tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
-
-    def run(self):
-        tdSql.prepare()
-
-        getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'"
-        tableNameMaxLen = int(
-            subprocess.check_output(
-                getTableNameLen, shell=True))
-        tdLog.info("table name max length is %d" % tableNameMaxLen)
-        chars = string.ascii_uppercase + string.ascii_lowercase
-        tb_name = ''.join(random.choices(chars, k=tableNameMaxLen))
-        tdLog.info('tb_name length %d' % len(tb_name))
-        tdLog.info('create table %s (ts timestamp, value int)' % tb_name)
-        tdSql.error(
-            'create table %s (ts timestamp, speed binary(4089))' %
-            tb_name)
-
-        tb_name = ''.join(random.choices(chars, k=191))
-        tdLog.info('tb_name length %d' % len(tb_name))
-        tdLog.info('create table %s (ts timestamp, value int)' % tb_name)
-        tdSql.execute(
-            'create table %s (ts timestamp, speed binary(4089))' %
-            tb_name)
-
-    def stop(self):
-        tdSql.close()
-        tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())