Merge pull request #14394 from taosdata/cpwu/3.0

test: fix the case and test-framework

tests/pytest/util/common.py

@@ -17,6 +17,7 @@ import string
 import requests
 import time
 import socket
+import json
 from .boundary import DataBoundary
 import taos
 from util.log import *
@@ -25,7 +26,7 @@ from util.cases import *
 from util.dnodes import *
 from util.common import *
 
-class TDCom: 
+class TDCom:
     def __init__(self):
         self.sml_type = None
         self.env_setting = None
@@ -206,12 +207,12 @@ class TDCom:
         """
             generate long name
             mode could be numbers/letters/letters_mixed/mixed
-        """    
-        if mode == "numbers": 
+        """
+        if mode == "numbers":
             chars = ''.join(random.choice(string.digits) for i in range(len))
-        elif mode == "letters": 
+        elif mode == "letters":
             chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len))
-        elif mode == "letters_mixed": 
+        elif mode == "letters_mixed":
             chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len))
         else:
             chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len))
@@ -276,7 +277,7 @@ class TDCom:
         vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions
         '''
         sqlString = f'create database if not exists {dbName} '
-        
+
         dbParams = ""
         if len(kwargs) > 0:
             for param, value in kwargs.items():
@@ -306,7 +307,7 @@ class TDCom:
     #     return
 
     # def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict):
-    #     tsql.execute("use %s" %dbName)        
+    #     tsql.execute("use %s" %dbName)
     #     tagsValues = ''
     #     for i in range(tagDict['int']):
     #         if i > 0:
@@ -323,7 +324,7 @@ class TDCom:
     #             sql = pre_create
     #     if sql != pre_create:
     #         tsql.execute(sql)
-        
+
     #     tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
     #     return
 
@@ -352,7 +353,7 @@ class TDCom:
     #         tsql.execute(sql)
     #     tdLog.debug("insert data ............ [OK]")
     #     return
-        
+
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -367,7 +368,7 @@ class TDCom:
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
-        return buildPath        
+        return buildPath
 
     def getClientCfgPath(self):
         buildPath = self.getBuildPath()
@@ -398,7 +399,7 @@ class TDCom:
         return newTdSql
 
     ################################################################################################################
-    # port from the common.py of new test frame 
+    # port from the common.py of new test frame
     ################################################################################################################
     def gen_default_tag_str(self):
         default_tag_str = ""
@@ -527,9 +528,9 @@ class TDCom:
                             tag_value_list.append(self.gen_random_type_value(tag_elm["type"], "", "", "", ""))
                 else:
                     continue
-        return tag_value_list        
+        return tag_value_list
 
-    def gen_column_value_list(self, column_elm_list, ts_value=None):        
+    def gen_column_value_list(self, column_elm_list, ts_value=None):
         if ts_value is None:
             ts_value = self.genTs()[0]
 
@@ -554,7 +555,7 @@ class TDCom:
         return column_value_list
 
     def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None,
-                     count=1, default_stbname_prefix="stb", **kwargs):        
+                     count=1, default_stbname_prefix="stb", **kwargs):
         colname_prefix = 'c'
         tagname_prefix = 't'
         stbname_index_start_num = 1
@@ -589,20 +590,20 @@ class TDCom:
                 tag_value_str += f'"{tag_value}", '
             else:
                 tag_value_str += f'{tag_value}, '
-        tag_value_str = tag_value_str.rstrip()[:-1] 
-            
+        tag_value_str = tag_value_str.rstrip()[:-1]
+
         if int(count) <= 1:
             create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num}  using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
             tsql.execute(create_ctable_sql)
         else:
             for _ in range(count):
                 create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
-                ctbname_index_start_num += 1                
+                ctbname_index_start_num += 1
                 tdLog.info("create ctb sql: %s"%create_ctable_sql)
                 tsql.execute(create_ctable_sql)
-    
+
     def create_table(self, tsql, dbname=None, tbname="ntb", column_elm_list=None, count=1, **kwargs):
-        tbname_index_start_num = 1 
+        tbname_index_start_num = 1
         tbname_prefix="ntb"
 
         tb_params = ""
@@ -632,7 +633,7 @@ class TDCom:
                 column_value_str += f'"{column_value}", '
             else:
                 column_value_str += f'{column_value}, '
-        column_value_str = column_value_str.rstrip()[:-1] 
+        column_value_str = column_value_str.rstrip()[:-1]
         if int(count) <= 1:
             insert_sql = f'insert into {self.tb_name} values ({column_value_str});'
             tsql.execute(insert_sql)
@@ -651,4 +652,16 @@ class TDCom:
             return res_list
         else:
             tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
+
+
+def is_json(msg):
+    if isinstance(msg, str):
+        try:
+            json.loads(msg)
+            return True
+        except:
+            return False
+    else:
+        return False
+
 tdCom = TDCom()

tests/pytest/util/constant.py

@@ -71,6 +71,34 @@ TAOS_KEYWORDS = [
     "COPY",			"IF",			"NOW",			"STABLES",		"WHERE",
 ]
 
+NUM_FUNC = [
+    "ABS", "ACOS", "ASIN", "ATAN", "CEIL", "COS", "FLOOR", "LOG", "POW", "ROUND", "SIN", "SQRT", "TAN",
+]
+
+STR_FUNC = [
+    "CHAR_LENGTH", "CONCAT", "CONCAT_WS", "LENGTH", "LOWER","LTRIM", "RTRIM", "SUBSTR", "UPPER",
+]
+
+CONVER_FUNC = ["CASR", "TO_ISO8601", "TO_JSON", "TP_UNIXTIMESTAMP"]
+
+SELECT_FUNC = [
+    "APERCENTILE", "BOTTOM", "FIRST", "INTERP", "LAST", "MAX", "MIN", "PERCENTILE", "TAIL", "TOP", "UNIQUE",
+]
+
+AGG_FUNC = [
+    "AVG", "COUNT", "ELAPSED", "LEASTSQUARES", "MODE", "SPREAD", "STDDEV", "SUM", "HYPERLOGLOG", "HISTOGRAM",
+]
+
+TS_FUNC = [
+    "CSUM", "DERIVATIVE", "DIFF", "IRATE", "MAVG", "SAMPLE", "STATECOUNT", "STATEDURATION", "TWA"
+]
+
+SYSINFO_FUCN = [
+    "DATABASE", "CLIENT_VERSION", "SERVER_VERSION", "SERVER_STATUS", "CURRENT_USER", "USER"
+]
+
+
+
 # basic data type boundary
 TINYINT_MAX             = 127
 TINYINT_MIN             = -128
@@ -84,16 +112,16 @@ SMALLINT_MIN            = -32768
 SMALLINT_UN_MAX         = 65535
 SMALLINT_UN_MIN         = 0
 
-INT_MAX                 = 2147483647
-INT_MIN                 = -2147483648
+INT_MAX                 = 2_147_483_647
+INT_MIN                 = -2_147_483_648
 
-INT_UN_MAX              = 4294967295
+INT_UN_MAX              = 4_294_967_295
 INT_UN_MIN              = 0
 
-BIGINT_MAX              = 9223372036854775807
-BIGINT_MIN              = -9223372036854775808
+BIGINT_MAX              = 9_223_372_036_854_775_807
+BIGINT_MIN              = -9_223_372_036_854_775_808
 
-BIGINT_UN_MAX           = 18446744073709551615
+BIGINT_UN_MAX           = 18_446_744_073_709_551_615
 BIGINT_UN_MIN           = 0
 
 FLOAT_MAX               = 3.40E+38
@@ -131,13 +159,13 @@ COL_COUNT_MIN           = 2
 TAG_COL_COUNT_MAX       = 4096
 TAG_COL_COUNT_MIN       = 3
 
-MNODE_SHM_SIZE_MAX      = 2147483647
-MNODE_SHM_SIZE_MIN      = 6292480
-MNODE_SHM_SIZE_DEFAULT  = 6292480
+MNODE_SHM_SIZE_MAX      = 2_147_483_647
+MNODE_SHM_SIZE_MIN      = 6_292_480
+MNODE_SHM_SIZE_DEFAULT  = 6_292_480
 
-VNODE_SHM_SIZE_MAX      = 2147483647
-VNODE_SHM_SIZE_MIN      = 6292480
-VNODE_SHM_SIZE_DEFAULT  = 31458304
+VNODE_SHM_SIZE_MAX      = 2_147_483_647
+VNODE_SHM_SIZE_MIN      = 6_292_480
+VNODE_SHM_SIZE_DEFAULT  = 31_458_304
 
 # time_init
 TIME_MS = 1
@@ -160,6 +188,7 @@ INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US
 # streams and related agg-function
 SMA_INDEX_FUNCTIONS = ["MIN", "MAX"]
 ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"]
+BLOCK_FUNCTIONS = ["SUM", "MIN", "MAX"]
 SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"]
 WATERMARK_MAX = 900000
 WATERMARK_MIN = 0

tests/pytest/util/sql.py

@@ -96,6 +96,15 @@ class TDSql:
             return self.queryResult
         return self.queryRows
 
+    def is_err_sql(self, sql):
+        err_flag = True
+        try:
+            self.cursor.execute(sql)
+        except BaseException:
+            err_flag = False
+
+        return False if err_flag else True
+
     def getVariable(self, search_attr):
         '''
             get variable of search_attr access "show variables"
@@ -249,7 +258,6 @@ class TDSql:
             raise Exception(repr(e))
         return self.queryResult
 
-
     def executeTimes(self, sql, times):
         for i in range(times):
             try:
@@ -336,6 +344,38 @@ class TDSql:
         elif precision == "ns":
             return int(times*1000*1000)
 
+    def get_type(self, col):
+        if self.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if self.cursor.istype(col, "INT"):
+            return "INT"
+        if self.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if self.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if self.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if self.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if self.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if self.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if self.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if self.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if self.cursor.istype(col, "JSON"):
+            return "JSON"
+        if self.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if self.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if self.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if self.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
     def taosdStatus(self, state):
         tdLog.sleep(5)
         pstate = 0

tests/system-test/2-query/explain.py

@@ -163,38 +163,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def explain_check(self):
         sqls = self.sql_list()
         tdLog.printNoPrefix("===step 1: curent case, must return query OK")

tests/system-test/2-query/hyperloglog.py

@@ -116,37 +116,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
 
     def hyperloglog_check(self):
         sqls = self.sql_list()
@@ -228,10 +197,10 @@ class TDTestCase:
             tag_sql += f"{k} {v},"
         tdSql.execute(f'create table if not exists {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})')
 
-    def __insert_data(self): 
+    def __insert_data(self):
 
         pass
-    
+
     def __hyperloglog_check_distribute(self):
         dbname = "dbtest"
         stbname = "stb"
@@ -286,7 +255,7 @@ class TDTestCase:
             tdSql.query(f"select hyperloglog({k}) from {stbname} group by {k}")
 
         tdSql.execute(f'drop database {dbname}')
-        
+
 
     def __insert_data(self, rows):
         now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)

tests/system-test/2-query/leastsquares.py

@@ -195,38 +195,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, current_sqls)), list(filter(None, err_sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def leastsquares_check(self):
         current_sqls, err_sqls = self.sql_list()
         for i in range(len(err_sqls)):

tests/system-test/2-query/spread.py

@@ -159,38 +159,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def spread_check(self):
         sqls = self.sql_list()
         tdLog.printNoPrefix("===step 1: curent case, must return query OK")