Merge remote-tracking branch 'origin/develop' into feature/wal

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.15-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.15-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.14</version>
+  <version>2.0.15</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>
@@ -36,7 +36,6 @@
     </developer>
   </developers>
 
-
   <dependencies>
     <dependency>
       <groupId>commons-logging</groupId>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.14</version>
+    <version>2.0.15</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -81,7 +81,7 @@ public class TSDBStatement implements Statement {
         }
 
         if (!this.connector.isUpdateQuery(pSql)) {
-        	TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer);
+            TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer);
             res.setBatchFetch(this.connection.getBatchFetch());
             return res;
         } else {
@@ -125,7 +125,8 @@ public class TSDBStatement implements Statement {
     }
 
     public int getMaxFieldSize() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        return 0;
+//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     public void setMaxFieldSize(int max) throws SQLException {
@@ -218,7 +219,8 @@ public class TSDBStatement implements Statement {
     }
 
     public int getFetchDirection() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        return ResultSet.FETCH_FORWARD;
+//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     /*

src/dnode/src/dnodeEps.c

@@ -237,7 +237,7 @@ PRASE_EPS_OVER:
   dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
 #else
   if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) {
-    dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
+    dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
     return -1;
   }
 #endif

tests/Jenkinsfile

+properties([pipelineTriggers([githubPush()])])
+node {
+    git url: 'https://github.com/liuyq-617/TDengine'
+}
+
+def pre_test(){
+    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                sudo rmtaos
+                '''
+    }
+    sh '''
+    cd ${WKC}
+    rm -rf *
+    cd ${WK}
+    git reset --hard
+    git checkout develop
+    git pull
+    cd ${WKC}
+    rm -rf *
+    mv ${WORKSPACE}/* .
+    cd ${WK}
+    export TZ=Asia/Harbin
+    date
+    rm -rf ${WK}/debug
+    mkdir debug
+    cd debug
+    cmake .. > /dev/null
+    make > /dev/null
+    make install > /dev/null
+    cd ${WKC}/tests
+    '''
+    return 1
+}
+pipeline {
+  agent none
+  environment{
+      WK = '/var/lib/jenkins/workspace/TDinternal'
+      WKC= '/var/lib/jenkins/workspace/TDinternal/community'
+  }
+
+  stages {
+      stage('Parallel test stage') {
+      parallel {
+        stage('python p1') {
+          agent{label 'p1'}
+          steps {
+            pre_test()
+            sh '''
+            cd ${WKC}/tests
+            ./test-all.sh p1
+            date'''
+          }
+        }
+        stage('test_b1') {
+          agent{label 'master'}
+          steps {
+            pre_test()
+            sh '''
+            cd ${WKC}/tests
+            ./test-all.sh b1
+            date'''
+          }
+        }
+
+        stage('test_crash_gen') {
+          agent{label "b2"}
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./crash_gen.sh -a -p -t 4 -s 2000
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./handle_crash_gen_val_log.sh
+                '''
+            }
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
+          }
+        }
+
+        stage('test_valgrind') {
+          agent{label "b3"}
+
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./valgrind-test.sh 2>&1 > mem-error-out.log
+                ./handle_val_log.sh
+                '''
+            }           
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b3
+            date'''
+          }
+        }
+       stage('python p2'){
+         agent{label "p2"}
+         steps{
+            pre_test()         
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh p2
+            date
+            '''
+          
+         }
+       }
+       
+          
+      }
+    }
+
+  }
+  
+}

tests/examples/JDBC/mybatisplus-demo/pom.xml

@@ -38,11 +38,16 @@
             <artifactId>h2</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <version>1.1.17</version>
+        </dependency>
 
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>
             <artifactId>taos-jdbcdriver</artifactId>
-            <version>2.0.11</version>
+            <version>2.0.14</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->

tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml

 spring:
   datasource:
-    #    driver-class-name: org.h2.Driver
-    #    schema: classpath:db/schema-mysql.sql
-    #    data: classpath:db/data-mysql.sql
-    #    url: jdbc:h2:mem:test
-    #    username: root
-    #    password: test
-
-    #    driver-class-name: com.mysql.jdbc.Driver
-    #    url: jdbc:mysql://master:3306/test?useSSL=false
-    #    username: root
-    #    password: 123456
-
     driver-class-name: com.taosdata.jdbc.TSDBDriver
     url: jdbc:TAOS://localhost:6030/mp_test
     user: root
@@ -20,6 +8,12 @@ spring:
     locale: en_US.UTF-8
     timezone: UTC-8
 
+    druid:
+      initial-size: 5
+      min-idle: 5
+      max-active: 5
+
+
 mybatis-plus:
   configuration:
     map-underscore-to-camel-case: false

tests/perftest-scripts/perftest-query.sh

@@ -65,7 +65,18 @@ function runQueryPerfTest {
 	echoInfo "Run Performance Test"
 	cd $WORK_DIR/TDengine/tests/pytest
 	
-	python3 query/queryPerformance.py 0 | tee -a $PERFORMANCE_TEST_REPORT
+	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -t 1000 -n 1000 > taosdemoperf.txt
+
+	CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'`
+	INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'`
+	REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'`
+	
+	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT
+	[ -f taosdemoperf.txt ] && rm taosdemoperf.txt
 }
 
 

tests/pytest/bug2265.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import taos
+if __name__ == "__main__":
+
+    logSql = True
+    deployPath = ""
+    testCluster = False
+    valgrind = 0
+
+    print("start to execute %s" % __file__)
+    tdDnodes.init(deployPath)
+    tdDnodes.setTestCluster(testCluster)
+    tdDnodes.setValgrind(valgrind)
+
+    tdDnodes.stopAll()
+    tdDnodes.addSimExtraCfg("maxSQLLength", "1048576")
+    tdDnodes.deploy(1)
+    tdDnodes.start(1)
+    host = '127.0.0.1'
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+
+    tdCases.logSql(logSql)
+    print('1')
+    conn = taos.connect(
+        host,
+        config=tdDnodes.getSimCfgPath())
+
+    tdSql.init(conn.cursor(), True)
+        
+    print("==========step1")
+    print("create table ")
+    tdSql.execute("create database db")
+    tdSql.execute("use db")
+    tdSql.execute("create table t1 (ts timestamp, c1 int,c2 int ,c3 int)")
+
+    print("==========step2")
+    print("insert maxSQLLength data ")
+    data = 'insert into t1 values'
+    ts = 1604298064000
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767 - 1) ):
+        data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000)
+        i+=1
+    tdSql.execute(data)
+    
+    print("==========step4")
+    print("insert data batch larger than 32767 ")
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767) ):
+        data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000)
+        i+=1
+    tdSql.error(data)
+
+    print("==========step4")
+    print("insert data larger than maxSQLLength ")
+    tdSql.execute("create table t2 (ts timestamp, c1 binary(50))")
+    data = 'insert into t2 values'
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767 - 1 ) ):
+        data += '(%s,%s)'%(ts+i,'a'*50)
+        i+=1
+    tdSql.error(data)      
+    tdSql.close()
+    tdLog.success("%s successfully executed" % __file__)
+        
+   
+

tests/pytest/client/noConnectionErrorTest.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        tdDnodes.stop(1)
+        sql = "use db"
+        
+        try:
+            tdSql.execute(sql)
+        except Exception as e:            
+            expectError = 'Unable to establish connection'
+            if expectError in str(e):                
+                pass
+            else:
+                caller = inspect.getframeinfo(inspect.stack()[1][1])
+                tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase()) 
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/fulltest.sh

@@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 python3 ./test.py -f insert/before_1970.py
+python3 bug2265.py
 
 python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
@@ -27,6 +28,8 @@ python3 ./test.py -f table/column_num.py
 python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
+python3 ./test.py  -f table/max_table_length.py
+
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
@@ -162,8 +165,10 @@ python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py
 python3 ./test.py -f query/bug2117.py
 python3 ./test.py -f query/bug2143.py
-python3 ./test.py -f query/sliding.py 
-
+python3 ./test.py -f query/sliding.py
+python3 ./test.py -f query/unionAllTest.py
+python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
 #stream
 python3 ./test.py -f stream/metric_1.py
 python3 ./test.py -f stream/new.py
@@ -179,6 +184,7 @@ python3 ./test.py -f alter/alter_table_crash.py
 python3 ./test.py -f client/client.py
 python3 ./test.py -f client/version.py
 python3 ./test.py -f client/alterDatabase.py
+python3 ./test.py -f client/noConnectionErrorTest.py
 
 # Misc
 python3 testCompress.py
@@ -213,6 +219,7 @@ python3 test.py -f query/queryFillTest.py
 python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/lowaTest.py
+python3 test.py -f tools/taosdemoTest2.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py

tests/pytest/insert/insertFromCSVPerformance.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class insertFromCSVPerformace:
+    def __init__(self, commitID, dbName, stbName, branchName):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.branchName = branchName
+        self.ts = 1500074556514
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)
+        
+    def writeCSV(self):
+        with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
+            writer = csv.writer(csvFile, dialect='excel')
+            for i in range(1000000):
+                newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000)
+                d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
+                dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
+                writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
+    
+    def removCSVHeader(self):
+        data = pd.read_csv("ordered.csv")
+        data = data.drop([0])
+        data.to_csv("ordered.csv", header = False, index = False)
+    
+    def createTables(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName)
+        cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName))
+
+        cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+        cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+
+        cursor.close()
+    
+    def run(self):
+        cursor = self.conn.cursor()
+        cursor.execute("use %s" % self.dbName)
+        print("==================== CSV insert performance ====================")
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t1 file 'outoforder.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t1")            
+        out_of_order_time = (float) (totalTime / 10)
+        print("Out of Order - Insert time: %f" % out_of_order_time)                      
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t2 file 'ordered.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t2")
+
+        in_order_time = (float) (totalTime / 10)
+        print("In order - Insert time: %f" % in_order_time)
+        cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID))
+        
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='csv_insert',
+        type=str,
+        help='Database name to be created (default: csv_insert)')
+    parser.add_argument(
+        '-b',
+        '--branch-name',
+        action='store',
+        default='develop',
+        type=str,
+        help='branch name (default: develop)')
+    
+    args = parser.parse_args()
+    perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name)
+
+    perftest.createTables()
+    perftest.run()
\ No newline at end of file

tests/pytest/pytest_1.sh

@@ -26,6 +26,7 @@ python3 ./test.py -f table/column_num.py
 python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
+python3 ./test.py  -f table/max_table_length.py
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
@@ -159,7 +160,9 @@ python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py 
-
+python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
+python3 bug2265.py
 #stream
 python3 ./test.py -f stream/metric_1.py
 python3 ./test.py -f stream/new.py

tests/pytest/query/bug2119.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)")
+       
+
+        print("==========step2")
+        print("query percentile from blank table")
+        tdSql.query('select percentile(c1,1) from t1')
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/query/bug2281.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)")
+        insertRows = 10
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(insertRows):
+            ret = tdSql.execute(
+                "insert into t1 values (%d , %d,%d)" %
+                (t0+i,i%100,i/2.0))
+
+        print("==========step2")
+        print("query diff && top")
+        tdSql.error('select diff(c1),top(c2) from t1')
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/query/queryPerformance.py

@@ -16,10 +16,16 @@ import sys
 import os
 import taos
 import time
+import argparse
 
 
 class taosdemoQueryPerformace:
-    def initConnection(self):
+    def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix):
+        self.clearCache = clearCache
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.tbPerfix = tbPerfix
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -30,92 +36,109 @@ class taosdemoQueryPerformace:
             self.password,
             self.config)
 
+    def createPerfTables(self):
+        cursor = self.conn.cursor()
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName)
+
+        sql = "select count(*) from test.meters"
+        tableid = 1
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters"
+        tableid = 2
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select count(*) from test.meters where loc='beijing'"
+        tableid = 3
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10"
+        tableid = 4
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)"
+        tableid = 5
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select last_row(*) from meters"
+        tableid = 6
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select * from meters"
+        tableid = 7
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'"
+        tableid = 8
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+
+        cursor.close()
 
     def query(self): 
-        cursor = self.conn.cursor()                  
-        cursor.execute("use test")
-
-        totalTime = 0
-        for i in range(100):       
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")  
-            startTime = time.time()            
-            cursor.execute("select count(*) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):  
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select count(*) from test.meters where loc='beijing'")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters where loc='beijing' %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters where areaid=10 %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):                
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.t10 interval(10s) %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select last_row(*) from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select * from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select * from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required        
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100))
+        cursor = self.conn.cursor() 
+        print("==================== query performance ====================")
+
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName)      
+
+        for data in cursor:
+            table_name = data[0]
+            query_id = data[1]
+            sql = data[2]            
+            
+            totalTime = 0            
+            cursor2 = self.conn.cursor()
+            cursor2.execute("use test")
+            for i in range(100):       
+                if(self.clearCache == True):
+                    # root permission is required
+                    os.system("echo 3 > /proc/sys/vm/drop_caches")                
+                
+                startTime = time.time()         
+                cursor2.execute(sql)
+                totalTime += time.time() - startTime
+            cursor2.close()    
+            print("query time for: %s %f seconds" % (sql, totalTime / 100))
+                        
+            cursor3 = self.conn.cursor()
+            cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID))
+
+        cursor3.close()
+        cursor.close()
 
 if __name__ == '__main__':
-    perftest = taosdemoQueryPerformace()
-    perftest.initConnection()
-    perftest.query()
\ No newline at end of file
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-r',
+        '--remove-cache',                
+        action='store_true',
+        default=False,
+        help='clear cache before query (default: False)')
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='query_tb',
+        type=str,
+        help='table name to be created (default: query_tb)')
+    parser.add_argument(
+        '-p',
+        '--table-perfix',
+        action='store',
+        default='q',
+        type=str,
+        help='table name perfix (default: q)')
+    
+    args = parser.parse_args()
+    perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix)
+    perftest.createPerfTables()
+    perftest.query()

tests/pytest/query/unionAllTest.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+import random
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1500000000000
+        self.num = 10
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table st(ts timestamp, c1 int) tags(loc nchar(20))")
+        tdSql.execute("create table t0 using st tags('nchar0')")
+        tdSql.execute("create table t1 using st tags('nchar1')")
+        tdSql.execute("create table t2 using st tags('nchar2')")
+        tdSql.execute("create table t3 using st tags('nchar3')")
+        tdSql.execute("create table t4 using st tags('nchar4')")
+        tdSql.execute("create table t5 using st tags('nchar5')")
+
+        for i in range(self.num):
+            tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t2 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t3 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t4 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t5 values(%d, %d)" % (self.ts + i, i))                
+
+        sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1'''
+        tdSql.query(sql)
+        tdSql.checkRows(5)
+
+        sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5' limit 1'''
+        tdSql.query(sql)
+        tdSql.checkRows(6)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/table/max_table_length.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, db_test.stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdLog.info("check nchar")
+        tdSql.error("create database anal (ts timestamp ,i nchar(4094))")
+        tdSql.execute(
+            "create table anal (ts timestamp ,i nchar(4093))")
+
+        print("==============step2")
+        tdLog.info("check binary")
+        tdSql.error("create database anal (ts timestamp ,i binary(16375))")
+        tdSql.execute(
+            "create table anal1 (ts timestamp ,i binary(16374))")
+        
+        print("==============step3")
+        tdLog.info("check int & binary")
+        tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)")
+        tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)")
+        tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)")
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemoPerformance.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class taosdemoPerformace:
+    def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.createTableTime = createTableTime
+        self.insertRecordsTime = insertRecordsTime
+        self.recordsPerSecond = recordsPerSecond        
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)    
+    
+    def createTablesAndStoreData(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))")
+        print("==================== taosdemo performance ====================")
+        print("create tables time: %f" % self.createTableTime)
+        print("insert records time: %f" % self.insertRecordsTime)
+        print("records per second: %f" % self.recordsPerSecond)
+        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID))
+        cursor.execute("drop database if exists taosdemo_insert_test")
+
+        cursor.close()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',        
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--create-table',
+        action='store',        
+        type=float,
+        help='create table time')
+    parser.add_argument(
+        '-i',
+        '--insert-records',
+        action='store',        
+        type=float,
+        help='insert records time')
+    parser.add_argument(
+        '-r',
+        '---records-per-second',
+        action='store',        
+        type=float,
+        help='records per request')
+    
+    args = parser.parse_args()
+
+    perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second)    
+    perftest.createTablesAndStoreData()
\ No newline at end of file

tests/pytest/tools/taosdemoTest2.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import threading
+import time
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.numberOfTables = 10
+        self.numberOfRecords = 1000000
+
+    def insertDataAndAlterTable(self, threadID):
+        if(threadID == 0):
+            os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords))
+        if(threadID == 1):
+            print("use test")
+            tdSql.execute("use test")
+            print("alter table test.meters add column f4 int")
+            tdSql.execute("alter table test.meters add column f4 int")
+            print("insert into test.t0 values (now, 1, 2, 3, 4)")
+            tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4)")
+
+    def run(self):        
+        tdSql.prepare()
+
+        t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, ))
+        t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, ))
+
+        t1.start()
+        time.sleep(2)
+        t2.start()
+        t1.join()
+        t2.join()
+
+        tdSql.query("select count(*) from test.meters")
+        tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase()) 
\ No newline at end of file