[TD-4366] updating the automatic version of test cases. Not sure if they can go through jenkins

tests/pytest/manualTest/manual_chang_time_2.py

-###################################################################
-#           Copyright (c) 2016 by TAOS Technologies, Inc.
-#                     All rights reserved.
-#
-#  This file is proprietary and confidential to TAOS Technologies.
-#  No part of this file may be reproduced, stored, transmitted,
-#  disclosed or used in any form or by any means other than as
-#  expressly provided by the written permission from Jianhui Tao
-#
-###################################################################
-
-# -*- coding: utf-8 -*-
-from fabric import Connection
-import sys
-from util.log import *
-from util.cases import *
-from util.sql import *
-from util.dnodes import tdDnodes
-from datetime import datetime
-import subprocess
-
-# class VMPnode
-
-# ##TODO: auto test version is currently unsupported, need to come up with 
-# #       an auto test version in the future
-# class TDTestCase:
-#     def init(self, conn, logSql):
-#         tdLog.debug("start to execute %s" % __file__)
-#         tdSql.init(conn.cursor(), logSql)
-
-#     def run(self):
-#         return
-
-
-
-#     def stop(self):
-#         tdSql.close()
-#         tdLog.debug("%s alter block manual check finish" % __file__)
-
-
-# tdCases.addWindows(__file__, TDTestCase())
-# tdCases.addLinux(__file__, TDTestCase())
-
-conn = conn = Connection("{}@{}".format('ubuntu', "lyq-q-1"), connect_kwargs={"password": "{}".format('tbase125!')})
-conn.run('sudo timedatectl set-ntp off')
-conn.run('sudo timedatectl set-time 2020-10-20')
\ No newline at end of file

tests/pytest/manualTest/manual_chang_time_1_1.py → tests/pytest/manualTest/manual_change_time_1_1.py

@@ -19,13 +19,29 @@ from util.dnodes import tdDnodes
 from datetime import datetime
 import subprocess
 
-##TODO: auto test version is currently unsupported, need to come up with 
-#       an auto test version in the future
+##TODO: this is now automatic, but not sure if this will run through jenkins
 class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
+    def getTDenginePath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+        print(projPath)
+        for root, dirs, files in os.walk(projPath):
+            if ("sim" in dirs):
+                print(root)
+                rootRealPath = os.path.realpath(root)
+                # if ("packaging" not in rootRealPath):
+                #     buildPath = root[:len(root)-len("/build/bin")]
+                    # break
+        return rootRealPath
+        
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -41,7 +57,7 @@ class TDTestCase:
                     buildPath = root[:len(root)-len("/build/bin")]
                     break
         return buildPath
-        
+
     def run(self):
         tdSql.prepare()
         buildPath = self.getBuildPath()
@@ -50,37 +66,42 @@ class TDTestCase:
         else:
             tdLog.info("taosd found in %s" % buildPath)
         binPath = buildPath+ "/build/bin/"  
+        TDenginePath = self.getTDenginePath()
+        print('TD '+ TDenginePath)
+        if (TDenginePath == ""):
+            tdLog.exit("TDengine not found!")
+        else:
+            tdLog.info("TDengine found in %s" % TDenginePath) 
 
         ## change system time to 2020/10/20
-        os.system ('timedatectl set-ntp off')
-        os.system ('timedatectl set-time 2020-10-20')
+        os.system('sudo timedatectl set-ntp off')
+        os.system('sudo timedatectl set-time 2020-10-20')
 
         #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20
         #11 data files should be generated
         #vnode at TDinternal/community/sim/dnode1/data/vnode
         os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) 
-        input("please the data file. After checking, press enter")
-
-        tdSql.query('select first(ts) from stb_0')
-        tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0))
-        tdSql.query('select last(ts) from stb_0')
-        tdSql.checkData(0,0,datetime(2020,10,20,23,59,59,0))
-
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 11:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.info("data file number correct")
+        
+        #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write
+        #leaving 7 data files.
         os.system ('timedatectl set-time 2020-10-25')
-
-        #run taosdemo to insert data. one row per second from 2020/10/16 to 2020/10/20
-        #4 oldest data file should be removed from tsdb/data
-        #vnode at TDinternal/community/sim/dnode1/data/vnode
-        #7 data file should be found 
         os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) 
-        input("please the data file. After checking, press enter")
-
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 7:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.info("data file number correct")
         tdSql.query('select first(ts) from stb_0')
-        tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0))
-        
-        ##test results
-        #2021/06/05 first check: 11 data files    second check: 7 data files
-        #           confirm with the assumption       Baosheng Chang
+        tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database
 
     def stop(self):
         tdSql.close()

tests/pytest/manualTest/manual_chang_time_1_2.py → tests/pytest/manualTest/manual_change_time_1_2.py

@@ -19,8 +19,7 @@ from util.dnodes import tdDnodes
 from datetime import datetime
 import subprocess
 
-##TODO: auto test version is currently unsupported, need to come up with 
-#       an auto test version in the future
+##TODO: this is now automatic, but not sure if this will run through jenkins
 class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
@@ -95,10 +94,11 @@ class TDTestCase:
 
         os.system ('timedatectl set-time 2020-10-25')
 
-        #restart taosd
+        #moves 5 days ahead to 2020/10/25 and restart taosd
         #4 oldest data file should be removed from tsdb/data
-        #vnode at TDinternal/community/sim/dnode1/data/vnode
         #7 data file should be found 
+        #vnode at TDinternal/community/sim/dnode1/data/vnode
+        os.system ('timedatectl set-time 2020-10-25')
         tdDnodes.stop(1)
         tdDnodes.start(1)
         tdSql.query('select first(ts) from stb_0')

tests/pytest/manualTest/manual_change_time_2.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from fabric import Connection
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from datetime import datetime
+import subprocess
+import time
+import taos
+##TODO: this is now automatic, but not sure if this will run through jenkins
+
+#the initial time used for this test is 2020/10/20
+
+#setting local machine's time for later connecting to the server
+os.system('sudo timedatectl set-ntp off')
+os.system('sudo timedatectl set-time 2020-10-25')
+
+#connect to VM lyq-1, and initalize the environment at lyq-1
+conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn1.run("sudo systemctl stop taosd")
+conn1.run('ls -l')
+conn1.run('sudo timedatectl set-ntp off')
+conn1.run('sudo timedatectl set-time 2020-10-20')
+
+with conn1.cd('/data/taos/log'):
+    conn1.run('sudo rm -rf *')
+
+with conn1.cd('/data/taos/data'):
+    conn1.run('sudo rm -rf *')
+
+#lanuch taosd and start taosdemo  
+conn1.run("sudo systemctl start taosd")
+time.sleep(5)
+with conn1.cd('~/bschang_test'):
+    conn1.run('taosdemo -f manual_change_time_1_1_A.json')
+
+#force everything onto disk
+conn1.run("sudo systemctl restart taosd")
+time.sleep(10)
+
+#change lyq-1 to 2020/10/25 for testing if the server
+#will send data that is out of time range
+conn1.run('sudo timedatectl set-time 2020-10-25')
+
+#connect to VM lyq-2, initalize the environment at lyq-2, and run taosd 
+#on that
+conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn2.run('sudo timedatectl set-ntp off')
+conn2.run('sudo timedatectl set-time 2020-10-20')
+conn2.run("sudo systemctl stop taosd")
+with conn2.cd('/data/taos/log'):
+    conn2.run('sudo rm -rf *')
+with conn2.cd('/data/taos/data'):
+    conn2.run('sudo rm -rf *')
+conn2.run("sudo systemctl start taosd")
+
+#set replica to 2
+connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
+c1 = connTaos.cursor()
+c1.execute('create dnode \'lyq-2:6030\'')
+c1.execute('alter database db replica 2')
+c1.close()
+connTaos.close()
+time.sleep(5)
+
+#force everything onto the disk for lyq-2
+#stopping taosd on lyq-1 for future testing
+conn2.run("sudo systemctl stop taosd")
+conn1.run("sudo systemctl stop taosd")
+
+#reset the time
+conn1.run('sudo timedatectl set-ntp on')
+conn2.run('sudo timedatectl set-ntp on')
+os.system('sudo timedatectl set-ntp on')
+
+#check if the number of file received is 7
+#the 4 oldest data files should be dropped
+#4 files because of moving 5 days ahead
+with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
+    result = conn2.run('ls -l |grep \'data\' |wc -l')
+    if result.stdout.strip() != '7':
+        tdLog.exit('the file number is wrong')
+    else:
+        tdLog.success('the file number is the same. test pass')

tests/pytest/manualTest/manual_change_time_3.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from fabric import Connection
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from datetime import datetime
+import subprocess
+import time
+import taos
+
+##TODO: this is now automatic, but not sure if this will run through jenkins
+
+#the initial time used for this test is 2020/10/20
+
+#setting local machine's time for later connecting to the server
+os.system('sudo timedatectl set-ntp off')
+os.system('sudo timedatectl set-time 2020-10-20')
+
+#connect to VM lyq-1, and initalize the environment at lyq-1
+conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn1.run("sudo systemctl stop taosd")
+conn1.run('sudo timedatectl set-ntp off')
+conn1.run('sudo timedatectl set-time 2020-10-20')
+with conn1.cd('/data/taos/log'):
+    conn1.run('sudo rm -rf *')
+
+with conn1.cd('/data/taos/data'):
+    conn1.run('sudo rm -rf *')
+
+#lanuch taosd and start taosdemo    
+conn1.run("sudo systemctl start taosd")
+time.sleep(5)
+with conn1.cd('~/bschang_test'):
+    conn1.run('taosdemo -f manual_change_time_1_1_A.json') #the json file is placed in lyq-1 already
+
+#force everything onto disk
+conn1.run("sudo systemctl restart taosd")
+time.sleep(10)
+
+#connect to VM lyq-2, and initalize the environment at lyq-2
+conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn2.run('sudo timedatectl set-ntp off')
+conn2.run('sudo timedatectl set-time 2020-10-20')
+conn2.run("sudo systemctl stop taosd")
+with conn2.cd('/data/taos/log'):
+    conn2.run('sudo rm -rf *')
+with conn2.cd('/data/taos/data'):
+    conn2.run('sudo rm -rf *')
+
+#the date of lyq-2 is going to be set to 2020/10/25
+#for testing if other pnode will accpet file out of local time range
+conn2.run("sudo systemctl start taosd")
+conn2.run('sudo timedatectl set-time 2020-10-25')
+
+#set the replica to 2
+connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
+c1 = connTaos.cursor()
+c1.execute('create dnode \'lyq-2:6030\'')
+c1.execute('alter database db replica 2')
+time.sleep(5)
+
+#force everything onto the disk for lyq-2
+#stopping taosd on lyq-1 for future testing
+conn2.run("sudo systemctl stop taosd")
+conn1.run("sudo systemctl stop taosd")
+
+#reset the time
+conn1.run('sudo timedatectl set-ntp on')
+conn2.run('sudo timedatectl set-ntp on')
+os.system('sudo timedatectl set-ntp on')
+
+#check if the number of file received is 7
+#the 4 oldest data files should be dropped
+#4 files because of moving 5 days ahead
+with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
+    result = conn2.run('ls -l |grep \'data\' |wc -l')
+    if result.stdout.strip() != '7':
+        tdLog.exit('the file number is wrong')
+    else:
+        tdLog.success('the file number is the same. test pass')
+
+c1.close()
+connTaos.close()
+