Merge remote-tracking branch 'origin/develop' into feature/m2d

tests/pytest/client/change_time_1_1.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.pathFinding import *
+from util.dnodes import tdDnodes
+from datetime import datetime
+import subprocess
+
+##TODO: this is now automatic, but not sure if this will run through jenkins
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        tdFindPath.init(__file__)
+
+    def run(self):
+        tdSql.prepare()
+        binPath = tdFindPath.getTaosdemoPath()
+        TDenginePath = tdFindPath.getTDenginePath()
+
+        ## change system time to 2020/10/20
+        os.system('sudo timedatectl set-ntp off')
+        os.system('sudo timedatectl set-time 2020-10-20')
+
+        #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20
+        #11 data files should be generated
+        #vnode at TDinternal/community/sim/dnode1/data/vnode
+        os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json")  
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 11:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.debug("data file number correct")
+        
+        #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write
+        #leaving 7 data files.
+        os.system ('timedatectl set-time 2020-10-25')
+        os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") 
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 7:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.debug("data file number correct")
+        tdSql.query('select first(ts) from stb_0')
+        tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database
+
+    def stop(self):
+        os.system('sudo timedatectl set-ntp on')
+        tdSql.close()
+        tdLog.success("alter block manual check finish")
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/client/change_time_1_2.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from util.pathFinding import *
+from datetime import datetime
+import subprocess
+
+##TODO: this is now automatic, but not sure if this will run through jenkins
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        tdFindPath.init(__file__)
+
+    def run(self):
+        tdSql.prepare()
+        binPath = tdFindPath.getTaosdemoPath()
+        TDenginePath = tdFindPath.getTDenginePath()
+
+        ## change system time to 2020/10/20
+        os.system ('timedatectl set-ntp off')
+        os.system ('timedatectl set-time 2020-10-20')
+
+        #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20
+        #11 data files should be generated
+        #vnode at TDinternal/community/sim/dnode1/data/vnode
+        os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") 
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 11:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.debug("data file number correct")
+
+        tdSql.query('select first(ts) from stb_0') #check the last data in the database
+        tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0))
+
+        os.system ('timedatectl set-time 2020-10-25')
+
+        #moves 5 days ahead to 2020/10/25 and restart taosd
+        #4 oldest data file should be removed from tsdb/data
+        #7 data file should be found 
+        #vnode at TDinternal/community/sim/dnode1/data/vnode
+        os.system ('timedatectl set-time 2020-10-25')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query('select first(ts) from stb_0')
+        tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database
+        commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
+        result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        print(result.count('data'))
+        if result.count('data') != 7:
+            tdLog.exit('wrong number of files')
+        else:
+            tdLog.debug("data file number correct")
+
+
+
+    def stop(self):
+        os.system('sudo timedatectl set-ntp on')
+        tdSql.close()
+        tdLog.success("alter block manual check finish")
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/fulltest.sh

@@ -256,6 +256,8 @@ python3 ./test.py -f client/client.py
 python3 ./test.py -f client/version.py
 python3 ./test.py -f client/alterDatabase.py
 python3 ./test.py -f client/noConnectionErrorTest.py
+python3 test.py -f client/change_time_1_1.py
+python3 test.py -f client/change_time_1_2.py
 
 # Misc
 python3 testCompress.py

tests/pytest/manualTest/manual_change_time_2.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from fabric import Connection
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from datetime import datetime
+import subprocess
+import time
+import taos
+##TODO: this is now automatic, but not sure if this will run through jenkins
+
+#the initial time used for this test is 2020/10/20
+
+#setting local machine's time for later connecting to the server
+os.system('sudo timedatectl set-ntp off')
+os.system('sudo timedatectl set-time 2020-10-25')
+
+#connect to VM lyq-1, and initalize the environment at lyq-1
+conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn1.run("sudo systemctl stop taosd")
+conn1.run('ls -l')
+conn1.run('sudo timedatectl set-ntp off')
+conn1.run('sudo timedatectl set-time 2020-10-20')
+
+with conn1.cd('/data/taos/log'):
+    conn1.run('sudo rm -rf *')
+
+with conn1.cd('/data/taos/data'):
+    conn1.run('sudo rm -rf *')
+
+#lanuch taosd and start taosdemo  
+conn1.run("sudo systemctl start taosd")
+time.sleep(5)
+with conn1.cd('~/bschang_test'):
+    conn1.run('taosdemo -f manual_change_time_1_1_A.json')
+
+#force everything onto disk
+conn1.run("sudo systemctl restart taosd")
+time.sleep(10)
+
+#change lyq-1 to 2020/10/25 for testing if the server
+#will send data that is out of time range
+conn1.run('sudo timedatectl set-time 2020-10-25')
+
+#connect to VM lyq-2, initalize the environment at lyq-2, and run taosd 
+#on that
+conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn2.run('sudo timedatectl set-ntp off')
+conn2.run('sudo timedatectl set-time 2020-10-20')
+conn2.run("sudo systemctl stop taosd")
+with conn2.cd('/data/taos/log'):
+    conn2.run('sudo rm -rf *')
+with conn2.cd('/data/taos/data'):
+    conn2.run('sudo rm -rf *')
+conn2.run("sudo systemctl start taosd")
+
+#set replica to 2
+connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
+c1 = connTaos.cursor()
+c1.execute('create dnode \'lyq-2:6030\'')
+c1.execute('alter database db replica 2')
+c1.close()
+connTaos.close()
+time.sleep(5)
+
+#force everything onto the disk for lyq-2
+#stopping taosd on lyq-1 for future testing
+conn2.run("sudo systemctl stop taosd")
+conn1.run("sudo systemctl stop taosd")
+
+#reset the time
+conn1.run('sudo timedatectl set-ntp on')
+conn2.run('sudo timedatectl set-ntp on')
+os.system('sudo timedatectl set-ntp on')
+
+#check if the number of file received is 7
+#the 4 oldest data files should be dropped
+#4 files because of moving 5 days ahead
+with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
+    result = conn2.run('ls -l |grep \'data\' |wc -l')
+    if result.stdout.strip() != '7':
+        tdLog.exit('the file number is wrong')
+    else:
+        tdLog.success('the file number is the same. test pass')

tests/pytest/manualTest/manual_change_time_3.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from fabric import Connection
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from datetime import datetime
+import subprocess
+import time
+import taos
+
+##TODO: this is now automatic, but not sure if this will run through jenkins
+
+#the initial time used for this test is 2020/10/20
+
+#setting local machine's time for later connecting to the server
+os.system('sudo timedatectl set-ntp off')
+os.system('sudo timedatectl set-time 2020-10-20')
+
+#connect to VM lyq-1, and initalize the environment at lyq-1
+conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn1.run("sudo systemctl stop taosd")
+conn1.run('sudo timedatectl set-ntp off')
+conn1.run('sudo timedatectl set-time 2020-10-20')
+with conn1.cd('/data/taos/log'):
+    conn1.run('sudo rm -rf *')
+
+with conn1.cd('/data/taos/data'):
+    conn1.run('sudo rm -rf *')
+
+#lanuch taosd and start taosdemo    
+conn1.run("sudo systemctl start taosd")
+time.sleep(5)
+with conn1.cd('~/bschang_test'):
+    conn1.run('taosdemo -f manual_change_time_1_1_A.json') #the json file is placed in lyq-1 already
+
+#force everything onto disk
+conn1.run("sudo systemctl restart taosd")
+time.sleep(10)
+
+#connect to VM lyq-2, and initalize the environment at lyq-2
+conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
+conn2.run('sudo timedatectl set-ntp off')
+conn2.run('sudo timedatectl set-time 2020-10-20')
+conn2.run("sudo systemctl stop taosd")
+with conn2.cd('/data/taos/log'):
+    conn2.run('sudo rm -rf *')
+with conn2.cd('/data/taos/data'):
+    conn2.run('sudo rm -rf *')
+
+#the date of lyq-2 is going to be set to 2020/10/25
+#for testing if other pnode will accpet file out of local time range
+conn2.run("sudo systemctl start taosd")
+conn2.run('sudo timedatectl set-time 2020-10-25')
+
+#set the replica to 2
+connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
+c1 = connTaos.cursor()
+c1.execute('create dnode \'lyq-2:6030\'')
+c1.execute('alter database db replica 2')
+time.sleep(5)
+
+#force everything onto the disk for lyq-2
+#stopping taosd on lyq-1 for future testing
+conn2.run("sudo systemctl stop taosd")
+conn1.run("sudo systemctl stop taosd")
+
+#reset the time
+conn1.run('sudo timedatectl set-ntp on')
+conn2.run('sudo timedatectl set-ntp on')
+os.system('sudo timedatectl set-ntp on')
+
+#check if the number of file received is 7
+#the 4 oldest data files should be dropped
+#4 files because of moving 5 days ahead
+with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
+    result = conn2.run('ls -l |grep \'data\' |wc -l')
+    if result.stdout.strip() != '7':
+        tdLog.exit('the file number is wrong')
+    else:
+        tdLog.success('the file number is the same. test pass')
+
+c1.close()
+connTaos.close()
+

tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",                
+            "replica": 1,
+            "days": 1,
+            "cache": 4,
+            "blocks": 3,
+            "precision": "ms",
+            "keep": 10,
+            "minRows": 1000,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",   
+            "childtable_count": 1,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "no",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "rand",       
+            "insert_mode": "taosc",      
+            "insert_rows": 864000,       
+            "childtable_limit": 1,      
+            "childtable_offset":0,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1000,         
+            "start_timestamp": "2020-10-11 00:00:00.000",  
+            "sample_format": "csv",       
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "BINARY", "len": 5120, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}]
+        }]
+    }]
+}
\ No newline at end of file

tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",                
+            "replica": 1,
+            "days": 1,
+            "cache": 4,
+            "blocks": 3,
+            "precision": "ms",
+            "keep": 10,
+            "minRows": 1000,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"yes",   
+            "childtable_count": 1,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "yes",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "rand",       
+            "insert_mode": "taosc",      
+            "insert_rows": 432000,       
+            "childtable_limit": 1,      
+            "childtable_offset":0,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1000,         
+            "start_timestamp": "2020-10-16 00:00:00.500",  
+            "sample_format": "csv",       
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "BINARY", "len": 5120, "count":1}],
+            "tags": [{"type": "TINYINT", "count":1}]
+        }]
+    }]
+}
\ No newline at end of file

tests/pytest/util/pathFinding.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+import os
+from util.log import *
+
+
+
+class TDFindPath:
+    """This class is for finding path within TDengine
+    """
+    def __init__(self):
+        self.file = ""
+
+
+    def init(self, file):
+        """[summary]
+
+        Args:
+            file (str): the file location you want to start the query. Generally using __file__
+        """
+        self.file = file
+
+    def getTaosdemoPath(self):
+        """for finding the path of directory containing taosdemo
+
+        Returns:
+            str: the path to directory containing taosdemo
+        """
+        selfPath = os.path.dirname(os.path.realpath(self.file))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info(f"taosd found in {buildPath}")
+        return buildPath + "/build/bin/"  
+
+    def getTDenginePath(self):
+        """for finding the root path of TDengine
+
+        Returns:
+            str: the root path of TDengine
+        """
+        selfPath = os.path.dirname(os.path.realpath(self.file))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+        print(projPath)
+        for root, dirs, files in os.walk(projPath):
+            if ("sim" in dirs):
+                print(root)
+                rootRealPath = os.path.realpath(root)
+        if (rootRealPath == ""):
+            tdLog.exit("TDengine not found!")
+        else:
+            tdLog.info(f"TDengine found in {rootRealPath}")  
+        return rootRealPath
+
+tdFindPath = TDFindPath()
\ No newline at end of file