diff --git a/README.md b/README.md index 280cf73ba0e8140b18e4a98902a7a58f8a72c881..158ae040fa0a95e6c2209ea71f8a3e5da6f848cc 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,251 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo - [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector) - [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector) +# How to run the test cases and how to add a new test case? + +### Prepare development environment + +1. sudo apt install + build-essential cmake net-tools python-pip python-setuptools python3-pip + python3-setuptools valgrind + +2. git clone ; cd TDengine + +3. mkdir debug; cd debug; cmake ..; make ; sudo make install + +4. pip install src/connector/python/linux/python2 ; pip3 install + src/connector/python/linux/python3 + +### How to run TSIM test suite + +1. cd \/tests/script + +2. sudo ./test.sh + +### How to run Python test suite + +1. cd \/tests/pytest + +2. ./smoketest.sh \# for smoke test + +3. ./smoketest.sh -g \# for memory leak detection test with valgrind + +4. ./fulltest.sh \# for full test + +> Note1: TDengine daemon's configuration and data files are stored in +> \/sim directory. As a historical design, it's same place with +> TSIM script. So after the TSIM script ran with sudo privilege, the directory +> has been used by TSIM then the python script cannot write it by a normal +> user. You need to remove the directory completely first before running the +> Python test case. We should consider using two different locations to store +> for TSIM and Python script. + +> Note2: if you need to debug crash problem with a core dump, you need +> manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited" +> before the script line. Then you can look for the core file in +> \/tests/pytest after the program crash. + +### How to add a new test case + +**1. add a new TSIM test cases:** + +TSIM test cases are now included in the new development branch and can be +added to the TDengine/tests/script/test.sh script based on the manual test +methods necessary to add test cases as described above. + +**2. add a new Python test cases:** + +**2.1 Please refer to \/tests/pytest/insert/basic.py to add a new +test case.** The new test case must implement 3 functions, where self.init() +and self.stop() simply copy the contents of insert/basic.py and the test +logic is implemented in self.run(). You can refer to the code in the util +directory for more information. + +**2.2 Edit smoketest.sh to add the path and filename of the new test case** + +Note: The Python test framework may continue to be improved in the future, +hopefully, to provide more functionality and ease of writing test cases. The +method of writing the test case above does not exclude that it will also be +affected. + +**2.3 What test.py does in detail:** + +test.py is the entry program for test case execution and monitoring. + +test.py has the following functions. + +\-f --file, Specifies the test case file name to be executed +-p --path, Specifies deployment path + +\-m --master, Specifies the master server IP for cluster deployment +-c--cluster, test cluster function +-s--stop, terminates all running nodes + +\-g--valgrind, load valgrind for memory leak detection test + +\-h--help, display help + +**2.4 What util/log.py does in detail:** + +log.py is quite simple, the main thing is that you can print the output in +different colors as needed. The success() should be called for successful +test case execution and the success() will print green text. The exit() will +print red text and exit the program, exit() should be called for test +failure. + +**util/log.py** + +... + +    def info(self, info): + +        printf("%s %s" % (datetime.datetime.now(), info)) + +  + +    def sleep(self, sec): + +        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + +        time.sleep(sec) + +  + +    def debug(self, err): + +        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def success(self, info): + +        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info)) + +  + +    def notice(self, err): + +        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def exit(self, err): + +        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +        sys.exit(1) + +  + +    def printNoPrefix(self, info): + +        printf("\\033[1;36m%s\\033[0m" % (info) + +... + +**2.5 What util/sql.py does in detail:** + +SQL.py is mainly used to execute SQL statements to manipulate the database, +and the code is extracted and commented as follows: + +**util/sql.py** + +\# prepare() is mainly used to set up the environment for testing table and +data, and to set up the database db for testing. do not call prepare() if you +need to test the database operation command. + +def prepare(self): + +tdLog.info("prepare database:db") + +self.cursor.execute('reset query cache') + +self.cursor.execute('drop database if exists db') + +self.cursor.execute('create database db') + +self.cursor.execute('use db') + +... + +\# query() is mainly used to execute select statements for normal syntax input + +def query(self, sql): + +... + +\# error() is mainly used to execute the select statement with the wrong syntax +input, the error will be caught as a reasonable behavior, if not caught it will +prove that the test failed + +def error() + +... + +\# checkRows() is used to check the number of returned lines after calling +query(select ...) after calling the query(select ...) to check the number of +rows of returned results. + +def checkRows(self, expectRows): + +... + +\# checkData() is used to check the returned result data after calling +query(select ...) after the query(select ...) is called, failure to meet +expectation is + +def checkData(self, row, col, data): + +... + +\# getData() returns the result data after calling query(select ...) to return +the resulting data after calling query(select ...) + +def getData(self, row, col): + +... + +\# execute() used to execute sql and return the number of affected rows + +def execute(self, sql): + +... + +\# executeTimes() Multiple executions of the same sql statement + +def executeTimes(self, sql, times): + +... + +\# CheckAffectedRows() Check if the number of affected rows is as expected + +def checkAffectedRows(self, expectAffectedRows): + +... + +> Note: Both Python2 and Python3 are currently supported by the Python test +> case. Since Python2 is no longer officially supported by January 1, 2020, it +> is recommended that subsequent test case development be guaranteed to run +> correctly on Python3. For Python2, please consider being compatible if +> appropriate without additional +> burden.   + +### CI Covenant submission adoption principle. + +- Every commit / PR compilation must pass. Currently, the warning is treated + as an error, so the warning must also be resolved. + +- Test cases that already exist must pass. + +- Because CI is very important to support build and automatically test + procedure, it is necessary to manually test the test case before adding it + and do as many iterations as possible to ensure that the test case provides + stable and reliable test results when added. + +> Note: In the future, according to the requirements and test development +> progress will add stress testing, performance testing, code style, +> and other features based on functional testing. + ### Third Party Connectors The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them. diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 77001609b61d35c746ef2ef37702aa6fb1460106..6ef54f1ba550ad22f906433801492082d599809c 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -146,6 +146,7 @@ class CTaosInterface(object): libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p + libtaos.taos_fetch_lengths.restype = ctypes.c_void_p def __init__(self, config=None): ''' @@ -314,6 +315,8 @@ class CTaosInterface(object): isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if data == None: @@ -323,7 +326,7 @@ class CTaosInterface(object): if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 726554e49028a2dd4e6a0a15afd5a21b7bedcab1..1e820be0e961620ef1f7458fc9cdca66a3c8c9aa 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -52,14 +52,14 @@ int32_t mgmtInitDServer() { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; + tsMgmtDServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS"); + tsMgmtDServerRpc = rpcOpen(&rpcInit); if (tsMgmtDServerRpc == NULL) { mError("failed to init server connection to dnode"); return -1; } - tsMgmtDServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS"); - mPrint("server connection to dnode is opened"); return 0; } diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index b4dd58cb3b5d7a9c41766de2c039c7f09ea4d39a..ecd3c217cae6af01ec1fc9a47de03a2d652377f7 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -396,7 +396,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -439,13 +439,13 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } mgmtSendSimpleResp(pMsg->thandle, code); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 0c1b9e314e4168b3bd3f9181a061dee007760a7f..b1228b0230537874442f3dc6fa5d497a13c9cfa6 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -180,7 +180,7 @@ void tsdbFitRetention(STsdbRepo *pRepo) { int mfid = tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision); - while (pGroup[0].fileId < mfid) { + while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) { tsdbRemoveFileGroup(pFileH, pGroup[0].fileId); } }