diff --git a/.drone.yml b/.drone.yml index f7ee4e976f7b6a077ea0b9a8cb7f9cadb5c3628f..b520f308ba68e2acd631795fbcfbf91a1b64c2a9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -25,15 +25,14 @@ steps: - master --- kind: pipeline -name: test_arm64 +name: test_arm64_bionic platform: os: linux arch: arm64 - steps: - name: build - image: gcc + image: arm64v8/ubuntu:bionic commands: - apt-get update - apt-get install -y cmake build-essential @@ -48,9 +47,87 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline -name: test_arm +name: test_arm64_focal + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/ubuntu:focal + commands: + - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + - apt-get update + - apt-get install -y -qq cmake build-essential + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm64_centos7 + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/centos:7 + commands: + - yum install -y gcc gcc-c++ make cmake git + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm64_centos8 + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/centos:8 + commands: + - dnf install -y gcc gcc-c++ make cmake epel-release git libarchive + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm_bionic platform: os: linux @@ -73,7 +150,6 @@ steps: branch: - develop - master - --- kind: pipeline name: build_trusty @@ -174,25 +250,3 @@ steps: - develop - master ---- -kind: pipeline -name: goodbye - -platform: - os: linux - arch: amd64 - -steps: -- name: 64-bit - image: alpine - commands: - - echo 64-bit is good. - when: - branch: - - develop - - master - - -depends_on: -- test_arm64 -- test_amd64 \ No newline at end of file diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 1064f0b0e5fff2e08efd1177570055fc90c94b89..e9266ec80da293571ece07dab9c724b5b8c12adf 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -35,7 +35,7 @@ fi if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh ${script_dir}/startPre.sh" else bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator\ ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 4a4d79099b076b8ff12d5b4fdbcba54049a6866d..a44ec1ca493ad01b2bf825b6418f69e11f548206 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d +Subproject commit a44ec1ca493ad01b2bf825b6418f69e11f548206 diff --git a/src/connector/python/README.md b/src/connector/python/README.md index a5dc2b72dafbeef0bf53fc1768f6afc66e714699..95ef26e1f0e73cee7d47ecb6ece1d6a95d2f89d3 100644 --- a/src/connector/python/README.md +++ b/src/connector/python/README.md @@ -1,6 +1,7 @@ # TDengine Connector for Python -[TDengine] connector for Python enables python programs to access TDengine, using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. ## Install @@ -11,8 +12,417 @@ pip install ./TDengine/src/connector/python ## Source Code -[TDengine] connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). -## License - AGPL +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 Keep same with [TDengine](https://github.com/taosdata/TDengine). diff --git a/src/connector/python/examples/bind-multi.py b/src/connector/python/examples/bind-multi.py new file mode 100644 index 0000000000000000000000000000000000000000..8530253aef58079e01f5eb71d8e12ab1649b7731 --- /dev/null +++ b/src/connector/python/examples/bind-multi.py @@ -0,0 +1,50 @@ +# encoding:UTF-8 +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt_multi" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() \ No newline at end of file diff --git a/src/connector/python/examples/bind-row.py b/src/connector/python/examples/bind-row.py new file mode 100644 index 0000000000000000000000000000000000000000..4ab9a9167ad23a6167c6586aac30ae6941dcee6d --- /dev/null +++ b/src/connector/python/examples/bind-row.py @@ -0,0 +1,57 @@ +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +# No need to explicitly close, but ok for you +# result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) + +# No need to explicitly close, but ok for you +# result.close() +# stmt.close() +# conn.close() diff --git a/src/connector/python/examples/insert-lines.py b/src/connector/python/examples/insert-lines.py new file mode 100644 index 0000000000000000000000000000000000000000..0096b7e8cdf1328ee78805a1ee3134ad7cdfc447 --- /dev/null +++ b/src/connector/python/examples/insert-lines.py @@ -0,0 +1,22 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', +] +conn.insert_lines(lines) +print("inserted") + +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) + + +conn.execute("drop database if exists %s" % dbname) diff --git a/src/connector/python/examples/pep-249.py b/src/connector/python/examples/pep-249.py new file mode 100644 index 0000000000000000000000000000000000000000..971a3c401f00b982096b8d429f65bce73cca4760 --- /dev/null +++ b/src/connector/python/examples/pep-249.py @@ -0,0 +1,9 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) diff --git a/src/connector/python/examples/query-async.py b/src/connector/python/examples/query-async.py new file mode 100644 index 0000000000000000000000000000000000000000..b600b796974e47d5e5fc7d88998e95ba46bb92cd --- /dev/null +++ b/src/connector/python/examples/query-async.py @@ -0,0 +1,62 @@ +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + # should explicitly close the result in fetch completed or cause error + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + # explicitly close result while query failed + result.close() + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + # conn.close() + + +if __name__ == "__main__": + test_query(connect()) \ No newline at end of file diff --git a/src/connector/python/examples/query-objectively.py b/src/connector/python/examples/query-objectively.py new file mode 100644 index 0000000000000000000000000000000000000000..104347cbf91e29e62fef26477b475053a8b8bc3e --- /dev/null +++ b/src/connector/python/examples/query-objectively.py @@ -0,0 +1,12 @@ +import taos + +conn = taos.connect() +conn.execute("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +conn.execute("drop database pytest") diff --git a/src/connector/python/examples/subscribe-async.py b/src/connector/python/examples/subscribe-async.py new file mode 100644 index 0000000000000000000000000000000000000000..3782ce5505152e78838406e313094eb911bea4a2 --- /dev/null +++ b/src/connector/python/examples/subscribe-async.py @@ -0,0 +1,43 @@ +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + # sub.close() + + conn.execute("drop database if exists %s" % dbname) + # conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + # conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) diff --git a/src/connector/python/examples/subscribe-sync.py b/src/connector/python/examples/subscribe-sync.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7f65f460280924ed3a577fe55b975fbf12c1a3 --- /dev/null +++ b/src/connector/python/examples/subscribe-sync.py @@ -0,0 +1,53 @@ +import taos +import random + +conn = taos.connect() +dbname = "pytest_taos_subscribe" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.execute("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + +sub = conn.subscribe(False, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +sub.close(True) +print("# keep progress consume") +sub = conn.subscribe(False, "test", "select * from log", 1000) +result = sub.consume() +rows = result.fetch_all() +# consume from latest subscription needs root privilege(for /var/lib/taos). +assert result.row_count == 0 +print("## consumed ", len(rows), "rows") + +print("# consume with a stop condition") +for i in range(10): + conn.execute("insert into log values(now, %d)" % random.randint(0, 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +# sub.close() + +conn.execute("drop database if exists %s" % dbname) +# conn.close() diff --git a/src/connector/python/pyproject.toml b/src/connector/python/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..a8099199563a0e5957a7d69e75bab65cca6d17db --- /dev/null +++ b/src/connector/python/pyproject.toml @@ -0,0 +1,27 @@ +[tool.poetry] +name = "taos" +version = "2.1.0" +description = "TDengine connector for python" +authors = ["Taosdata Inc. "] +license = "AGPL-3.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^2.7 || ^3.4" +typing = "*" + +[tool.poetry.dev-dependencies] +pytest = [ + { version = "^4.6", python = "^2.7" }, + { version = "^6.2", python = "^3.7" } +] +pdoc = { version = "^7.1.1", python = "^3.7" } +mypy = { version = "^0.910", python = "^3.6" } +black = { version = "^21.7b0", python = "^3.6" } + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 119 diff --git a/src/connector/python/setup.py b/src/connector/python/setup.py index 284861ca87dd77b1bc6799ad4cb32ff4c489e239..b7e10001737bc40c04173ea4a65e95248965ffda 100644 --- a/src/connector/python/setup.py +++ b/src/connector/python/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.11", + version="2.1.0", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/taos/__init__.py b/src/connector/python/taos/__init__.py index 52c6db311ecc4c2f944372ae3334fdc58cb6e779..75138eade3d60f7894d814babe58cec7aecc9a20 100644 --- a/src/connector/python/taos/__init__.py +++ b/src/connector/python/taos/__init__.py @@ -1,20 +1,478 @@ +# encoding:UTF-8 +""" +# TDengine Connector for Python -from .connection import TDengineConnection -from .cursor import TDengineCursor +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. -# For some reason, the following is needed for VS Code (through PyLance) to +## Install + +```sh +git clone --depth 1 https://github.com/taosdata/TDengine.git +pip install ./TDengine/src/connector/python +``` + +## Source Code + +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). + +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 + +Keep same with [TDengine](https://github.com/taosdata/TDengine). +""" +from .connection import TaosConnection + +# For some reason, the following is needed for VS Code (through PyLance) to # recognize that "error" is a valid module of the "taos" package. -from .error import ProgrammingError +from .error import * +from .bind import * +from .field import * +from .cursor import * +from .result import * +from .statement import * +from .subscription import * + +try: + import importlib.metadata + + __version__ = importlib.metadata.version("taos") +except: + None # Globals threadsafety = 0 -paramstyle = 'pyformat' - -__all__ = ['connection', 'cursor'] +paramstyle = "pyformat" +__all__ = [ + # functions + "connect", + "new_bind_param", + "new_bind_params", + "new_multi_binds", + "new_multi_bind", + # objects + "TaosBind", + "TaosConnection", + "TaosCursor", + "TaosResult", + "TaosRows", + "TaosRow", + "TaosStmt", + "PrecisionEnum", +] def connect(*args, **kwargs): - """ Function to return a TDengine connector object + # type: (..., ...) -> TaosConnection + """Function to return a TDengine connector object Current supporting keyword parameters: @dsn: Data source name as string @@ -25,4 +483,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) + return TaosConnection(*args, **kwargs) diff --git a/src/connector/python/taos/bind.py b/src/connector/python/taos/bind.py new file mode 100644 index 0000000000000000000000000000000000000000..ede6381628ae0fd5ff8794ef23db2f5afcfb5f3d --- /dev/null +++ b/src/connector/python/taos/bind.py @@ -0,0 +1,432 @@ +# encoding:UTF-8 +import ctypes +from .constants import FieldType +from .error import * +from .precision import * +from datetime import datetime +from ctypes import * +import sys + +_datetime_epoch = datetime.utcfromtimestamp(0) + +def _is_not_none(obj): + obj != None +class TaosBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_size_t)), + ("is_null", POINTER(c_int)), + ("is_unsigned", c_int), + ("error", POINTER(c_int)), + ("u", c_int64), + ("allocated", c_int), + ] + + def null(self): + self.buffer_type = FieldType.C_NULL + self.is_null = pointer(c_int(1)) + + def bool(self, value): + self.buffer_type = FieldType.C_BOOL + self.buffer = cast(pointer(c_bool(value)), c_void_p) + self.buffer_length = sizeof(c_bool) + + def tinyint(self, value): + self.buffer_type = FieldType.C_TINYINT + self.buffer = cast(pointer(c_int8(value)), c_void_p) + self.buffer_length = sizeof(c_int8) + + def smallint(self, value): + self.buffer_type = FieldType.C_SMALLINT + self.buffer = cast(pointer(c_int16(value)), c_void_p) + self.buffer_length = sizeof(c_int16) + + def int(self, value): + self.buffer_type = FieldType.C_INT + self.buffer = cast(pointer(c_int32(value)), c_void_p) + self.buffer_length = sizeof(c_int32) + + def bigint(self, value): + self.buffer_type = FieldType.C_BIGINT + self.buffer = cast(pointer(c_int64(value)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def float(self, value): + self.buffer_type = FieldType.C_FLOAT + self.buffer = cast(pointer(c_float(value)), c_void_p) + self.buffer_length = sizeof(c_float) + + def double(self, value): + self.buffer_type = FieldType.C_DOUBLE + self.buffer = cast(pointer(c_double(value)), c_void_p) + self.buffer_length = sizeof(c_double) + + def binary(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_BINARY + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def timestamp(self, value, precision=PrecisionEnum.Milliseconds): + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + ts = value + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(pointer(c_int64(ts)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def nchar(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_NCHAR + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def tinyint_unsigned(self, value): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer = cast(pointer(c_uint8(value)), c_void_p) + self.buffer_length = sizeof(c_uint8) + + def smallint_unsigned(self, value): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer = cast(pointer(c_uint16(value)), c_void_p) + self.buffer_length = sizeof(c_uint16) + + def int_unsigned(self, value): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer = cast(pointer(c_uint32(value)), c_void_p) + self.buffer_length = sizeof(c_uint32) + + def bigint_unsigned(self, value): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer = cast(pointer(c_uint64(value)), c_void_p) + self.buffer_length = sizeof(c_uint64) + + +def _datetime_to_timestamp(value, precision): + # type: (datetime | float | int | str | c_int64, PrecisionEnum) -> c_int64 + if value is None: + return FieldType.C_BIGINT_NULL + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + return int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + return c_int64(value) + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif isinstance(value, c_int64): + return value + return FieldType.C_BIGINT_NULL + + +class TaosMultiBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_int32)), + ("is_null", c_char_p), + ("num", c_int), + ] + + def null(self, num): + self.buffer_type = FieldType.C_NULL + self.is_null = cast((c_char * num)(*[1 for _ in range(num)]), c_char_p) + self.buffer = c_void_p(None) + self.num = num + + def bool(self, values): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BOOL_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + self.buffer_type = FieldType.C_BOOL + self.buffer_length = sizeof(c_bool) + + def tinyint(self, values): + self.buffer_type = FieldType.C_TINYINT + self.buffer_length = sizeof(c_int8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint(self, values): + self.buffer_type = FieldType.C_SMALLINT + self.buffer_length = sizeof(c_int16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int(self, values): + self.buffer_type = FieldType.C_INT + self.buffer_length = sizeof(c_int32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint(self, values): + self.buffer_type = FieldType.C_BIGINT + self.buffer_length = sizeof(c_int64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def float(self, values): + self.buffer_type = FieldType.C_FLOAT + self.buffer_length = sizeof(c_float) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_float * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_FLOAT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def double(self, values): + self.buffer_type = FieldType.C_DOUBLE + self.buffer_length = sizeof(c_double) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_double * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_DOUBLE_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def binary(self, values): + self.num = len(values) + self.buffer = cast(c_char_p("".join(filter(_is_not_none, values)).encode("utf-8")), c_void_p) + self.length = (c_int * len(values))(*[len(value) if value is not None else 0 for value in values]) + self.buffer_type = FieldType.C_BINARY + self.is_null = cast((c_byte * self.num)(*[1 if v == None else 0 for v in values]), c_char_p) + + def timestamp(self, values, precision=PrecisionEnum.Milliseconds): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + buffer = buffer_type(*[_datetime_to_timestamp(value, precision) for value in values]) + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(buffer, c_void_p) + self.buffer_length = sizeof(c_int64) + self.num = len(values) + + def nchar(self, values): + # type: (list[str]) -> None + if sys.version_info < (3, 0): + _bytes = [bytes(value) if value is not None else None for value in values] + buffer_length = max(len(b) + 1 for b in _bytes if b is not None) + buffers = [ + create_string_buffer(b, buffer_length) if b is not None else create_string_buffer(buffer_length) + for b in _bytes + ] + buffer_all = b''.join(v[:] for v in buffers) + self.buffer = cast(c_char_p(buffer_all), c_void_p) + else: + _bytes = [value.encode("utf-8") if value is not None else None for value in values] + buffer_length = max(len(b) for b in _bytes if b is not None) + self.buffer = cast( + c_char_p( + b"".join( + [ + create_string_buffer(b, buffer_length) + if b is not None + else create_string_buffer(buffer_length) + for b in _bytes + ] + ) + ), + c_void_p, + ) + self.length = (c_int32 * len(values))(*[len(b) if b is not None else 0 for b in _bytes]) + self.buffer_length = buffer_length + self.num = len(values) + self.is_null = cast((c_byte * self.num)(*[1 if v == None else 0 for v in values]), c_char_p) + self.buffer_type = FieldType.C_NCHAR + + def tinyint_unsigned(self, values): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer_length = sizeof(c_uint8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint_unsigned(self, values): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer_length = sizeof(c_uint16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int_unsigned(self, values): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer_length = sizeof(c_uint32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint_unsigned(self, values): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer_length = sizeof(c_uint64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + +def new_bind_param(): + # type: () -> TaosBind + return TaosBind() + + +def new_bind_params(size): + # type: (int) -> Array[TaosBind] + return (TaosBind * size)() + + +def new_multi_bind(): + # type: () -> TaosMultiBind + return TaosMultiBind() + + +def new_multi_binds(size): + # type: (int) -> Array[TaosMultiBind] + return (TaosMultiBind * size)() diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 660707bfcd04edb9d815b38d8ae806f35d2bfe2b..51e9a8667ddcab3d5d67da8be429f460f33d9eed 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -1,295 +1,839 @@ +# encoding:UTF-8 + import ctypes -from .constants import FieldType -from .error import * -import math -import datetime import platform +import sys +from ctypes import * +try: + from typing import Any +except: + pass + +from .error import * +from .bind import * +from .field import * + + +# stream callback +stream_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p) +stream_callback2_type = CFUNCTYPE(None, c_void_p) + +# C interface class +class TaosOption: + Locale = (0,) + Charset = (1,) + Timezone = (2,) + ConfigDir = (3,) + ShellActivityTimer = (4,) + MaxOptions = (5,) + + +def _load_taos_linux(): + return ctypes.CDLL("libtaos.so") + + +def _load_taos_darwin(): + return ctypes.CDLL("libtaos.dylib") + + +def _load_taos_windows(): + return ctypes.windll.LoadLibrary("taos") -def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=milli/1000.0) +def _load_taos(): + load_func = { + "Linux": _load_taos_linux, + "Darwin": _load_taos_darwin, + "Windows": _load_taos_windows, + } + try: + return load_func[platform.system()]() + except: + sys.exit("unsupported platform to TDengine connector") -def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=micro / 1000000.0) +_libtaos = _load_taos() +_libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) +_libtaos.taos_init.restype = None +_libtaos.taos_connect.restype = ctypes.c_void_p +_libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) +_libtaos.taos_errstr.restype = ctypes.c_char_p +_libtaos.taos_subscribe.restype = ctypes.c_void_p +_libtaos.taos_consume.restype = ctypes.c_void_p +_libtaos.taos_fetch_lengths.restype = ctypes.POINTER(ctypes.c_int) +_libtaos.taos_free_result.restype = None +_libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +try: + _libtaos.taos_stmt_errstr.restype = c_char_p +except AttributeError: + None +finally: + None -def _convert_nanosecond_to_datetime(nanosec): - return nanosec +_libtaos.taos_options.restype = None -def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row + +def taos_options(option, *args): + # type: (TaosOption, Any) -> None + _libtaos.taos_options(option, *args) + + +def taos_init(): + # type: () -> None """ - _timestamp_converter = _convert_millisecond_to_datetime - if precision == FieldType.C_TIMESTAMP_MILLI: - _timestamp_converter = _convert_millisecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_MICRO: - _timestamp_converter = _convert_microsecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_NANO: - _timestamp_converter = _convert_nanosecond_to_datetime - else: - raise DatabaseError("Unknown precision returned from database") + C: taos_init + """ + _libtaos.taos_init() + + +_libtaos.taos_cleanup.restype = None - return [ - None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_int64))[ - :abs(num_of_rows)]] +def taos_cleanup(): + # type: () -> None + """Cleanup workspace.""" + _libtaos.taos_cleanup() -def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row + +_libtaos.taos_get_client_info.restype = c_char_p + + +def taos_get_client_info(): + # type: () -> str + """Get client version info. + 获取客户端版本信息。 """ - return [ - None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_byte))[ - :abs(num_of_rows)]] + return _libtaos.taos_get_client_info().decode() + + +_libtaos.taos_get_server_info.restype = c_char_p +_libtaos.taos_get_server_info.argtypes = (c_void_p,) + + +def taos_get_server_info(connection): + # type: (c_void_p) -> str + return _libtaos.taos_get_server_info(connection).decode() + + +_libtaos.taos_close.restype = None +_libtaos.taos_close.argtypes = (c_void_p,) + + +def taos_close(connection): + # type: (c_void_p) -> None + """Close the TAOS* connection""" + _libtaos.taos_close(connection) + + +_libtaos.taos_connect.restype = c_void_p +_libtaos.taos_connect.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + + +def taos_connect(host=None, user="root", password="taosdata", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p + """Create TDengine database connection. + 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - password: user password / 用户密码 + - db: database name (optional) + - port: server port -def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + @rtype: c_void_p, TDengine handle """ - return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # password + try: + _password = c_char_p(password.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") + # port + try: + _port = c_uint16(port) + except TypeError: + raise TypeError("port is expected as an uint16") -def _crow_tinyint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + connection = cast(_libtaos.taos_connect(_host, _user, _password, _db, _port), c_void_p) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_connect_auth.restype = c_void_p +_libtaos.taos_connect_auth.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + + +def taos_connect_auth(host=None, user="root", auth="", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p """ - return [ - None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ubyte))[ - :abs(num_of_rows)]] + 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - auth: base64 encoded auth token + - db: database name (optional) + - port: server port -def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row + @rtype: c_void_p, TDengine handle """ - return [ - None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_short))[ - :abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # auth + try: + _auth = c_char_p(auth.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") + + # port + try: + _port = c_int(port) + except TypeError: + raise TypeError("port is expected as an int") + + connection = c_void_p(_libtaos.taos_connect_auth(_host, _user, _auth, _db, _port)) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_query.restype = c_void_p +_libtaos.taos_query.argtypes = c_void_p, c_char_p + + +def taos_query(connection, sql): + # type: (c_void_p, str) -> c_void_p + """Run SQL + + - sql: str, sql string to run + + @return: TAOS_RES*, result pointer -def _crow_smallint_unsigned_to_python( - data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row """ - return [ - None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ushort))[ - :abs(num_of_rows)]] + try: + ptr = c_char_p(sql.encode("utf-8")) + res = c_void_p(_libtaos.taos_query(connection, ptr)) + errno = taos_errno(res) + if errno != 0: + errstr = taos_errstr(res) + taos_free_result(res) + raise ProgrammingError(errstr, errno) + return res + except AttributeError: + raise AttributeError("sql is expected as a string") -def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row +async_query_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_query_a.restype = None +_libtaos.taos_query_a.argtypes = c_void_p, c_char_p, async_query_callback_type, c_void_p + + +def taos_query_a(connection, sql, callback, param): + # type: (c_void_p, str, async_query_callback_type, c_void_p) -> c_void_p + _libtaos.taos_query_a(connection, c_char_p(sql.encode("utf-8")), async_query_callback_type(callback), param) + + +async_fetch_rows_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_fetch_rows_a.restype = None +_libtaos.taos_fetch_rows_a.argtypes = c_void_p, async_fetch_rows_callback_type, c_void_p + + +def taos_fetch_rows_a(result, callback, param): + # type: (c_void_p, async_fetch_rows_callback_type, c_void_p) -> c_void_p + _libtaos.taos_fetch_rows_a(result, async_fetch_rows_callback_type(callback), param) + + +def taos_affected_rows(result): + # type: (c_void_p) -> c_int + """The affected rows after runing query""" + return _libtaos.taos_affected_rows(result) + + +subscribe_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p, c_int) +_libtaos.taos_subscribe.restype = c_void_p +# _libtaos.taos_subscribe.argtypes = c_void_p, c_int, c_char_p, c_char_p, subscribe_callback_type, c_void_p, c_int + + +def taos_subscribe(connection, restart, topic, sql, interval, callback=None, param=None): + # type: (c_void_p, bool, str, str, c_int, subscribe_callback_type, c_void_p | None) -> c_void_p + """Create a subscription + @restart boolean, + @sql string, sql statement for data query, must be a 'select' statement. + @topic string, name of this subscription """ - return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + if callback != None: + callback = subscribe_callback_type(callback) + if param != None: + param = c_void_p(param) + return c_void_p( + _libtaos.taos_subscribe( + connection, + 1 if restart else 0, + c_char_p(topic.encode("utf-8")), + c_char_p(sql.encode("utf-8")), + callback or None, + param, + interval, + ) + ) + + +_libtaos.taos_consume.restype = c_void_p +_libtaos.taos_consume.argstype = c_void_p, + + +def taos_consume(sub): + """Consume data of a subscription""" + return c_void_p(_libtaos.taos_consume(sub)) + + +_libtaos.taos_unsubscribe.restype = None +_libtaos.taos_unsubscribe.argstype = c_void_p, c_int + + +def taos_unsubscribe(sub, keep_progress): + """Cancel a subscription""" + _libtaos.taos_unsubscribe(sub, 1 if keep_progress else 0) + + +def taos_use_result(result): + """Use result after calling self.query, it's just for 1.6.""" + fields = [] + pfields = taos_fetch_fields_raw(result) + for i in range(taos_field_count(result)): + fields.append( + { + "name": pfields[i].name, + "bytes": pfields[i].bytes, + "type": pfields[i].type, + } + ) + + return fields + + +_libtaos.taos_fetch_block.restype = c_int +_libtaos.taos_fetch_block.argtypes = c_void_p, c_void_p + + +def taos_fetch_block_raw(result): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + return pblock, abs(num_of_rows) + + +def taos_fetch_block(result, fields=None, field_count=None): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + precision = taos_result_precision(result) + if fields == None: + fields = taos_fetch_fields(result) + if field_count == None: + field_count = taos_field_count(result) + blocks = [None] * field_count + fieldLen = taos_fetch_lengths(result, field_count) + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]["type"] not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i]["type"]](data, num_of_rows, fieldLen[i], precision) + + return blocks, abs(num_of_rows) + + +_libtaos.taos_fetch_row.restype = c_void_p +_libtaos.taos_fetch_row.argtypes = (c_void_p,) + + +def taos_fetch_row_raw(result): + # type: (c_void_p) -> c_void_p + row = c_void_p(_libtaos.taos_fetch_row(result)) + if row: + return row + return None + + +def taos_fetch_row(result, fields): + # type: (c_void_p, Array[TaosField]) -> tuple(c_void_p, int) + pblock = ctypes.c_void_p(0) + pblock = taos_fetch_row_raw(result) + if pblock: + num_of_rows = 1 + precision = taos_result_precision(result) + field_count = taos_field_count(result) + blocks = [None] * field_count + field_lens = taos_fetch_lengths(result, field_count) + for i in range(field_count): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = [None] + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, num_of_rows, field_lens[i], precision) + else: + return None, 0 + return blocks, abs(num_of_rows) + + +_libtaos.taos_free_result.argtypes = (c_void_p,) -def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row +def taos_free_result(result): + # type: (c_void_p) -> None + if result != None: + _libtaos.taos_free_result(result) + + +_libtaos.taos_field_count.restype = c_int +_libtaos.taos_field_count.argstype = (c_void_p,) + + +def taos_field_count(result): + # type: (c_void_p) -> int + return _libtaos.taos_field_count(result) + + +def taos_num_fields(result): + # type: (c_void_p) -> int + return _libtaos.taos_num_fields(result) + + +_libtaos.taos_fetch_fields.restype = c_void_p +_libtaos.taos_fetch_fields.argstype = (c_void_p,) + + +def taos_fetch_fields_raw(result): + # type: (c_void_p) -> c_void_p + return c_void_p(_libtaos.taos_fetch_fields(result)) + + +def taos_fetch_fields(result): + # type: (c_void_p) -> TaosFields + fields = taos_fetch_fields_raw(result) + count = taos_field_count(result) + return TaosFields(fields, count) + + +def taos_fetch_lengths(result, field_count=None): + # type: (c_void_p, int) -> Array[int] + """Make sure to call taos_fetch_row or taos_fetch_block before fetch_lengths""" + lens = _libtaos.taos_fetch_lengths(result) + if field_count == None: + field_count = taos_field_count(result) + if not lens: + raise OperationalError("field length empty, use taos_fetch_row/block before it") + return lens[:field_count] + + +def taos_result_precision(result): + # type: (c_void_p) -> c_int + return _libtaos.taos_result_precision(result) + + +_libtaos.taos_errno.restype = c_int +_libtaos.taos_errno.argstype = (c_void_p,) + + +def taos_errno(result): + # type: (ctypes.c_void_p) -> c_int + """Return the error number.""" + return _libtaos.taos_errno(result) + + +_libtaos.taos_errstr.restype = c_char_p +_libtaos.taos_errstr.argstype = (c_void_p,) + + +def taos_errstr(result=c_void_p(None)): + # type: (ctypes.c_void_p) -> str + """Return the error styring""" + return _libtaos.taos_errstr(result).decode("utf-8") + + +_libtaos.taos_stop_query.restype = None +_libtaos.taos_stop_query.argstype = (c_void_p,) + + +def taos_stop_query(result): + # type: (ctypes.c_void_p) -> None + """Stop current query""" + return _libtaos.taos_stop_query(result) + + +_libtaos.taos_load_table_info.restype = c_int +_libtaos.taos_load_table_info.argstype = (c_void_p, c_char_p) + + +def taos_load_table_info(connection, tables): + # type: (ctypes.c_void_p, str) -> None + """Stop current query""" + errno = _libtaos.taos_load_table_info(connection, c_char_p(tables.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + raise OperationalError(msg, errno) + + +_libtaos.taos_validate_sql.restype = c_int +_libtaos.taos_validate_sql.argstype = (c_void_p, c_char_p) + + +def taos_validate_sql(connection, sql): + # type: (ctypes.c_void_p, str) -> None | str + """Get taosd server info""" + errno = _libtaos.taos_validate_sql(connection, ctypes.c_char_p(sql.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + return msg + return None + + +_libtaos.taos_print_row.restype = c_int +_libtaos.taos_print_row.argstype = (c_char_p, c_void_p, c_void_p, c_int) + + +def taos_print_row(row, fields, num_fields, buffer_size=4096): + # type: (ctypes.c_void_p, ctypes.c_void_p | TaosFields, int, int) -> str + """Print an row to string""" + p = ctypes.create_string_buffer(buffer_size) + if isinstance(fields, TaosFields): + _libtaos.taos_print_row(p, row, fields.as_ptr(), num_fields) + else: + _libtaos.taos_print_row(p, row, fields, num_fields) + if p: + return p.value.decode("utf-8") + raise OperationalError("taos_print_row failed") + + +_libtaos.taos_select_db.restype = c_int +_libtaos.taos_select_db.argstype = (c_void_p, c_char_p) + + +def taos_select_db(connection, db): + # type: (ctypes.c_void_p, str) -> None + """Select database, eq to sql: use """ + res = _libtaos.taos_select_db(connection, ctypes.c_char_p(db.encode("utf-8"))) + if res != 0: + raise DatabaseError("select database error", res) + + +try: + _libtaos.taos_open_stream.restype = c_void_p + _libtaos.taos_open_stream.argstype = c_void_p, c_char_p, stream_callback_type, c_int64, c_void_p, Any +except: + pass + + +def taos_open_stream(connection, sql, callback, stime=0, param=None, callback2=None): + # type: (ctypes.c_void_p, str, stream_callback_type, c_int64, c_void_p, c_void_p) -> ctypes.pointer + if callback2 != None: + callback2 = stream_callback2_type(callback2) + """Open an stream""" + return c_void_p( + _libtaos.taos_open_stream( + connection, ctypes.c_char_p(sql.encode("utf-8")), stream_callback_type(callback), stime, param, callback2 + ) + ) + + +_libtaos.taos_close_stream.restype = None +_libtaos.taos_close_stream.argstype = (c_void_p,) + + +def taos_close_stream(stream): + # type: (c_void_p) -> None + """Open an stream""" + return _libtaos.taos_close_stream(stream) + + +_libtaos.taos_stmt_init.restype = c_void_p +_libtaos.taos_stmt_init.argstype = (c_void_p,) + + +def taos_stmt_init(connection): + # type: (c_void_p) -> (c_void_p) + """Create a statement query + @param(connection): c_void_p TAOS* + @rtype: c_void_p, *TAOS_STMT + """ + return c_void_p(_libtaos.taos_stmt_init(connection)) + + +_libtaos.taos_stmt_prepare.restype = c_int +_libtaos.taos_stmt_prepare.argstype = (c_void_p, c_char_p, c_int) + + +def taos_stmt_prepare(stmt, sql): + # type: (ctypes.c_void_p, str) -> None + """Prepare a statement query + @stmt: c_void_p TAOS_STMT* """ - return [ - None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint))[ - :abs(num_of_rows)]] + buffer = sql.encode("utf-8") + res = _libtaos.taos_stmt_prepare(stmt, ctypes.c_char_p(buffer), len(buffer)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +_libtaos.taos_stmt_close.restype = c_int +_libtaos.taos_stmt_close.argstype = (c_void_p,) -def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row + +def taos_stmt_close(stmt): + # type: (ctypes.c_void_p) -> None + """Close a statement query + @stmt: c_void_p TAOS_STMT* """ - return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_close(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_errstr.restype = c_char_p + _libtaos.taos_stmt_errstr.argstype = (c_void_p,) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_errstr" % taos_get_client_info()) -def _crow_bigint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row + +def taos_stmt_errstr(stmt): + # type: (ctypes.c_void_p) -> str + """Get error message from stetement query + @stmt: c_void_p TAOS_STMT* """ - return [ - None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint64))[ - :abs(num_of_rows)]] + err = c_char_p(_libtaos.taos_stmt_errstr(stmt)) + if err: + return err.value.decode("utf-8") + +try: + _libtaos.taos_stmt_set_tbname.restype = c_int + _libtaos.taos_stmt_set_tbname.argstype = (c_void_p, c_char_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname" % taos_get_client_info()) -def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C float row to python row + +def taos_stmt_set_tbname(stmt, name): + # type: (ctypes.c_void_p, str) -> None + """Set table name of a statement query if exists. + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_set_tbname(stmt, c_char_p(name.encode("utf-8"))) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_set_tbname_tags.restype = c_int + _libtaos.taos_stmt_set_tbname_tags.argstype = (c_void_p, c_char_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname_tags" % taos_get_client_info()) + + + +def taos_stmt_set_tbname_tags(stmt, name, tags): + # type: (c_void_p, str, c_void_p) -> None + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* + """ + res = _libtaos.taos_stmt_set_tbname_tags(stmt, ctypes.c_char_p(name.encode("utf-8")), tags) + + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C double row to python row +_libtaos.taos_stmt_is_insert.restype = c_int +_libtaos.taos_stmt_is_insert.argstype = (c_void_p, POINTER(c_int)) + + +def taos_stmt_is_insert(stmt): + # type: (ctypes.c_void_p) -> bool + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + is_insert = ctypes.c_int() + res = _libtaos.taos_stmt_is_insert(stmt, ctypes.byref(is_insert)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return is_insert == 0 + + +_libtaos.taos_stmt_num_params.restype = c_int +_libtaos.taos_stmt_num_params.argstype = (c_void_p, POINTER(c_int)) -def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row +def taos_stmt_num_params(stmt): + # type: (ctypes.c_void_p) -> int + """Params number of the current statement query. + @stmt: TAOS_STMT* """ - assert(nbytes is not None) - return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( - 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + num_params = ctypes.c_int() + res = _libtaos.taos_stmt_num_params(stmt, ctypes.byref(num_params)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return num_params.value -def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row +_libtaos.taos_stmt_bind_param.restype = c_int +_libtaos.taos_stmt_bind_param.argstype = (c_void_p, c_void_p) + + +def taos_stmt_bind_param(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - if num_of_rows >= 0: - tmpstr = ctypes.c_char_p(data) - res.append(tmpstr.value.decode()) - else: - res.append((ctypes.cast(data + nbytes * i, - ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) - except ValueError: - res.append(None) + # ptr = ctypes.cast(bind, POINTER(TaosBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_bind_param_batch.restype = c_int + _libtaos.taos_stmt_bind_param_batch.argstype = (c_void_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_param_batch" % taos_get_client_info()) - return res -def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row +def taos_stmt_bind_param_batch(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosMultiBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - rbyte = ctypes.cast( - data + nbytes * i, - ctypes.POINTER( - ctypes.c_short))[ - :1].pop() - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()[0:rbyte]) - except ValueError: - res.append(None) - return res - - -def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row + # ptr = ctypes.cast(bind, POINTER(TaosMultiBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param_batch(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_bind_single_param_batch.restype = c_int + _libtaos.taos_stmt_bind_single_param_batch.argstype = (c_void_p, c_void_p, c_int) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_single_param_batch" % taos_get_client_info()) + + +def taos_stmt_bind_single_param_batch(stmt, bind, col): + # type: (ctypes.c_void_p, Array[TaosMultiBind], c_int) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_MULTI_BIND* + @col: column index """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()) - except ValueError: - res.append(None) - return res - - -_CONVERT_FUNC = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -_CONVERT_FUNC_BLOCK = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python_block, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -# Corresponding TAOS_FIELD structure in C - - -class TaosField(ctypes.Structure): - _fields_ = [('name', ctypes.c_char * 65), - ('type', ctypes.c_char), - ('bytes', ctypes.c_short)] + res = _libtaos.taos_stmt_bind_single_param_batch(stmt, bind, col) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -# C interface class +_libtaos.taos_stmt_add_batch.restype = c_int +_libtaos.taos_stmt_add_batch.argstype = (c_void_p,) -def _load_taos_linux(): - return ctypes.CDLL('libtaos.so') +def taos_stmt_add_batch(stmt): + # type: (ctypes.c_void_p) -> None + """Add current params into batch + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_add_batch(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos_darwin(): - return ctypes.CDLL('libtaos.dylib') +_libtaos.taos_stmt_execute.restype = c_int +_libtaos.taos_stmt_execute.argstype = (c_void_p,) -def _load_taos_windows(): - return ctypes.windll.LoadLibrary('taos') +def taos_stmt_execute(stmt): + # type: (ctypes.c_void_p) -> None + """Execute a statement query + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_execute(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos(): - load_func = { - 'Linux': _load_taos_linux, - 'Darwin': _load_taos_darwin, - 'Windows': _load_taos_windows, - } - try: - return load_func[platform.system()]() - except: - sys.exit('unsupported platform to TDengine connector') +_libtaos.taos_stmt_use_result.restype = c_void_p +_libtaos.taos_stmt_use_result.argstype = (c_void_p,) -class CTaosInterface(object): - libtaos = _load_taos() - - libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) - libtaos.taos_init.restype = None - libtaos.taos_connect.restype = ctypes.c_void_p - # libtaos.taos_use_result.restype = ctypes.c_void_p - libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) - libtaos.taos_errstr.restype = ctypes.c_char_p - libtaos.taos_subscribe.restype = ctypes.c_void_p - libtaos.taos_consume.restype = ctypes.c_void_p - libtaos.taos_fetch_lengths.restype = ctypes.c_void_p - libtaos.taos_free_result.restype = None - libtaos.taos_errno.restype = ctypes.c_int - libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +def taos_stmt_use_result(stmt): + # type: (ctypes.c_void_p) -> None + """Get result of the statement. + @stmt: TAOS_STMT* + """ + result = c_void_p(_libtaos.taos_stmt_use_result(stmt)) + if result == None: + raise StatementError(taos_stmt_errstr(stmt)) + return result + +try: + _libtaos.taos_insert_lines.restype = c_int + _libtaos.taos_insert_lines.argstype = c_void_p, c_void_p, c_int +except AttributeError: + print("WARNING: libtaos(%s) does not support insert_lines" % taos_get_client_info()) + + + +def taos_insert_lines(connection, lines): + # type: (c_void_p, list[str] | tuple(str)) -> None + num_of_lines = len(lines) + lines = (c_char_p(line.encode("utf-8")) for line in lines) + lines_type = ctypes.c_char_p * num_of_lines + p_lines = lines_type(*lines) + errno = _libtaos.taos_insert_lines(connection, p_lines, num_of_lines) + if errno != 0: + raise LinesError("insert lines error", errno) + + +class CTaosInterface(object): def __init__(self, config=None): - ''' + """ Function to initialize the class @host : str, hostname to connect @user : str, username to connect to server @@ -298,304 +842,46 @@ class CTaosInterface(object): @config : str, config directory @rtype : None - ''' + """ if config is None: self._config = ctypes.c_char_p(None) else: try: - self._config = ctypes.c_char_p(config.encode('utf-8')) + self._config = ctypes.c_char_p(config.encode("utf-8")) except AttributeError: raise AttributeError("config is expected as a str") if config is not None: - CTaosInterface.libtaos.taos_options(3, self._config) + taos_options(3, self._config) - CTaosInterface.libtaos.taos_init() + taos_init() @property def config(self): - """ Get current config - """ + """Get current config""" return self._config - def connect( - self, - host=None, - user="root", - password="taosdata", - db=None, - port=0): - ''' + def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + """ Function to connect to server @rtype: c_void_p, TDengine handle - ''' - # host - try: - _host = ctypes.c_char_p(host.encode( - "utf-8")) if host is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("host is expected as a str") - - # user - try: - _user = ctypes.c_char_p(user.encode("utf-8")) - except AttributeError: - raise AttributeError("user is expected as a str") - - # password - try: - _password = ctypes.c_char_p(password.encode("utf-8")) - except AttributeError: - raise AttributeError("password is expected as a str") - - # db - try: - _db = ctypes.c_char_p( - db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("db is expected as a str") - - # port - try: - _port = ctypes.c_int(port) - except TypeError: - raise TypeError("port is expected as an int") - - connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( - _host, _user, _password, _db, _port)) - - if connection.value is None: - print('connect to TDengine failed') - raise ConnectionError("connect to TDengine failed") - # sys.exit(1) - # else: - # print('connect to TDengine success') - - return connection - - @staticmethod - def close(connection): - '''Close the TDengine handle - ''' - CTaosInterface.libtaos.taos_close(connection) - # print('connection is closed') - - @staticmethod - def query(connection, sql): - '''Run SQL - - @sql: str, sql string to run - - @rtype: 0 on success and -1 on failure - ''' - try: - return CTaosInterface.libtaos.taos_query( - connection, ctypes.c_char_p(sql.encode('utf-8'))) - except AttributeError: - raise AttributeError("sql is expected as a string") - # finally: - # CTaosInterface.libtaos.close(connection) - - @staticmethod - def affectedRows(result): - """The affected rows after runing query - """ - return CTaosInterface.libtaos.taos_affected_rows(result) - - @staticmethod - def insertLines(connection, lines): - ''' - insert through lines protocol - @lines: list of str - @rtype: tsdb error codes - ''' - numLines = len(lines) - c_lines_type = ctypes.c_char_p*numLines - c_lines = c_lines_type() - for i in range(numLines): - c_lines[i] = ctypes.c_char_p(lines[i].encode('utf-8')) - return CTaosInterface.libtaos.taos_insert_lines(connection, c_lines, ctypes.c_int(numLines)) - - @staticmethod - def subscribe(connection, restart, topic, sql, interval): - """Create a subscription - @restart boolean, - @sql string, sql statement for data query, must be a 'select' statement. - @topic string, name of this subscription - """ - return ctypes.c_void_p(CTaosInterface.libtaos.taos_subscribe( - connection, - 1 if restart else 0, - ctypes.c_char_p(topic.encode('utf-8')), - ctypes.c_char_p(sql.encode('utf-8')), - None, - None, - interval)) - - @staticmethod - def consume(sub): - """Consume data of a subscription - """ - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_consume(sub)) - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.libtaos.taos_num_fields(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - return result, fields - - @staticmethod - def unsubscribe(sub, keepProgress): - """Cancel a subscription - """ - CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) - - @staticmethod - def useResult(result): - '''Use result after calling self.query - ''' - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - - return fields - - @staticmethod - def fetchBlock(result, fields): - pblock = ctypes.c_void_p(0) - num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - result, ctypes.byref(pblock)) - if num_of_rows == 0: - return None, 0 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: - raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - - return blocks, abs(num_of_rows) - - @staticmethod - def fetchRow(result, fields): - pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock: - num_of_rows = 1 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError( - "Invalid data type returned from database") - if data is None: - blocks[i] = [None] - else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - else: - return None, 0 - return blocks, abs(num_of_rows) - - @staticmethod - def freeResult(result): - CTaosInterface.libtaos.taos_free_result(result) - result.value = None - - @staticmethod - def fieldsCount(result): - return CTaosInterface.libtaos.taos_field_count(result) - - @staticmethod - def fetchFields(result): - return CTaosInterface.libtaos.taos_fetch_fields(result) - - # @staticmethod - # def fetchRow(result, fields): - # l = [] - # row = CTaosInterface.libtaos.taos_fetch_row(result) - # if not row: - # return None - - # for i in range(len(fields)): - # l.append(CTaosInterface.getDataValue( - # row[i], fields[i]['type'], fields[i]['bytes'])) - - # return tuple(l) - - # @staticmethod - # def getDataValue(data, dtype, byte): - # ''' - # ''' - # if not data: - # return None - - # if (dtype == CTaosInterface.TSDB_DATA_TYPE_BOOL): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TINYINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_SMALLINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): - # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): - # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') - - @staticmethod - def errno(result): - """Return the error number. - """ - return CTaosInterface.libtaos.taos_errno(result) - - @staticmethod - def errStr(result): - """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') + return taos_connect(host, user, password, db, port) -if __name__ == '__main__': +if __name__ == "__main__": cinter = CTaosInterface() conn = cinter.connect() - result = cinter.query(conn, 'show databases') + result = cinter.query(conn, "show databases") - print('Query Affected rows: {}'.format(cinter.affectedRows(result))) + print("Query Affected rows: {}".format(cinter.affected_rows(result))) - fields = CTaosInterface.useResult(result) + fields = taos_fetch_fields_raw(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, fields) + data, num_of_rows = taos_fetch_block(result, fields) print(data) - cinter.freeResult(result) + cinter.free_result(result) cinter.close(conn) diff --git a/src/connector/python/taos/connection.py b/src/connector/python/taos/connection.py index 88d06cd7186018788aeb25c982fc205441193cb8..7857c8c706dbe27fd9440e6bf2eb698b6822650e 100644 --- a/src/connector/python/taos/connection.py +++ b/src/connector/python/taos/connection.py @@ -1,11 +1,15 @@ -from .cursor import TDengineCursor -from .subscription import TDengineSubscription -from .cinterface import CTaosInterface +# encoding:UTF-8 +from types import FunctionType +from .cinterface import * +from .cursor import TaosCursor +from .subscription import TaosSubscription +from .statement import TaosStmt +from .stream import TaosStream +from .result import * -class TDengineConnection(object): - """ TDengine connection object - """ +class TaosConnection(object): + """TDengine connection object""" def __init__(self, *args, **kwargs): self._conn = None @@ -21,63 +25,130 @@ class TDengineConnection(object): def config(self, **kwargs): # host - if 'host' in kwargs: - self._host = kwargs['host'] + if "host" in kwargs: + self._host = kwargs["host"] # user - if 'user' in kwargs: - self._user = kwargs['user'] + if "user" in kwargs: + self._user = kwargs["user"] # password - if 'password' in kwargs: - self._password = kwargs['password'] + if "password" in kwargs: + self._password = kwargs["password"] # database - if 'database' in kwargs: - self._database = kwargs['database'] + if "database" in kwargs: + self._database = kwargs["database"] # port - if 'port' in kwargs: - self._port = kwargs['port'] + if "port" in kwargs: + self._port = kwargs["port"] # config - if 'config' in kwargs: - self._config = kwargs['config'] + if "config" in kwargs: + self._config = kwargs["config"] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect( - self._host, - self._user, - self._password, - self._database, - self._port) + self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) def close(self): - """Close current connection. - """ - return CTaosInterface.close(self._conn) - - def subscribe(self, restart, topic, sql, interval): - """Create a subscription. - """ + """Close current connection.""" + if self._conn: + taos_close(self._conn) + self._conn = None + + @property + def client_info(self): + # type: () -> str + return taos_get_client_info() + + @property + def server_info(self): + # type: () -> str + return taos_get_server_info(self._conn) + + def select_db(self, database): + # type: (str) -> None + taos_select_db(self._conn, database) + + def execute(self, sql): + # type: (str) -> None + """Simplely execute sql ignoring the results""" + res = taos_query(self._conn, sql) + taos_free_result(res) + + def query(self, sql): + # type: (str) -> TaosResult + result = taos_query(self._conn, sql) + return TaosResult(result, True, self) + + def query_a(self, sql, callback, param): + # type: (str, async_query_callback_type, c_void_p) -> None + """Asynchronously query a sql with callback function""" + taos_query_a(self._conn, sql, callback, param) + + def subscribe(self, restart, topic, sql, interval, callback=None, param=None): + # type: (bool, str, str, int, subscribe_callback_type, c_void_p) -> TaosSubscription + """Create a subscription.""" if self._conn is None: return None - sub = CTaosInterface.subscribe( - self._conn, restart, topic, sql, interval) - return TDengineSubscription(sub) + sub = taos_subscribe(self._conn, restart, topic, sql, interval, callback, param) + return TaosSubscription(sub, callback != None) - def insertLines(self, lines): - """ - insert lines through line protocol - """ + def statement(self, sql=None): + # type: (str | None) -> TaosStmt if self._conn is None: return None - return CTaosInterface.insertLines(self._conn, lines) - - def cursor(self): - """Return a new Cursor object using the connection. + stmt = taos_stmt_init(self._conn) + if sql != None: + taos_stmt_prepare(stmt, sql) + + return TaosStmt(stmt) + + def load_table_info(self, tables): + # type: (str) -> None + taos_load_table_info(self._conn, tables) + + def stream(self, sql, callback, stime=0, param=None, callback2=None): + # type: (str, Callable[[Any, TaosResult, TaosRows], None], int, Any, c_void_p) -> TaosStream + # cb = cast(callback, stream_callback_type) + # ref = byref(cb) + + stream = taos_open_stream(self._conn, sql, callback, stime, param, callback2) + return TaosStream(stream) + + def insert_lines(self, lines): + # type: (list[str]) -> None + """Line protocol and schemaless support + + ## Example + + ```python + import taos + conn = taos.connect() + conn.exec("drop database if exists test") + conn.select_db("test") + lines = [ + 'ste,t2=5,t3=L"ste" c1=true,c2=4,c3="string" 1626056811855516532', + ] + conn.insert_lines(lines) + ``` + + ## Exception + + ```python + try: + conn.insert_lines(lines) + except SchemalessError as err: + print(err) + ``` """ - return TDengineCursor(self) + return taos_insert_lines(self._conn, lines) + + def cursor(self): + # type: () -> TaosCursor + """Return a new Cursor object using the connection.""" + return TaosCursor(self) def commit(self): """Commit any pending transaction to the database. @@ -87,17 +158,18 @@ class TDengineConnection(object): pass def rollback(self): - """Void functionality - """ + """Void functionality""" pass def clear_result_set(self): - """Clear unused result set on this connection. - """ + """Clear unused result set on this connection.""" pass + def __del__(self): + self.close() + if __name__ == "__main__": - conn = TDengineConnection(host='192.168.1.107') + conn = TaosConnection() conn.close() print("Hello world") diff --git a/src/connector/python/taos/constants.py b/src/connector/python/taos/constants.py index 85689b02db76b0032558b983d8ae8d9297229c42..b500df627c22919e7aab964504ccbe50c573c1c5 100644 --- a/src/connector/python/taos/constants.py +++ b/src/connector/python/taos/constants.py @@ -1,12 +1,11 @@ +# encoding:UTF-8 + """Constants in TDengine python """ -from .dbapi import * - - class FieldType(object): - """TDengine Field Types - """ + """TDengine Field Types""" + # type_code C_NULL = 0 C_BOOL = 1 @@ -34,9 +33,9 @@ class FieldType(object): C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 C_BIGINT_UNSIGNED_NULL = 18446744073709551615 - C_FLOAT_NULL = float('nan') - C_DOUBLE_NULL = float('nan') - C_BINARY_NULL = bytearray([int('0xff', 16)]) + C_FLOAT_NULL = float("nan") + C_DOUBLE_NULL = float("nan") + C_BINARY_NULL = bytearray([int("0xff", 16)]) # Timestamp precision definition C_TIMESTAMP_MILLI = 0 C_TIMESTAMP_MICRO = 1 diff --git a/src/connector/python/taos/cursor.py b/src/connector/python/taos/cursor.py index d443ec95d07bd970da96c6280674dd5c9477a38f..5d21ff95af5d81367e7143d001cc688d90877b67 100644 --- a/src/connector/python/taos/cursor.py +++ b/src/connector/python/taos/cursor.py @@ -1,18 +1,18 @@ -from .cinterface import CTaosInterface +# encoding:UTF-8 +from .cinterface import * from .error import * from .constants import FieldType +from .result import * -# querySeqNum = 0 - -class TDengineCursor(object): +class TaosCursor(object): """Database cursor which is used to manage the context of a fetch operation. Attributes: .description: Read-only attribute consists of 7-item sequences: - > name (mondatory) - > type_code (mondatory) + > name (mandatory) + > type_code (mandatory) > display_size > internal_size > precision @@ -55,8 +55,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow( - self._result, self._fields) + block, self._block_rows = taos_fetch_row(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -69,20 +68,17 @@ class TDengineCursor(object): @property def description(self): - """Return the description of the object. - """ + """Return the description of the object.""" return self._description @property def rowcount(self): - """Return the rowcount of the object - """ + """Return the rowcount of the object""" return self._rowcount @property def affected_rows(self): - """Return the rowcount of insertion - """ + """Return the rowcount of insertion""" return self._affected_rows def callproc(self, procname, *args): @@ -96,8 +92,7 @@ class TDengineCursor(object): self._logfile = logfile def close(self): - """Close the cursor. - """ + """Close the cursor.""" if self._connection is None: return False @@ -107,8 +102,7 @@ class TDengineCursor(object): return True def execute(self, operation, params=None): - """Prepare and execute a database operation (query or command). - """ + """Prepare and execute a database operation (query or command).""" if not operation: return None @@ -124,104 +118,91 @@ class TDengineCursor(object): # global querySeqNum # querySeqNum += 1 - # localSeqNum = querySeqNum # avoid raice condition + # localSeqNum = querySeqNum # avoid race condition # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) - self._result = CTaosInterface.query(self._connection._conn, stmt) + self._result = taos_query(self._connection._conn, stmt) # print(" << Query ({}) Exec Done".format(localSeqNum)) - if (self._logfile): + if self._logfile: with open(self._logfile, "a") as logfile: logfile.write("%s;\n" % operation) - errno = CTaosInterface.libtaos.taos_errno(self._result) - if errno == 0: - if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows( - self._result) - return CTaosInterface.affectedRows(self._result) - else: - self._fields = CTaosInterface.useResult( - self._result) - return self._handle_result() + if taos_field_count(self._result) == 0: + affected_rows = taos_affected_rows(self._result) + self._affected_rows += affected_rows + return affected_rows else: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + self._fields = taos_fetch_fields(self._result) + return self._handle_result() def executemany(self, operation, seq_of_parameters): - """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. - """ + """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.""" pass def fetchone(self): - """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. - """ + """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available.""" pass def fetchmany(self): pass def istype(self, col, dataType): - if (dataType.upper() == "BOOL"): - if (self._description[col][1] == FieldType.C_BOOL): + if dataType.upper() == "BOOL": + if self._description[col][1] == FieldType.C_BOOL: return True - if (dataType.upper() == "TINYINT"): - if (self._description[col][1] == FieldType.C_TINYINT): + if dataType.upper() == "TINYINT": + if self._description[col][1] == FieldType.C_TINYINT: return True - if (dataType.upper() == "TINYINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + if dataType.upper() == "TINYINT UNSIGNED": + if self._description[col][1] == FieldType.C_TINYINT_UNSIGNED: return True - if (dataType.upper() == "SMALLINT"): - if (self._description[col][1] == FieldType.C_SMALLINT): + if dataType.upper() == "SMALLINT": + if self._description[col][1] == FieldType.C_SMALLINT: return True - if (dataType.upper() == "SMALLINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + if dataType.upper() == "SMALLINT UNSIGNED": + if self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED: return True - if (dataType.upper() == "INT"): - if (self._description[col][1] == FieldType.C_INT): + if dataType.upper() == "INT": + if self._description[col][1] == FieldType.C_INT: return True - if (dataType.upper() == "INT UNSIGNED"): - if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + if dataType.upper() == "INT UNSIGNED": + if self._description[col][1] == FieldType.C_INT_UNSIGNED: return True - if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_BIGINT): + if dataType.upper() == "BIGINT": + if self._description[col][1] == FieldType.C_BIGINT: return True - if (dataType.upper() == "BIGINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): + if dataType.upper() == "BIGINT UNSIGNED": + if self._description[col][1] == FieldType.C_BIGINT_UNSIGNED: return True - if (dataType.upper() == "FLOAT"): - if (self._description[col][1] == FieldType.C_FLOAT): + if dataType.upper() == "FLOAT": + if self._description[col][1] == FieldType.C_FLOAT: return True - if (dataType.upper() == "DOUBLE"): - if (self._description[col][1] == FieldType.C_DOUBLE): + if dataType.upper() == "DOUBLE": + if self._description[col][1] == FieldType.C_DOUBLE: return True - if (dataType.upper() == "BINARY"): - if (self._description[col][1] == FieldType.C_BINARY): + if dataType.upper() == "BINARY": + if self._description[col][1] == FieldType.C_BINARY: return True - if (dataType.upper() == "TIMESTAMP"): - if (self._description[col][1] == FieldType.C_TIMESTAMP): + if dataType.upper() == "TIMESTAMP": + if self._description[col][1] == FieldType.C_TIMESTAMP: return True - if (dataType.upper() == "NCHAR"): - if (self._description[col][1] == FieldType.C_NCHAR): + if dataType.upper() == "NCHAR": + if self._description[col][1] == FieldType.C_NCHAR: return True return False def fetchall_row(self): - """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. - """ + """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.""" if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_row(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -230,19 +211,16 @@ class TDengineCursor(object): return list(map(tuple, zip(*buffer))) def fetchall(self): - if self._result is None or self._fields is None: + if self._result is None: raise OperationalError("Invalid use of fetchall") - - buffer = [[] for i in range(len(self._fields))] + fields = self._fields if self._fields is not None else taos_fetch_fields(self._result) + buffer = [[] for i in range(len(fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -250,9 +228,12 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def stop_query(self): + if self._result != None: + taos_stop_query(self._result) + def nextset(self): - """ - """ + """ """ pass def setinputsize(self, sizes): @@ -262,12 +243,11 @@ class TDengineCursor(object): pass def _reset_result(self): - """Reset the result to unused version. - """ + """Reset the result to unused version.""" self._description = [] self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + taos_free_result(self._result) self._result = None self._fields = None self._block = None @@ -276,11 +256,12 @@ class TDengineCursor(object): self._affected_rows = 0 def _handle_result(self): - """Handle the return result from query. - """ + """Handle the return result from query.""" self._description = [] for ele in self._fields: - self._description.append( - (ele['name'], ele['type'], None, None, None, None, False)) + self._description.append((ele["name"], ele["type"], None, None, None, None, False)) return self._result + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/dbapi.py b/src/connector/python/taos/dbapi.py deleted file mode 100644 index 594681ada953abf388e503c23199043cf686e1a3..0000000000000000000000000000000000000000 --- a/src/connector/python/taos/dbapi.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Type Objects and Constructors. -""" - -import time -import datetime - - -class DBAPITypeObject(object): - def __init__(self, *values): - self.values = values - - def __com__(self, other): - if other in self.values: - return 0 - if other < self.values: - return 1 - else: - return -1 - - -Date = datetime.date -Time = datetime.time -Timestamp = datetime.datetime - - -def DataFromTicks(ticks): - return Date(*time.localtime(ticks)[:3]) - - -def TimeFromTicks(ticks): - return Time(*time.localtime(ticks)[3:6]) - - -def TimestampFromTicks(ticks): - return Timestamp(*time.localtime(ticks)[:6]) - - -Binary = bytes - -# STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) -# BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) -# NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) -# DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() diff --git a/src/connector/python/taos/error.py b/src/connector/python/taos/error.py index c584badce8320cd35dc81e8f6b613c56163b1a29..a30adbb162f1c194bdfcf4cca5c43f01107a9776 100644 --- a/src/connector/python/taos/error.py +++ b/src/connector/python/taos/error.py @@ -1,66 +1,86 @@ +# encoding:UTF-8 """Python exceptions """ class Error(Exception): - def __init__(self, msg=None, errno=None): + def __init__(self, msg=None, errno=0xffff): self.msg = msg - self._full_msg = self.msg self.errno = errno + self._full_msg = "[0x%04x]: %s" % (self.errno & 0xffff, self.msg) def __str__(self): return self._full_msg class Warning(Exception): - """Exception raised for important warnings like data truncations while inserting. - """ + """Exception raised for important warnings like data truncations while inserting.""" + pass class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. - """ + """Exception raised for errors that are related to the database interface rather than the database itself.""" + pass class DatabaseError(Error): - """Exception raised for errors that are related to the database. - """ + """Exception raised for errors that are related to the database.""" + pass +class ConnectionError(Error): + """Exceptin raised for connection failed""" + pass class DataError(DatabaseError): - """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. - """ + """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range.""" + pass class OperationalError(DatabaseError): - """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer - """ + """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer""" + pass class IntegrityError(DatabaseError): - """Exception raised when the relational integrity of the database is affected. - """ + """Exception raised when the relational integrity of the database is affected.""" + pass class InternalError(DatabaseError): - """Exception raised when the database encounters an internal error. - """ + """Exception raised when the database encounters an internal error.""" + pass class ProgrammingError(DatabaseError): - """Exception raised for programming errors. - """ + """Exception raised for programming errors.""" + pass class NotSupportedError(DatabaseError): - """Exception raised in case a method or database API was used which is not supported by the database,. - """ + """Exception raised in case a method or database API was used which is not supported by the database,.""" + pass + + +class StatementError(DatabaseError): + """Exception raised in STMT API.""" + + pass + +class ResultError(DatabaseError): + """Result related APIs.""" + + pass + +class LinesError(DatabaseError): + """taos_insert_lines errors.""" + + pass \ No newline at end of file diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py new file mode 100644 index 0000000000000000000000000000000000000000..445cd8afdba6f2512c73be95c9b0dbd8dc00da8a --- /dev/null +++ b/src/connector/python/taos/field.py @@ -0,0 +1,302 @@ +# encoding:UTF-8 +import ctypes +import math +import datetime +from ctypes import * + +from .constants import FieldType +from .error import * + +_datetime_epoch = datetime.datetime.fromtimestamp(0) + +def _convert_millisecond_to_datetime(milli): + return _datetime_epoch + datetime.timedelta(seconds=milli / 1000.0) + + +def _convert_microsecond_to_datetime(micro): + return _datetime_epoch + datetime.timedelta(seconds=micro / 1000000.0) + + +def _convert_nanosecond_to_datetime(nanosec): + return nanosec + + +def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + _timestamp_converter = _convert_millisecond_to_datetime + if precision == FieldType.C_TIMESTAMP_MILLI: + _timestamp_converter = _convert_millisecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_MICRO: + _timestamp_converter = _convert_microsecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_NANO: + _timestamp_converter = _convert_nanosecond_to_datetime + else: + raise DatabaseError("Unknown precision returned from database") + + return [ + None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ubyte))[: abs(num_of_rows)] + ] + + +def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[: abs(num_of_rows)] + ] + + +def _crow_smallint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ushort))[: abs(num_of_rows)] + ] + + +def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[: abs(num_of_rows)] + ] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[: abs(num_of_rows)] + ] + + +def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bigint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint64))[: abs(num_of_rows)] + ] + + +def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C float row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[: abs(num_of_rows)] + ] + + +def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C double row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[: abs(num_of_rows)] + ] + + +def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + return [ + None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode("utf-8") + for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[: abs(num_of_rows)] + ] + + +def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + if num_of_rows >= 0: + tmpstr = ctypes.c_char_p(data) + res.append(tmpstr.value.decode()) + else: + res.append( + ( + ctypes.cast( + data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4)), + ) + )[0].value + ) + except ValueError: + res.append(None) + + return res + + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast(data + nbytes * i, ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + return res + + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) + except ValueError: + res.append(None) + return res + + +CONVERT_FUNC = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +# Corresponding TAOS_FIELD structure in C + + +class TaosField(ctypes.Structure): + _fields_ = [ + ("_name", ctypes.c_char * 65), + ("_type", ctypes.c_uint8), + ("_bytes", ctypes.c_uint16), + ] + + @property + def name(self): + return self._name.decode("utf-8") + + @property + def length(self): + """alias to self.bytes""" + return self._bytes + + @property + def bytes(self): + return self._bytes + + @property + def type(self): + return self._type + + def __dict__(self): + return {"name": self.name, "type": self.type, "bytes": self.length} + + def __str__(self): + return "{name: %s, type: %d, bytes: %d}" % (self.name, self.type, self.length) + + def __getitem__(self, item): + return getattr(self, item) + + +class TaosFields(object): + def __init__(self, fields, count): + if isinstance(fields, c_void_p): + self._fields = cast(fields, POINTER(TaosField)) + if isinstance(fields, POINTER(TaosField)): + self._fields = fields + self._count = count + self._iter = 0 + + def as_ptr(self): + return self._fields + + @property + def count(self): + return self._count + + @property + def fields(self): + return self._fields + + def __next__(self): + return self._next_field() + + def next(self): + return self._next_field() + + def _next_field(self): + if self._iter < self.count: + field = self._fields[self._iter] + self._iter += 1 + return field + else: + raise StopIteration + + def __getitem__(self, item): + return self._fields[item] + + def __iter__(self): + return self + + def __len__(self): + return self.count diff --git a/src/connector/python/taos/precision.py b/src/connector/python/taos/precision.py new file mode 100644 index 0000000000000000000000000000000000000000..d67da592cce6d2121ec8f2eed78a30d6fa0c446b --- /dev/null +++ b/src/connector/python/taos/precision.py @@ -0,0 +1,12 @@ +class PrecisionEnum(object): + """Precision enums""" + + Milliseconds = 0 + Microseconds = 1 + Nanoseconds = 2 + + +class PrecisionError(Exception): + """Python datetime does not support nanoseconds error""" + + pass diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py new file mode 100644 index 0000000000000000000000000000000000000000..81151733615d1b7fdc3318b6e53888ae39d32b14 --- /dev/null +++ b/src/connector/python/taos/result.py @@ -0,0 +1,245 @@ +from .cinterface import * + +# from .connection import TaosConnection +from .error import * + + +class TaosResult(object): + """TDengine result interface""" + + def __init__(self, result, close_after=False, conn=None): + # type: (c_void_p, bool, TaosConnection) -> TaosResult + # to make the __del__ order right + self._conn = conn + self._close_after = close_after + self._result = result + self._fields = None + self._field_count = None + self._precision = None + + self._block = None + self._block_length = None + self._row_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + # fetch next row + return self._next_row() + + def _next_row(self): + if self._result is None or self.fields is None: + raise OperationalError("Invalid use of fetch iterator") + + if self._block == None or self._block_iter >= self._block_length: + self._block, self._block_length = self.fetch_block() + self._block_iter = 0 + # self._row_count += self._block_length + + raw = self._block[self._block_iter] + self._block_iter += 1 + return raw + + @property + def fields(self): + """fields definitions of the current result""" + if self._result is None: + raise ResultError("no result object setted") + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + + return self._fields + + @property + def field_count(self): + """Field count of the current result, eq to taos_field_count(result)""" + return self.fields.count + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._row_count + + @property + def precision(self): + if self._precision == None: + self._precision = taos_result_precision(self._result) + return self._precision + + @property + def affected_rows(self): + return taos_affected_rows(self._result) + + # @property + def field_lengths(self): + return taos_fetch_lengths(self._result, self.field_count) + + def rows_iter(self, num_of_rows=None): + return TaosRows(self, num_of_rows) + + def blocks_iter(self): + return TaosBlocks(self) + + def fetch_block(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + + block, length = taos_fetch_block_raw(self._result) + if length == 0: + raise StopIteration + precision = self.precision + field_count = self.field_count + fields = self.fields + blocks = [None] * field_count + lengths = self.field_lengths() + for i in range(field_count): + data = ctypes.cast(block, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i].type](data, length, lengths[i], precision) + + return list(map(tuple, zip(*blocks))), length + + def fetch_all(self): + if self._result is None: + raise OperationalError("Invalid use of fetchall") + + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + buffer = [[] for i in range(len(self._fields))] + self._row_count = 0 + while True: + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) + if errno != 0: + raise ProgrammingError(taos_errstr(self._result), errno) + if num_of_fields == 0: + break + self._row_count += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetch_rows_a(self, callback, param): + taos_fetch_rows_a(self._result, callback, param) + + def stop_query(self): + return taos_stop_query(self._result) + + def errno(self): + """**DO NOT** use this directly unless you know what you are doing""" + return taos_errno(self._result) + + def errstr(self): + return taos_errstr(self._result) + + def check_error(self, errno=None, close=True): + if errno == None: + errno = self.errno() + if errno != 0: + msg = self.errstr() + self.close() + raise OperationalError(msg, errno) + + def close(self): + """free result object.""" + if self._result != None and self._close_after: + taos_free_result(self._result) + self._result = None + self._fields = None + self._field_count = None + self._field_lengths = None + + def __del__(self): + self.close() + + +class TaosRows: + """TDengine result rows iterator""" + + def __init__(self, result, num_of_rows=None): + self._result = result + self._num_of_rows = num_of_rows + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + return self._next_row() + + def _next_row(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + if self._num_of_rows != None and self._num_of_rows <= self._result._row_count: + raise StopIteration + + row = taos_fetch_row_raw(self._result._result) + if not row: + raise StopIteration + self._result._row_count += 1 + return TaosRow(self._result, row) + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._result._row_count + + +class TaosRow: + def __init__(self, result, row): + self._result = result + self._row = row + + def __str__(self): + return taos_print_row(self._row, self._result.fields, self._result.field_count) + + def __call__(self): + return self.as_tuple() + + def _astuple(self): + return self.as_tuple() + + def __iter__(self): + return self.as_tuple() + + def as_ptr(self): + return self._row + + def as_tuple(self): + precision = self._result.precision + field_count = self._result.field_count + blocks = [None] * field_count + fields = self._result.fields + field_lens = self._result.field_lengths() + for i in range(field_count): + data = ctypes.cast(self._row, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = None + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, 1, field_lens[i], precision)[0] + return tuple(blocks) + + +class TaosBlocks: + """TDengine result blocks iterator""" + + def __init__(self, result): + self._result = result + + def __iter__(self): + return self + + def __next__(self): + return self._result.fetch_block() + + def next(self): + return self._result.fetch_block() diff --git a/src/connector/python/taos/statement.py b/src/connector/python/taos/statement.py new file mode 100644 index 0000000000000000000000000000000000000000..155e98173b7f920640aa84d0fcda618d2669bb1e --- /dev/null +++ b/src/connector/python/taos/statement.py @@ -0,0 +1,85 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStmt(object): + """TDengine STMT interface""" + + def __init__(self, stmt, conn = None): + self._conn = conn + self._stmt = stmt + + def set_tbname(self, name): + """Set table name if needed. + + Note that the set_tbname* method should only used in insert statement + """ + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname(self._stmt, name) + + def prepare(self, sql): + # type: (str) -> None + taos_stmt_prepare(self._stmt, sql) + + def set_tbname_tags(self, name, tags): + # type: (str, Array[TaosBind]) -> None + """Set table name with tags, tags is array of BindParams""" + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname_tags(self._stmt, name, tags) + + def bind_param(self, params, add_batch=True): + # type: (Array[TaosBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param(self._stmt, params) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def bind_param_batch(self, binds, add_batch=True): + # type: (Array[TaosMultiBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param_batch(self._stmt, binds) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def add_batch(self): + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_add_batch(self._stmt) + + def execute(self): + if self._stmt is None: + raise StatementError("Invalid use of execute") + taos_stmt_execute(self._stmt) + + def use_result(self): + result = taos_stmt_use_result(self._stmt) + return TaosResult(result) + + def close(self): + """Close stmt.""" + if self._stmt is None: + return + taos_stmt_close(self._stmt) + self._stmt = None + + def __del__(self): + self.close() + + +if __name__ == "__main__": + from taos.connection import TaosConnection + + conn = TaosConnection() + + stmt = conn.statement("select * from log.log limit 10") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + stmt.close() + conn.close() diff --git a/src/connector/python/taos/stream.py b/src/connector/python/taos/stream.py new file mode 100644 index 0000000000000000000000000000000000000000..fe3c8c85e3279511972293882224bf20c30dfa64 --- /dev/null +++ b/src/connector/python/taos/stream.py @@ -0,0 +1,22 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStream(object): + """TDengine Stream interface""" + + def __init__(self, stream): + self._raw = stream + + def as_ptr(self): + return self._raw + + def close(self): + """Close stmt.""" + if self._raw is not None: + taos_close_stream(self._raw) + self._raw = None + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/subscription.py b/src/connector/python/taos/subscription.py index 270d9de09217fc58a389981a3542698dd1c0428a..3c6958b6f8d55791b9753a84a4bbd7653bdae780 100644 --- a/src/connector/python/taos/subscription.py +++ b/src/connector/python/taos/subscription.py @@ -1,49 +1,41 @@ -from .cinterface import CTaosInterface +from taos.result import TaosResult +from .cinterface import * from .error import * -class TDengineSubscription(object): - """TDengine subscription object - """ +class TaosSubscription(object): + """TDengine subscription object""" - def __init__(self, sub): + def __init__(self, sub, with_callback = False): self._sub = sub + self._with_callback = with_callback def consume(self): - """Consume rows of a subscription - """ + """Consume rows of a subscription""" if self._sub is None: raise OperationalError("Invalid use of consume") - - result, fields = CTaosInterface.consume(self._sub) - buffer = [[] for i in range(len(fields))] - while True: - block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: - break - for i in range(len(fields)): - buffer[i].extend(block[i]) - - self.fields = fields - return list(map(tuple, zip(*buffer))) + if self._with_callback: + raise OperationalError("DONOT use consume method in an subscription with callback") + result = taos_consume(self._sub) + return TaosResult(result) def close(self, keepProgress=True): - """Close the Subscription. - """ + """Close the Subscription.""" if self._sub is None: return False - CTaosInterface.unsubscribe(self._sub, keepProgress) + taos_unsubscribe(self._sub, keepProgress) + self._sub = None return True + + def __del__(self): + self.close() + +if __name__ == "__main__": + from .connection import TaosConnection -if __name__ == '__main__': - from .connection import TDengineConnection - conn = TDengineConnection( - host="127.0.0.1", - user="root", - password="taosdata", - database="test") + conn = TaosConnection(host="127.0.0.1", user="root", password="taosdata", database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) diff --git a/src/connector/python/taos/timestamp.py b/src/connector/python/taos/timestamp.py new file mode 100644 index 0000000000000000000000000000000000000000..ab5679fdf12e2942aa94f76716ff98e6d2a88d69 --- /dev/null +++ b/src/connector/python/taos/timestamp.py @@ -0,0 +1,17 @@ + +class TimestampType(object): + """Choose which type that parsing TDengine timestamp data to + + - DATETIME: use python datetime.datetime, note that it does not support nanosecond precision, + and python taos will use raw c_int64 as a fallback for nanosecond results. + - NUMPY: use numpy.datetime64 type. + - RAW: use raw c_int64. + - TAOS: use taos' TaosTimestamp. + """ + DATETIME = 0, + NUMPY = 1, + RAW = 2, + TAOS = 3, + +class TaosTimestamp: + pass diff --git a/src/connector/python/tests/test_ctaos.py b/src/connector/python/tests/test_ctaos.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9566931f2b29dcbdc8646d2f087ebf40e716cc --- /dev/null +++ b/src/connector/python/tests/test_ctaos.py @@ -0,0 +1,162 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * + +import time +import datetime +import pytest + +@pytest.fixture +def conn(): + return CTaosInterface().connect() + + +def test_simple(conn, caplog): + dbname = "pytest_ctaos_simple" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, level tinyint, content binary(100), ipaddr binary(134))", + ) + taos_free_result(res) + + res = taos_query(conn, "insert into log values(now, 1, 'hello', 'test')") + taos_free_result(res) + + res = taos_query(conn, "select level,content,ipaddr from log limit 1") + + fields = taos_fetch_fields_raw(res) + field_count = taos_field_count(res) + + fields = taos_fetch_fields(res) + for field in fields: + print(field) + + # field_lengths = taos_fetch_lengths(res, field_count) + # if not field_lengths: + # raise "fetch lengths error" + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, field_count) + assert rowstr == "1 hello test" + + row, num = taos_fetch_row(res, fields) + print(row) + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + + +def test_stmt(conn, caplog): + dbname = "pytest_ctaos_stmt" + try: + res = taos_query(conn, "drop database if exists %s" % dbname) + taos_free_result(res) + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100))", + ) + taos_free_result(res) + + stmt = taos_stmt_init(conn) + + taos_stmt_prepare(stmt, "insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + + params = new_bind_params(14) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].null() + params[2].tinyint(2) + params[3].smallint(3) + params[4].int(4) + params[5].bigint(5) + params[6].tinyint_unsigned(6) + params[7].smallint_unsigned(7) + params[8].int_unsigned(8) + params[9].bigint_unsigned(9) + params[10].float(10.1) + params[11].double(10.11) + params[12].binary("hello") + params[13].nchar("stmt") + taos_stmt_bind_param(stmt, params) + taos_stmt_add_batch(stmt) + taos_stmt_execute(stmt) + + res = taos_query(conn, "select * from log limit 1") + + fields = taos_fetch_fields(res) + filed_count = taos_field_count(res) + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, filed_count, 100) + + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + + assert rowstr == "1626861392589 NULL 2 3 4 5 6 7 8 9 10.100000 10.110000 hello stmt" + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + +def stream_callback(param, result, row): + # type: (c_void_p, c_void_p, c_void_p) -> None + try: + if result == None or row == None: + return + result = c_void_p(result) + row = c_void_p(row) + fields = taos_fetch_fields_raw(result) + num_fields = taos_field_count(result) + s = taos_print_row(row, fields, num_fields) + print(s) + taos_stop_query(result) + except Exception as err: + print(err) + +def test_stream(conn, caplog): + dbname = "pytest_ctaos_stream" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, n int)", + ) + taos_free_result(res) + + res = taos_query(conn, "select count(*) from log interval(5s)") + cc = taos_num_fields(res) + assert cc == 2 + + stream = taos_open_stream(conn, "select count(*) from log interval(5s)", stream_callback, 0, None, None) + print("waiting for data") + time.sleep(1) + + for i in range(0, 2): + res = taos_query(conn, "insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + taos_free_result(res) + time.sleep(2) + taos_close_stream(stream) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err diff --git a/src/connector/python/tests/test_info.py b/src/connector/python/tests/test_info.py new file mode 100644 index 0000000000000000000000000000000000000000..bddfec7ef9ddbc203adfcadd262839048466592c --- /dev/null +++ b/src/connector/python/tests/test_info.py @@ -0,0 +1,23 @@ +from taos.cinterface import * + +from taos import * + +import pytest + +@pytest.fixture +def conn(): + return connect() + +def test_client_info(): + print(taos_get_client_info()) + None + +def test_server_info(conn): + # type: (TaosConnection) -> None + print(conn.client_info) + print(conn.server_info) + None + +if __name__ == "__main__": + test_client_info() + test_server_info(connect()) diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9d2cdb39d6f4f2612581ce7284c057c456ef91 --- /dev/null +++ b/src/connector/python/tests/test_lines.py @@ -0,0 +1,57 @@ +from taos.error import OperationalError +from taos import connect, new_bind_params, PrecisionEnum +from taos import * + +from ctypes import * +import taos +import pytest + + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + + +def test_insert_lines(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_insert_lines" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + + lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + result = conn.query("select * from st") + print(*result.fields) + all = result.rows_iter() + for row in all: + print(row) + result.close() + print(result.row_count) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err + + +if __name__ == "__main__": + test_insert_lines(connect()) diff --git a/src/connector/python/tests/test_query.py b/src/connector/python/tests/test_query.py new file mode 100644 index 0000000000000000000000000000000000000000..f4e139b1f14df29e8b6304dd2ca03519ea274f43 --- /dev/null +++ b/src/connector/python/tests/test_query.py @@ -0,0 +1,43 @@ +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + return taos.connect() + +def test_query(conn): + """This test will use fetch_block for rows fetching, significantly faster than rows_iter""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result: + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +def test_query_row_iter(conn): + """This test will use fetch_row for each row fetching, this is the only way in async callback""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result.rows_iter(): + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +if __name__ == "__main__": + test_query(taos.connect(database = "log")) + test_query_row_iter(taos.connect(database = "log")) diff --git a/src/connector/python/tests/test_query_a.py b/src/connector/python/tests/test_query_a.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4be5695a87f1fd1017435b13983df7c4f70f06 --- /dev/null +++ b/src/connector/python/tests/test_query_a.py @@ -0,0 +1,66 @@ +from taos import * +from ctypes import * +import taos +import pytest +import time + + +@pytest.fixture +def conn(): + return taos.connect() + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(taos.connect()) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py new file mode 100644 index 0000000000000000000000000000000000000000..938ba10eb3d2377a63f7972deb99dbd47f7de1b2 --- /dev/null +++ b/src/connector/python/tests/test_stmt.py @@ -0,0 +1,149 @@ +from taos import * + +from ctypes import * +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + +def test_stmt_insert(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + + stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + params = new_bind_params(16) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].bool(True) + params[2].null() + params[3].tinyint(2) + params[4].smallint(3) + params[5].int(4) + params[6].bigint(5) + params[7].tinyint_unsigned(6) + params[8].smallint_unsigned(7) + params[9].int_unsigned(8) + params[10].bigint_unsigned(9) + params[11].float(10.1) + params[12].double(10.11) + params[13].binary("hello") + params[14].nchar("stmt") + params[15].timestamp(1626861392589, PrecisionEnum.Milliseconds) + + stmt.bind_param(params) + stmt.execute() + + result = stmt.use_result() + assert result.affected_rows == 1 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + row = result.next() + print(row) + assert row[2] == None + for i in range(3, 11): + assert row[i] == i - 1 + #float == may not work as expected + # assert row[10] == c_float(10.1) + assert row[12] == 10.11 + assert row[13] == "hello" + assert row[14] == "stmt" + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + +def test_stmt_insert_multi(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt_multi" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + start = datetime.now() + stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + + params = new_multi_binds(16) + params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) + params[1].bool((True, None, False)) + params[2].tinyint([-128, -128, None]) # -128 is tinyint null + params[3].tinyint([0, 127, None]) + params[4].smallint([3, None, 2]) + params[5].int([3, 4, None]) + params[6].bigint([3, 4, None]) + params[7].tinyint_unsigned([3, 4, None]) + params[8].smallint_unsigned([3, 4, None]) + params[9].int_unsigned([3, 4, None]) + params[10].bigint_unsigned([3, 4, None]) + params[11].float([3, None, 1]) + params[12].double([3, None, 1.2]) + params[13].binary(["abc", "dddafadfadfadfadfa", None]) + params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) + params[15].timestamp([None, None, 1626861392591]) + stmt.bind_param_batch(params) + + stmt.execute() + end = datetime.now() + print("elapsed time: ", end - start) + result = stmt.use_result() + assert result.affected_rows == 3 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + result.close() + + stmt.close() + + # start = datetime.now() + # conn.query("insert into log values(1626861392660, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392661, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392662, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)") + + # end = datetime.now() + # print("elapsed time: ", end - start) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err +if __name__ == "__main__": + test_stmt_insert(connect()) + test_stmt_insert_multi(connect()) \ No newline at end of file diff --git a/src/connector/python/tests/test_stream.py b/src/connector/python/tests/test_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..de6e20928b176e51bc6d350fb01268459f4e7f95 --- /dev/null +++ b/src/connector/python/tests/test_stream.py @@ -0,0 +1,70 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * +from taos import * +from ctypes import * +import time +import pytest + + +@pytest.fixture +def conn(): + return connect() + + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.execute("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) diff --git a/src/connector/python/tests/test_subscribe.py b/src/connector/python/tests/test_subscribe.py new file mode 100644 index 0000000000000000000000000000000000000000..99fe5b263625c63200f416ec98fcb561773becd8 --- /dev/null +++ b/src/connector/python/tests/test_subscribe.py @@ -0,0 +1,100 @@ +from taos.subscription import TaosSubscription +from taos import * +from ctypes import * +import taos +import pytest +import time +from random import random + + +@pytest.fixture +def conn(): + return taos.connect() + + +def test_subscribe(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + + sub = conn.subscribe(True, "test", "select * from log", 1000) + print("# consume from begin") + for ts, n in sub.consume(): + print(ts, n) + + print("# consume new data") + for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + + print("# consume with a stop condition") + for i in range(10): + conn.execute("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe(taos.connect()) + test_subscribe_callback(taos.connect()) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 890d1b34836b3e177fc665556a0909a38b9a609b..3838344a8b59beafabeecbf4556123ffc5763eb0 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5989,7 +5989,7 @@ static int32_t prepareStbStmtBind( int64_t startTime, int32_t recSeq, bool isColumn) { - char *bindBuffer = calloc(1, g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, g_args.len_of_binary); @@ -6324,8 +6324,8 @@ static void printStatPerThread(threadInfo *pThreadInfo) pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows, - (pThreadInfo->totalDelay/1000.0)? - (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000.0)): + (pThreadInfo->totalDelay)? + (double)(pThreadInfo->totalAffectedRows/((double)pThreadInfo->totalDelay/1000000.0)): FLT_MAX); } @@ -6545,7 +6545,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { verbosePrint("[%d] %s() LN%d, buffer=%s\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - startTs = taosGetTimestampMs(); + startTs = taosGetTimestampUs(); if (recOfBatch == 0) { errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", @@ -6561,10 +6561,10 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); - endTs = taosGetTimestampMs(); + endTs = taosGetTimestampUs(); uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", - __func__, __LINE__, delay); + performancePrint("%s() LN%d, insert execution time is %10.2f ms\n", + __func__, __LINE__, delay / 1000.0); verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, affectedRows); @@ -6721,14 +6721,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; - startTs = taosGetTimestampMs(); + startTs = taosGetTimestampUs(); int32_t affectedRows = execInsert(pThreadInfo, generated); - endTs = taosGetTimestampMs(); + endTs = taosGetTimestampUs(); uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", - __func__, __LINE__, delay); + performancePrint("%s() LN%d, insert execution time is %10.f ms\n", + __func__, __LINE__, delay/1000.0); verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID, __func__, __LINE__, affectedRows); @@ -7266,11 +7266,15 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); + fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); } //taos_close(taos); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index e3f3880f0cee97b3297b23d48b2229a12ff4c682..bea6e6510683e3a400623c2ef843d0ce6e34521c 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -231,8 +231,8 @@ static struct argp_option options[] = { {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00:000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, #if TSDB_SUPPORT_NANOSECOND == 1 {"precision", 'C', "PRECISION", 0, "Specify precision for converting human-readable time to epoch. Valid value is one of ms, us, and ns. Default is ms.", 6}, #else diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 15326e33b139d0e5e2266fcd38a213299ea1ba2d..8ff30123779e618d5537df79a1e687dc9e6896b9 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -167,11 +167,11 @@ python3 test.py -f tools/taosdemoTestInterlace.py python3 test.py -f tools/taosdemoTestQuery.py # nano support -# python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py -# python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py -# python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py -# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertTime_step.py -# python3 test.py -f tools/taosdumpTestNanoSupport.py +python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py +python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.py +python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py +python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestInsertTime_step.py +python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py # update python3 ./test.py -f update/allow_update.py @@ -383,7 +383,7 @@ python3 ./test.py -f insert/flushwhiledrop.py python3 ./test.py -f insert/schemalessInsert.py #======================p4-end=============== -# python3 test.py -f tools/taosdemoAllTest/pytest.py + diff --git a/tests/pytest/insert/line_insert.py b/tests/pytest/insert/line_insert.py index 53eaa55aa50a1369b4aff9c49421263788205038..92fdd0f28e612994df414ea1b560152a3f2001a8 100644 --- a/tests/pytest/insert/line_insert.py +++ b/tests/pytest/insert/line_insert.py @@ -42,18 +42,18 @@ class TDTestCase: "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines(lines) + print("insert_lines result {}".format(code)) lines2 = [ "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns" ] - code = self._conn.insertLines([ lines2[0] ]) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines([ lines2[0] ]) + print("insert_lines result {}".format(code)) - self._conn.insertLines([ lines2[1] ]) - print("insertLines result {}".format(code)) + self._conn.insert_lines([ lines2[1] ]) + print("insert_lines result {}".format(code)) tdSql.query("select * from st") tdSql.checkRows(4) @@ -73,7 +73,7 @@ class TDTestCase: tdSql.query("describe stf") tdSql.checkData(2, 2, 14) - self._conn.insertLines([ + self._conn.insert_lines([ "sth,t1=4i64,t2=5f64,t4=5f64,ID=\"childtable\" c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641ms", "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654ms" ]) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 5c93095a1ee414a1559f48595a3e2e412463e829..49c32235883483c9d709b6d7a41dc9419dd95461 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -11,8 +11,10 @@ # -*- coding: utf-8 -*- +import traceback import random import string +from taos.error import LinesError import time from copy import deepcopy import numpy as np @@ -292,7 +294,7 @@ class TDTestCase: def resCmp(self, input_sql, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None): expect_list = self.inputHandle(input_sql) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) query_sql = f"{query_sql} {stb_name} {condition}" res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) if ts == 0: @@ -312,7 +314,9 @@ class TDTestCase: expect_list[0].pop(j) tdSql.checkEqual(res_row_list[0], expect_list[0]) tdSql.checkEqual(res_field_list_without_ts, expect_list[1]) - tdSql.checkEqual(res_type_list, expect_list[2]) + for i in range(len(res_type_list)): + tdSql.checkEqual(res_type_list[i], expect_list[2][i]) + # tdSql.checkEqual(res_type_list, expect_list[2]) def cleanStb(self): query_sql = "show stables" @@ -405,13 +409,14 @@ class TDTestCase: """ for input_sql in [self.genLongSql(128, 1)[0], self.genLongSql(1, 4094)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) - + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass + def idIllegalNameCheckCase(self): """ test illegal id name @@ -421,8 +426,10 @@ class TDTestCase: rstr = list("`~!@#$¥%^&*()-+={}|[]、「」【】\:;《》<>?") for i in rstr: input_sql = self.genFullTypeSql(tb_name=f"\"aaa{i}bbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def idStartWithNumCheckCase(self): """ @@ -430,8 +437,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(tb_name=f"\"1aaabbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def nowTsCheckCase(self): """ @@ -439,8 +448,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="now")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def dateFormatTsCheckCase(self): """ @@ -448,8 +459,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="2021-07-21\ 19:01:46.920")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def illegalTsCheckCase(self): """ @@ -457,8 +470,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="16260068336390us19")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def tagValueLengthCheckCase(self): """ @@ -471,8 +486,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(t1=t1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i16 for t2 in ["-32767i16", "32767i16"]: @@ -480,8 +497,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(t2=t2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i32 for t3 in ["-2147483647i32", "2147483647i32"]: @@ -489,8 +508,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(t3=t3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i64 for t4 in ["-9223372036854775807i64", "9223372036854775807i64"]: @@ -498,8 +519,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(t4=t4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass # f32 for t5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -508,8 +531,12 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for t5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(t5=t5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) + # f64 for t6 in [f'{-1.79769*(10**308)}f64', f'{-1.79769*(10**308)}f64']: @@ -518,27 +545,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16375, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + pass # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4094, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def colValueLengthCheckCase(self): """ @@ -552,16 +588,22 @@ class TDTestCase: for c1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(c1=c1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i16 for c2 in ["-32767i16"]: input_sql, stb_name = self.genFullTypeSql(c2=c2) self.resCmp(input_sql, stb_name) for c2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(c2=c2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i32 for c3 in ["-2147483647i32"]: @@ -569,8 +611,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(c3=c3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i64 for c4 in ["-9223372036854775807i64"]: @@ -578,8 +623,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(c4=c4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f32 for c5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -588,8 +636,11 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for c5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(c5=c5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f64 for c6 in [f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64', f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64']: @@ -598,27 +649,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16375, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4094, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def tagColIllegalValueCheckCase(self): @@ -629,11 +689,17 @@ class TDTestCase: # bool for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: input_sql1 = self.genFullTypeSql(t0=i)[0] - code = self._conn.insertLines([input_sql1]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql1]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql2 = self.genFullTypeSql(c0=i)[0] - code = self._conn.insertLines([input_sql2]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql2]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i8 i16 i32 i64 f32 f64 for input_sql in [ @@ -651,8 +717,11 @@ class TDTestCase: self.genFullTypeSql(c6="11.1s45f64")[0], self.genFullTypeSql(c9="1s1u64")[0] ]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check binary and nchar blank stb_name = self.getLongName(7, "letters") @@ -661,18 +730,19 @@ class TDTestCase: input_sql3 = f'{stb_name},t0=t,t1="abc aaa" c0=f 1626006833639000000ns' input_sql4 = f'{stb_name},t0=t,t1=L"abc aaa" c0=f 1626006833639000000ns' for input_sql in [input_sql1, input_sql2, input_sql3, input_sql4]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check accepted binary and nchar symbols # # * ~!@#$¥%^&*()-+={}|[]、「」:; for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): input_sql1 = f'{stb_name},t0=t c0=f,c1="abc{symbol}aaa" 1626006833639000000ns' input_sql2 = f'{stb_name},t0=t,t1="abc{symbol}aaa" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql1]) - tdSql.checkEqual(code, 0) - code = self._conn.insertLines([input_sql2]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) def duplicateIdTagColInsertCheckCase(self): @@ -681,23 +751,35 @@ class TDTestCase: """ self.cleanStb() input_sql_id = self.genFullTypeSql(id_double_tag=True)[0] - code = self._conn.insertLines([input_sql_id]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_id]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_tag = input_sql.replace("t5", "t6") - code = self._conn.insertLines([input_sql_tag]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_tag]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "c6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "C6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) ##### stb exist ##### def noIdStbExistCheckCase(self): @@ -705,7 +787,7 @@ class TDTestCase: case no id when stb exist """ self.cleanStb() - input_sql, stb_name = self.genFullTypeSql(t0="f", c0="f") + input_sql, stb_name = self.genFullTypeSql(tb_name="sub_table_0123456", t0="f", c0="f") self.resCmp(input_sql, stb_name) input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, id_noexist_tag=True, t0="f", c0="f") self.resCmp(input_sql, stb_name, condition='where tbname like "t_%"') @@ -720,8 +802,7 @@ class TDTestCase: self.cleanStb() input_sql, stb_name = self.genFullTypeSql() self.resCmp(input_sql, stb_name) - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) self.resCmp(input_sql, stb_name) def tagColBinaryNcharLengthCheckCase(self): @@ -788,7 +869,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(tb_name1, tb_name2) input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True, ct_add_tag=True) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) tb_name3 = self.getNoIdTbName(stb_name) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) @@ -803,29 +884,35 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(5, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(6, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError: + pass tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) # # * check col，col+ts max in describe ---> 16143 input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(12, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(13, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -838,28 +925,32 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + code = self._conn.insert_lines([input_sql]) # * legal nchar could not be larger than 16374/4 input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(1, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(2, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(4, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(5, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -880,8 +971,7 @@ class TDTestCase: "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns", "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(lines) def multiInsertCheckCase(self, count): """ @@ -894,8 +984,7 @@ class TDTestCase: for i in range(count): input_sql = self.genFullTypeSql(stb_name=stb_name, t7=f'"{self.getLongName(8, "letters")}"', c7=f'"{self.getLongName(8, "letters")}"', id_noexist_tag=True)[0] sql_list.append(input_sql) - code = self._conn.insertLines(sql_list) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(sql_list) def batchErrorInsertCheckCase(self): """ @@ -905,8 +994,11 @@ class TDTestCase: stb_name = self.getLongName(8, "letters") lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", f"{stb_name},t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"] - code = self._conn.insertLines(lines) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines(lines) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def genSqlList(self, count=5, stb_name="", tb_name=""): """ @@ -957,7 +1049,7 @@ class TDTestCase: def genMultiThreadSeq(self, sql_list): tlist = list() for insert_sql in sql_list: - t = threading.Thread(target=self._conn.insertLines,args=([insert_sql[0]],)) + t = threading.Thread(target=self._conn.insert_lines,args=([insert_sql[0]],)) tlist.append(t) return tlist @@ -1155,16 +1247,18 @@ class TDTestCase: def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" input_sql2 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64 c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64 1626006933640000000ns" - code = self._conn.insertLines([input_sql1]) - code = self._conn.insertLines([input_sql2]) - print(code) - # self._conn.insertLines([input_sql2]) + try: + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) + except LinesError as err: + print(err.errno) + # self._conn.insert_lines([input_sql2]) # input_sql3 = f'abcd,id="cc¥Ec",t0=True,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="ndsfdrum",t8=L"ncharTagValue" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="igwoehkm",c8=L"ncharColValue",c9=7u64 0' # print(input_sql3) # input_sql4 = 'hmemeb,id="kilrcrldgf",t0=F,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="fysodjql",t8=L"ncharTagValue" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="waszbfvc",c8=L"ncharColValue",c9=7u64 0' - # code = self._conn.insertLines([input_sql3]) + # code = self._conn.insert_lines([input_sql3]) # print(code) - # self._conn.insertLines([input_sql4]) + # self._conn.insert_lines([input_sql4]) def runAll(self): self.initCheckCase() @@ -1222,7 +1316,11 @@ class TDTestCase: def run(self): print("running {}".format(__file__)) self.createDb() - self.runAll() + try: + self.runAll() + except Exception as err: + print(''.join(traceback.format_exception(None, err, err.__traceback__))) + raise err # self.tagColIllegalValueCheckCase() # self.test() diff --git a/tests/pytest/manualTest/cal_unavaliable_time_kill.py b/tests/pytest/manualTest/cal_unavaliable_time_kill.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ffc2506f9aae956e16d68fe52e30a15494301f --- /dev/null +++ b/tests/pytest/manualTest/cal_unavaliable_time_kill.py @@ -0,0 +1,50 @@ +# -*-coding: utf-8-*- +# for TD-5159 +import time +import taos +import sys, time, os, re, platform +from RemoteModule import RemoteModule +class Cal(): + def __init__(self): + master_ip = "192.168.1.189" + master_ssh_port = "22" + ssh_user = "root" + ssh_passwd = "tbase125!" + log_dir = "" + remote_dir = "" + self.RM_master = RemoteModule(master_ip, master_ssh_port, ssh_user, ssh_passwd, log_dir, remote_dir) + + def execShellCmd(self, shell_cmd): + result = os.popen(shell_cmd).read().strip() + return result + + def caltimeFromKill(self): + try: + conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/home/ubuntu/abt_taos") + while "failed" in str(conn): + conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/home/ubuntu/abt_taos") + if "failed" not in str(conn): + break + c1 = conn.cursor() + c1.execute("use test") + insert_tag = 0 + times = 0 + self.RM_master.exec_cmd('ps -ef | grep taosd | grep -v grep | awk \'{print $2}\' | xargs sudo kill -9') + start_time = time.time() + while insert_tag == 0 and times < 10: + insert_res = c1.execute('insert into stb_22 values (now,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)') + if insert_res == 1: + insert_tag = 1 + end_time = time.time() + break + else: + times += 1 + use_time = end_time - start_time + print(use_time) + return use_time + except Exception: + print("last failed") +if __name__ == '__main__': + cal = Cal() + cal.caltimeFromKill() + diff --git a/tests/pytest/manualTest/cal_unavaliable_time_stop.py b/tests/pytest/manualTest/cal_unavaliable_time_stop.py new file mode 100644 index 0000000000000000000000000000000000000000..2c60bb69fb95bfffd9bf7c8904eb8b72a7569b68 --- /dev/null +++ b/tests/pytest/manualTest/cal_unavaliable_time_stop.py @@ -0,0 +1,47 @@ +# -*-coding: utf-8-*- +# for TD-5159 +import time +import taos +import sys, time, os, re, platform +from RemoteModule import RemoteModule +class Cal(): + def __init__(self): + master_ip = "192.168.1.189" + master_ssh_port = "22" + #ssh_user = "ubuntu" + ssh_user = "root" + #ssh_passwd = "1" + ssh_passwd = "tbase125!" + log_dir = "" + remote_dir = "" + self.RM_master = RemoteModule(master_ip, master_ssh_port, ssh_user, ssh_passwd, log_dir, remote_dir) + + def execShellCmd(self, shell_cmd): + result = os.popen(shell_cmd).read().strip() + return result + + def caltimeFromKill(self): + try: + self.RM_master.exec_cmd('screen -d -m systemctl stop taosd') + start_time = time.time() + res = self.execShellCmd('taos -c /home/ubuntu/abt_taos -s "use test; insert into stb_22 values (now,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);"') + while "failed" in str(res): + start_time_shell = time.time() + res = self.execShellCmd('taos -c /home/ubuntu/abt_taos -s "use test; insert into stb_22 values (now,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);"') + end_time_shell = time.time() + shell_time = end_time_shell - start_time_shell + if "failed" not in str(res): + break + print(f'shell_time --- {shell_time}') + sql_time = str(res).split("database")[-1].replace('(', "").replace(')', "").replace('s', "").strip() + print(f'sql_time----{sql_time}') + end_time = time.time() + use_time = end_time - start_time - float(shell_time) + float(sql_time) + print(f'use_time---{use_time}') + return use_time + except Exception: + print("failed") +if __name__ == '__main__': + cal = Cal() + cal.caltimeFromKill() + diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_samples.csv b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..5fc779b41b44eda002d246d9554f0abcea03c8d3 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_samples.csv @@ -0,0 +1,100 @@ +8.855,"binary_str0" ,1626870128248246976 +8.75,"binary_str1" ,1626870128249060032 +5.44,"binary_str2" ,1626870128249067968 +8.45,"binary_str3" ,1626870128249072064 +4.07,"binary_str4" ,1626870128249075904 +6.97,"binary_str5" ,1626870128249078976 +6.86,"binary_str6" ,1626870128249082048 +1.585,"binary_str7" ,1626870128249085120 +1.4,"binary_str8" ,1626870128249087936 +5.135,"binary_str9" ,1626870128249092032 +3.15,"binary_str10" ,1626870128249095104 +1.765,"binary_str11" ,1626870128249097920 +7.71,"binary_str12" ,1626870128249100992 +3.91,"binary_str13" ,1626870128249104064 +5.615,"binary_str14" ,1626870128249106880 +9.495,"binary_str15" ,1626870128249109952 +3.825,"binary_str16" ,1626870128249113024 +1.94,"binary_str17" ,1626870128249117120 +5.385,"binary_str18" ,1626870128249119936 +7.075,"binary_str19" ,1626870128249123008 +5.715,"binary_str20" ,1626870128249126080 +1.83,"binary_str21" ,1626870128249128896 +6.365,"binary_str22" ,1626870128249131968 +6.55,"binary_str23" ,1626870128249135040 +6.315,"binary_str24" ,1626870128249138112 +3.82,"binary_str25" ,1626870128249140928 +2.455,"binary_str26" ,1626870128249145024 +7.795,"binary_str27" ,1626870128249148096 +2.47,"binary_str28" ,1626870128249150912 +1.37,"binary_str29" ,1626870128249155008 +5.39,"binary_str30" ,1626870128249158080 +5.13,"binary_str31" ,1626870128249160896 +4.09,"binary_str32" ,1626870128249163968 +5.855,"binary_str33" ,1626870128249167040 +0.17,"binary_str34" ,1626870128249170112 +1.955,"binary_str35" ,1626870128249173952 +0.585,"binary_str36" ,1626870128249178048 +0.33,"binary_str37" ,1626870128249181120 +7.925,"binary_str38" ,1626870128249183936 +9.685,"binary_str39" ,1626870128249187008 +2.6,"binary_str40" ,1626870128249191104 +5.705,"binary_str41" ,1626870128249193920 +3.965,"binary_str42" ,1626870128249196992 +4.43,"binary_str43" ,1626870128249200064 +8.73,"binary_str44" ,1626870128249202880 +3.105,"binary_str45" ,1626870128249205952 +9.39,"binary_str46" ,1626870128249209024 +2.825,"binary_str47" ,1626870128249212096 +9.675,"binary_str48" ,1626870128249214912 +9.99,"binary_str49" ,1626870128249217984 +4.51,"binary_str50" ,1626870128249221056 +4.94,"binary_str51" ,1626870128249223872 +7.72,"binary_str52" ,1626870128249226944 +4.135,"binary_str53" ,1626870128249231040 +2.325,"binary_str54" ,1626870128249234112 +4.585,"binary_str55" ,1626870128249236928 +8.76,"binary_str56" ,1626870128249240000 +4.715,"binary_str57" ,1626870128249243072 +0.56,"binary_str58" ,1626870128249245888 +5.35,"binary_str59" ,1626870128249249984 +5.075,"binary_str60" ,1626870128249253056 +6.665,"binary_str61" ,1626870128249256128 +7.13,"binary_str62" ,1626870128249258944 +2.775,"binary_str63" ,1626870128249262016 +5.775,"binary_str64" ,1626870128249265088 +1.62,"binary_str65" ,1626870128249267904 +1.625,"binary_str66" ,1626870128249270976 +8.15,"binary_str67" ,1626870128249274048 +0.75,"binary_str68" ,1626870128249277120 +3.265,"binary_str69" ,1626870128249280960 +8.585,"binary_str70" ,1626870128249284032 +1.88,"binary_str71" ,1626870128249287104 +8.44,"binary_str72" ,1626870128249289920 +5.12,"binary_str73" ,1626870128249295040 +2.58,"binary_str74" ,1626870128249298112 +9.42,"binary_str75" ,1626870128249300928 +1.765,"binary_str76" ,1626870128249304000 +2.66,"binary_str77" ,1626870128249308096 +1.405,"binary_str78" ,1626870128249310912 +5.595,"binary_str79" ,1626870128249315008 +2.28,"binary_str80" ,1626870128249318080 +9.24,"binary_str81" ,1626870128249320896 +9.03,"binary_str82" ,1626870128249323968 +6.055,"binary_str83" ,1626870128249327040 +1.74,"binary_str84" ,1626870128249330112 +5.77,"binary_str85" ,1626870128249332928 +1.97,"binary_str86" ,1626870128249336000 +0.3,"binary_str87" ,1626870128249339072 +7.145,"binary_str88" ,1626870128249342912 +0.88,"binary_str89" ,1626870128249345984 +8.025,"binary_str90" ,1626870128249349056 +4.81,"binary_str91" ,1626870128249351872 +0.725,"binary_str92" ,1626870128249355968 +3.85,"binary_str93" ,1626870128249359040 +9.455,"binary_str94" ,1626870128249362112 +2.265,"binary_str95" ,1626870128249364928 +3.985,"binary_str96" ,1626870128249368000 +9.375,"binary_str97" ,1626870128249371072 +0.2,"binary_str98" ,1626870128249373888 +6.95,"binary_str99" ,1626870128249377984 diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv new file mode 100644 index 0000000000000000000000000000000000000000..18fb855d6d9f55c29325c6ea6f77120effa72884 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv @@ -0,0 +1,100 @@ +"string0",7,8.615 +"string1",4,9.895 +"string2",3,2.92 +"string3",3,5.62 +"string4",7,1.615 +"string5",6,1.45 +"string6",5,7.48 +"string7",7,3.01 +"string8",5,4.76 +"string9",10,7.09 +"string10",2,8.38 +"string11",7,8.65 +"string12",5,5.025 +"string13",10,5.765 +"string14",2,4.57 +"string15",2,1.03 +"string16",7,6.98 +"string17",10,0.23 +"string18",7,5.815 +"string19",1,2.37 +"string20",10,8.865 +"string21",3,1.235 +"string22",2,8.62 +"string23",9,1.045 +"string24",8,4.34 +"string25",1,5.455 +"string26",2,4.475 +"string27",1,6.95 +"string28",2,3.39 +"string29",3,6.79 +"string30",7,9.735 +"string31",1,9.79 +"string32",10,9.955 +"string33",1,5.095 +"string34",3,3.86 +"string35",9,5.105 +"string36",10,4.22 +"string37",1,2.78 +"string38",9,6.345 +"string39",1,0.975 +"string40",5,6.16 +"string41",4,7.735 +"string42",5,6.6 +"string43",8,2.845 +"string44",1,0.655 +"string45",3,2.995 +"string46",9,3.6 +"string47",8,3.47 +"string48",3,7.98 +"string49",6,2.225 +"string50",9,5.44 +"string51",4,6.335 +"string52",3,2.955 +"string53",1,0.565 +"string54",6,5.575 +"string55",6,9.905 +"string56",9,6.025 +"string57",8,0.94 +"string58",10,0.15 +"string59",8,1.555 +"string60",4,2.28 +"string61",2,8.29 +"string62",9,6.22 +"string63",6,3.35 +"string64",10,6.7 +"string65",3,9.345 +"string66",7,9.815 +"string67",1,5.365 +"string68",10,3.81 +"string69",1,6.405 +"string70",8,2.715 +"string71",3,8.58 +"string72",8,6.34 +"string73",2,7.49 +"string74",4,8.64 +"string75",3,8.995 +"string76",7,3.465 +"string77",1,7.64 +"string78",6,3.65 +"string79",6,1.4 +"string80",6,5.875 +"string81",2,1.22 +"string82",5,7.87 +"string83",9,8.41 +"string84",9,8.9 +"string85",9,3.89 +"string86",2,5.0 +"string87",2,4.495 +"string88",4,2.835 +"string89",3,5.895 +"string90",7,8.41 +"string91",5,5.125 +"string92",7,9.165 +"string93",5,8.315 +"string94",10,7.485 +"string95",7,4.635 +"string96",2,6.015 +"string97",8,0.595 +"string98",3,8.79 +"string99",4,1.72 diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd5ddbae8d5ce81269626165b2d275d05135ea5 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb3", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json new file mode 100644 index 0000000000000000000000000000000000000000..5408a9841ab8a40e4ca7564724b7f6c7f941e0e0 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json new file mode 100644 index 0000000000000000000000000000000000000000..13eb80f3cf7f751398babed8e922f9e5b3a4242e --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "us", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestInsertTime_step.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestInsertTime_step.py new file mode 100644 index 0000000000000000000000000000000000000000..b248eda6b0ce42d8cb681a74d5c807d0275e6e14 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestInsertTime_step.py @@ -0,0 +1,115 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + + # check the params of taosdemo about time_step is nano + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json -y " % binPath) + tdSql.execute("use testdb1") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.getData(9, 1) + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.000099000") + + # check the params of taosdemo about time_step is us + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json -y " % binPath) + tdSql.execute("use testdb2") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.getData(9, 1) + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.099000") + + # check the params of taosdemo about time_step is ms + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json -y " % binPath) + tdSql.execute("use testdb3") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:01:39.000") + + + os.system("rm -rf ./res.txt") + os.system("rm -rf ./*.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json new file mode 100644 index 0000000000000000000000000000000000000000..38ac666fac5097d616c17bdfc7e900256827ddf4 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdb", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json new file mode 100644 index 0000000000000000000000000000000000000000..9ef4a0af66e852a01d8ca7d677de4467ea316097 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json @@ -0,0 +1,84 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "subnsdb", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/NanoTestCase/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/NanoTestCase/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json new file mode 100644 index 0000000000000000000000000000000000000000..a09dec21fa9cf3720b68a1fa2e843b49be0544ee --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdb2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "now", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json new file mode 100644 index 0000000000000000000000000000000000000000..e99c528c6d62e3b9ce59565e60d21fb562bb836d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json @@ -0,0 +1,84 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdbcsv", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 3600, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/NanoTestCase/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/NanoTestCase/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/NanoTestCase/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcb2631251ae7062ee5f61a47fcb88057eddc75 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py @@ -0,0 +1,156 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert data from a special timestamp + # check stable stb0 + + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json -y " % binPath) + tdSql.execute("use nsdb") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") + + # check stable stb1 which is insert with disord + + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb1_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 10000) + # check c8 is an nano timestamp + tdSql.query("describe stb1") + tdSql.checkDataType(9, 1,"TIMESTAMP") + # check insert timestamp_step is nano_second + tdSql.query("select last(ts) from stb1") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") + + # insert data from now time + + # check stable stb0 + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json -y " % binPath) + + tdSql.execute("use nsdb2") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + # check c8 is an nano timestamp + tdSql.query("describe stb0") + tdSql.checkDataType(9,1,"TIMESTAMP") + + # insert by csv files and timetamp is long int , strings in ts and cols + + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json -y " % binPath) + tdSql.execute("use nsdbcsv") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(3, 1, "TIMESTAMP") + tdSql.query("select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") + tdSql.checkData(0, 0, 5000) + tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") + tdSql.checkData(0, 0, 10000) + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNano*.py.sql") + + # taosdemo test insert with command and parameter , detals show taosdemo --help + os.system("%staosdemo -u root -P taosdata -p 6030 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) + tdSql.query("select count(*) from test.meters") + tdSql.checkData(0, 0, 600) + # check taosdemo -s + + sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 3600 days 6 update 1;', + 'use nsdbsql;','CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', + 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76);'] + + with open("./taosdemoTestNanoCreateDB.sql",mode ="a" ) as sql_files: + for sql in sqls_ls: + sql_files.write(sql+"\n") + sql_files.close() + + sleep(10) + + os.system("%staosdemo -s taosdemoTestNanoCreateDB.sql -y " % binPath) + tdSql.query("select count(*) from nsdbsql.meters") + tdSql.checkData(0, 0, 2) + + os.system("rm -rf ./res.txt") + os.system("rm -rf ./*.py.sql") + os.system("rm -rf ./taosdemoTestNanoCreateDB.sql") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.json new file mode 100644 index 0000000000000000000000000000000000000000..fff1017588bb10f55a82aa2bd7bc6997df71abfd --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.json @@ -0,0 +1,92 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "nsdb", + "query_times": 10, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 1, + "concurrent": 2, + "sqls": [ + { + "sql": "select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000 \" ;", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from stb0 where ts>now-20d ;", + "result": "./query_res2.txt" + }, + { + "sql": "select max(c10) from stb0;", + "result": "./query_res3.txt" + }, + { + "sql": "select min(c1) from stb0;", + "result": "./query_res4.txt" + }, + { + "sql": "select avg(c1) from stb0;", + "result": "./query_res5.txt" + }, + { + "sql":"select count(*) from stb0 group by tbname;", + "result":"./query_res6.txt" + } + + ] + }, + "super_table_query": { + "stblname": "stb0", + "query_interval": 0, + "threads": 4, + "sqls": [ + { + "sql": "select count(*) from xxxx where ts>\"2021-07-01 00:01:00.000000000 \" ;", + "result": "./query_res_tb0.txt" + }, + { + "sql":"select count(*) from xxxx where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;", + "result": "./query_res_tb1.txt" + }, + { + "sql":"select first(*) from xxxx ;", + "result": "./query_res_tb2.txt" + }, + { + "sql":"select last(*) from xxxx;", + "result": "./query_res_tb3.txt" + + }, + { + "sql":"select last_row(*) from xxxx ;", + "result": "./query_res_tb4.txt" + + }, + { + "sql":"select max(c10) from xxxx ;", + "result": "./query_res_tb5.txt" + + }, + { + "sql":"select min(c1) from xxxx ;", + "result": "./query_res_tb6.txt" + + }, + { + "sql":"select avg(c10) from xxxx ;", + "result": "./query_res_tb7.txt" + + } + + ] + } + } \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.py new file mode 100644 index 0000000000000000000000000000000000000000..6c3e4d6c8a0d72a4b468cca22b2b1a6a25659db5 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.py @@ -0,0 +1,157 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # query: query test for nanoSecond with where and max min groupby order + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json -y " % binPath) + + tdSql.execute("use nsdb") + + # use where to filter + + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:00:00.590000000 \" ") + tdSql.checkData(0, 0, 4000) + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:00:00.000000000\" and ts <=\"2021-07-01 00:00:00.590000000\" ") + tdSql.checkData(0, 0, 5900) + + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:00:00.590000000 \" ;") + tdSql.checkData(0, 0, 40) + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:00:00.000000000\" and ts <=\"2021-07-01 00:00:00.590000000\" ") + tdSql.checkData(0, 0, 59) + + + # select max min avg from special col + tdSql.query("select max(c10) from stb0;") + print("select max(c10) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select max(c10) from tb0_0;") + print("select max(c10) from tb0_0 : " , tdSql.getData(0, 0)) + + + tdSql.query("select min(c1) from stb0;") + print( "select min(c1) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select min(c1) from tb0_0;") + print( "select min(c1) from tb0_0 : " , tdSql.getData(0, 0)) + + tdSql.query("select avg(c1) from stb0;") + print( "select avg(c1) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select avg(c1) from tb0_0;") + print( "select avg(c1) from tb0_0 : " , tdSql.getData(0, 0)) + + tdSql.query("select count(*) from stb0 group by tbname;") + tdSql.checkData(0, 0, 100) + tdSql.checkData(10, 0, 100) + + # query : query above sqls by taosdemo and continuously + + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.json -y " % binPath) + + + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json -y " % binPath) + tdSql.execute("use nsdbcsv") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(3, 1, "TIMESTAMP") + tdSql.query("select count(*) from stb0 where ts >\"2021-07-01 00:00:00.490000000\"") + tdSql.checkData(0, 0, 5000) + tdSql.query("select count(*) from stb0 where ts 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 < 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 = 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 != 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 <> 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 > "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 < "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 = "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 != "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 <> "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where ts between "2021-07-01 00:00:00.000000000" and "2021-07-01 00:00:00.990000000"') + tdSql.execute('select count(*) from stb0 where ts between 1625068800000000000 and 1625068801000000000') + tdSql.query('select avg(c0) from stb0 interval(5000000000b)') + tdSql.checkRows(1) + + tdSql.query('select avg(c0) from stb0 interval(100000000b)') + tdSql.checkRows(10) + + tdSql.error('select avg(c0) from stb0 interval(1b)') + tdSql.error('select avg(c0) from stb0 interval(999b)') + + tdSql.query('select avg(c0) from stb0 interval(1000b)') + tdSql.checkRows(100) + + tdSql.query('select avg(c0) from stb0 interval(1u)') + tdSql.checkRows(100) + + tdSql.query('select avg(c0) from stb0 interval(100000000b) sliding (100000000b)') + tdSql.checkRows(10) + + # query : query above sqls by taosdemo and continuously + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuerycsv.json -y " % binPath) + + os.system("rm -rf ./query_res*.txt*") + os.system("rm -rf tools/taosdemoAllTest/NanoTestCase/*.py.sql") + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuerycsv.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuerycsv.json new file mode 100644 index 0000000000000000000000000000000000000000..2323b0f3702b34bb56c4f82880e9820181891768 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuerycsv.json @@ -0,0 +1,110 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "nsdbcsv", + "query_times": 10, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 1, + "concurrent": 2, + "sqls": [ + { + "sql": "select count(*) from stb0 where ts> \"2021-07-01 00:00:00.490000000\" ;", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb0 where ts < now -22d-1h-3s ;", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from stb0 where ts < 1626918583000000000 ;", + "result": "./query_res2.txt" + }, + { + "sql": "select count(*) from stb0 where c2 <> 162687012800000000;", + "result": "./query_res3.txt" + }, + { + "sql": "select count(*) from stb0 where c2 != \"2021-07-21 20:22:08.248246976\";", + "result": "./query_res4.txt" + }, + { + "sql": "select count(*) from stb0 where ts between \"2021-07-01 00:00:00.000000000\" and \"2021-07-01 00:00:00.990000000\";", + "result": "./query_res5.txt" + }, + { + "sql":"select count(*) from stb0 group by tbname;", + "result":"./query_res6.txt" + }, + { + "sql":"select count(*) from stb0 where ts between 1625068800000000000 and 1625068801000000000;", + "result":"./query_res7.txt" + }, + { + "sql":"select avg(c0) from stb0 interval(5000000000b);", + "result":"./query_res8.txt" + }, + { + "sql":"select avg(c0) from stb0 interval(100000000b) sliding (100000000b);", + "result":"./query_res9.txt" + } + + ] + }, + "super_table_query": { + "stblname": "stb0", + "query_interval": 0, + "threads": 4, + "sqls": [ + { + "sql": "select count(*) from xxxx where ts > \"2021-07-01 00:00:00.490000000\" ;", + "result": "./query_res_tb0.txt" + }, + { + "sql":"select count(*) from xxxx where ts between \"2021-07-01 00:00:00.000000000\" and \"2021-07-01 00:00:00.990000000\" ;", + "result": "./query_res_tb1.txt" + }, + { + "sql":"select first(*) from xxxx ;", + "result": "./query_res_tb2.txt" + }, + { + "sql":"select last(*) from xxxx;", + "result": "./query_res_tb3.txt" + + }, + { + "sql":"select last_row(*) from xxxx ;", + "result": "./query_res_tb4.txt" + + }, + { + "sql":"select max(c0) from xxxx ;", + "result": "./query_res_tb5.txt" + + }, + { + "sql":"select min(c0) from xxxx ;", + "result": "./query_res_tb6.txt" + + }, + { + "sql":"select avg(c0) from xxxx ;", + "result": "./query_res_tb7.txt" + + }, + { + "sql":"select avg(c0) from xxxx interval(100000000b) sliding (100000000b) ;", + "result": "./query_res_tb8.txt" + + } + + + ] + } + } \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json new file mode 100644 index 0000000000000000000000000000000000000000..1cc834164e7c966a9ce565f1ce481d823b1ed2d1 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json @@ -0,0 +1,32 @@ +{ + "filetype":"subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "subnsdb", + "confirm_parameter_prompt": "no", + "specified_table_query": + { + "concurrent":2, + "mode":"sync", + "interval":10000, + "restart":"yes", + "keepProgress":"yes", + "sqls": [ + { + "sql": "select * from stb0 where ts>= \"2021-07-01 00:00:00.000000000\" ;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select * from stb0 where ts < now -2d-1h-3s ;", + "result": "./subscribe_res1.txt" + }, + { + "sql": "select * from stb0 where ts < 1626918583000000000 ;", + "result": "./subscribe_res2.txt" + }] + + } +} diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py new file mode 100644 index 0000000000000000000000000000000000000000..95c1a731bcde4c78d0aa272183bd2e7b7a4b168b --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py @@ -0,0 +1,125 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time +from datetime import datetime +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # get the number of subscriptions + def subTimes(self,filename): + self.filename = filename + command = 'cat %s |wc -l'% filename + times = int(subprocess.getstatusoutput(command)[1]) + return times + + # assert results + def assertCheck(self,filename,subResult,expectResult): + self.filename = filename + self.subResult = subResult + self.expectResult = expectResult + args0 = (filename, subResult, expectResult) + assert subResult == expectResult , "Queryfile:%s ,result is %s != expect: %s" % args0 + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # clear env + os.system("ps -ef |grep 'taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json' |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + os.system("rm -rf ./subscribe_res*") + os.system("rm -rf ./all_subscribe_res*") + + + # insert data + os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json" % binPath) + os.system("nohup %staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json &" % binPath) + query_pid = int(subprocess.getstatusoutput('ps aux|grep "taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json" |grep -v "grep"|awk \'{print $2}\'')[1]) + + + # merge result files + sleep(5) + os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") + os.system("cat subscribe_res1.txt* > all_subscribe_res1.txt") + os.system("cat subscribe_res2.txt* > all_subscribe_res2.txt") + + + # correct subscribeTimes testcase + subTimes0 = self.subTimes("all_subscribe_res0.txt") + self.assertCheck("all_subscribe_res0.txt",subTimes0 ,200) + + subTimes1 = self.subTimes("all_subscribe_res1.txt") + self.assertCheck("all_subscribe_res1.txt",subTimes1 ,200) + + subTimes2 = self.subTimes("all_subscribe_res2.txt") + self.assertCheck("all_subscribe_res2.txt",subTimes2 ,200) + + + # insert extral data + tdSql.execute("use subnsdb") + tdSql.execute("insert into tb0_0 values(now,100.1000,'subtest1',now-1s)") + sleep(15) + + os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") + subTimes0 = self.subTimes("all_subscribe_res0.txt") + print("pass") + self.assertCheck("all_subscribe_res0.txt",subTimes0 ,202) + + + + # correct data testcase + os.system("kill -9 %d" % query_pid) + sleep(3) + os.system("rm -rf ./subscribe_res*") + os.system("rm -rf ./all_subscribe*") + os.system("rm -rf ./*.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py similarity index 100% rename from tests/pytest/tools/taosdumpTestNanoSupport.py rename to tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py diff --git a/tests/pytest/util/sub.py b/tests/pytest/util/sub.py index 2e3c2a96b7312c176c25bd35e109e146e5c4593f..664d830b86290e81e8ac1726f92193380d8c7715 100644 --- a/tests/pytest/util/sub.py +++ b/tests/pytest/util/sub.py @@ -29,9 +29,10 @@ class TDSub: self.sub.close(keepProgress) def consume(self): - self.data = self.sub.consume() - self.consumedRows = len(self.data) - self.consumedCols = len(self.sub.fields) + self.result = self.sub.consume() + self.result.fetch_all() + self.consumedRows = self.result.row_count + self.consumedCols = self.result.field_count return self.consumedRows def checkRows(self, expectRows):