diff --git a/src/connector/python/README.md b/src/connector/python/README.md index a5dc2b72dafbeef0bf53fc1768f6afc66e714699..95ef26e1f0e73cee7d47ecb6ece1d6a95d2f89d3 100644 --- a/src/connector/python/README.md +++ b/src/connector/python/README.md @@ -1,6 +1,7 @@ # TDengine Connector for Python -[TDengine] connector for Python enables python programs to access TDengine, using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. ## Install @@ -11,8 +12,417 @@ pip install ./TDengine/src/connector/python ## Source Code -[TDengine] connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). -## License - AGPL +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 Keep same with [TDengine](https://github.com/taosdata/TDengine). diff --git a/src/connector/python/examples/bind-multi.py b/src/connector/python/examples/bind-multi.py new file mode 100644 index 0000000000000000000000000000000000000000..8530253aef58079e01f5eb71d8e12ab1649b7731 --- /dev/null +++ b/src/connector/python/examples/bind-multi.py @@ -0,0 +1,50 @@ +# encoding:UTF-8 +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt_multi" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() \ No newline at end of file diff --git a/src/connector/python/examples/bind-row.py b/src/connector/python/examples/bind-row.py new file mode 100644 index 0000000000000000000000000000000000000000..4ab9a9167ad23a6167c6586aac30ae6941dcee6d --- /dev/null +++ b/src/connector/python/examples/bind-row.py @@ -0,0 +1,57 @@ +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +# No need to explicitly close, but ok for you +# result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) + +# No need to explicitly close, but ok for you +# result.close() +# stmt.close() +# conn.close() diff --git a/src/connector/python/examples/insert-lines.py b/src/connector/python/examples/insert-lines.py new file mode 100644 index 0000000000000000000000000000000000000000..0096b7e8cdf1328ee78805a1ee3134ad7cdfc447 --- /dev/null +++ b/src/connector/python/examples/insert-lines.py @@ -0,0 +1,22 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', +] +conn.insert_lines(lines) +print("inserted") + +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) + + +conn.execute("drop database if exists %s" % dbname) diff --git a/src/connector/python/examples/pep-249.py b/src/connector/python/examples/pep-249.py new file mode 100644 index 0000000000000000000000000000000000000000..971a3c401f00b982096b8d429f65bce73cca4760 --- /dev/null +++ b/src/connector/python/examples/pep-249.py @@ -0,0 +1,9 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) diff --git a/src/connector/python/examples/query-async.py b/src/connector/python/examples/query-async.py new file mode 100644 index 0000000000000000000000000000000000000000..b600b796974e47d5e5fc7d88998e95ba46bb92cd --- /dev/null +++ b/src/connector/python/examples/query-async.py @@ -0,0 +1,62 @@ +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + # should explicitly close the result in fetch completed or cause error + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + # explicitly close result while query failed + result.close() + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + # conn.close() + + +if __name__ == "__main__": + test_query(connect()) \ No newline at end of file diff --git a/src/connector/python/examples/query-objectively.py b/src/connector/python/examples/query-objectively.py new file mode 100644 index 0000000000000000000000000000000000000000..104347cbf91e29e62fef26477b475053a8b8bc3e --- /dev/null +++ b/src/connector/python/examples/query-objectively.py @@ -0,0 +1,12 @@ +import taos + +conn = taos.connect() +conn.execute("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +conn.execute("drop database pytest") diff --git a/src/connector/python/examples/subscribe-async.py b/src/connector/python/examples/subscribe-async.py new file mode 100644 index 0000000000000000000000000000000000000000..3782ce5505152e78838406e313094eb911bea4a2 --- /dev/null +++ b/src/connector/python/examples/subscribe-async.py @@ -0,0 +1,43 @@ +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + # sub.close() + + conn.execute("drop database if exists %s" % dbname) + # conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + # conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) diff --git a/src/connector/python/examples/subscribe-sync.py b/src/connector/python/examples/subscribe-sync.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7f65f460280924ed3a577fe55b975fbf12c1a3 --- /dev/null +++ b/src/connector/python/examples/subscribe-sync.py @@ -0,0 +1,53 @@ +import taos +import random + +conn = taos.connect() +dbname = "pytest_taos_subscribe" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.execute("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + +sub = conn.subscribe(False, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +sub.close(True) +print("# keep progress consume") +sub = conn.subscribe(False, "test", "select * from log", 1000) +result = sub.consume() +rows = result.fetch_all() +# consume from latest subscription needs root privilege(for /var/lib/taos). +assert result.row_count == 0 +print("## consumed ", len(rows), "rows") + +print("# consume with a stop condition") +for i in range(10): + conn.execute("insert into log values(now, %d)" % random.randint(0, 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +# sub.close() + +conn.execute("drop database if exists %s" % dbname) +# conn.close() diff --git a/src/connector/python/pyproject.toml b/src/connector/python/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..a8099199563a0e5957a7d69e75bab65cca6d17db --- /dev/null +++ b/src/connector/python/pyproject.toml @@ -0,0 +1,27 @@ +[tool.poetry] +name = "taos" +version = "2.1.0" +description = "TDengine connector for python" +authors = ["Taosdata Inc. "] +license = "AGPL-3.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^2.7 || ^3.4" +typing = "*" + +[tool.poetry.dev-dependencies] +pytest = [ + { version = "^4.6", python = "^2.7" }, + { version = "^6.2", python = "^3.7" } +] +pdoc = { version = "^7.1.1", python = "^3.7" } +mypy = { version = "^0.910", python = "^3.6" } +black = { version = "^21.7b0", python = "^3.6" } + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 119 diff --git a/src/connector/python/setup.py b/src/connector/python/setup.py index 284861ca87dd77b1bc6799ad4cb32ff4c489e239..b7e10001737bc40c04173ea4a65e95248965ffda 100644 --- a/src/connector/python/setup.py +++ b/src/connector/python/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.11", + version="2.1.0", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/taos/__init__.py b/src/connector/python/taos/__init__.py index 52c6db311ecc4c2f944372ae3334fdc58cb6e779..75138eade3d60f7894d814babe58cec7aecc9a20 100644 --- a/src/connector/python/taos/__init__.py +++ b/src/connector/python/taos/__init__.py @@ -1,20 +1,478 @@ +# encoding:UTF-8 +""" +# TDengine Connector for Python -from .connection import TDengineConnection -from .cursor import TDengineCursor +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. -# For some reason, the following is needed for VS Code (through PyLance) to +## Install + +```sh +git clone --depth 1 https://github.com/taosdata/TDengine.git +pip install ./TDengine/src/connector/python +``` + +## Source Code + +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). + +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 + +Keep same with [TDengine](https://github.com/taosdata/TDengine). +""" +from .connection import TaosConnection + +# For some reason, the following is needed for VS Code (through PyLance) to # recognize that "error" is a valid module of the "taos" package. -from .error import ProgrammingError +from .error import * +from .bind import * +from .field import * +from .cursor import * +from .result import * +from .statement import * +from .subscription import * + +try: + import importlib.metadata + + __version__ = importlib.metadata.version("taos") +except: + None # Globals threadsafety = 0 -paramstyle = 'pyformat' - -__all__ = ['connection', 'cursor'] +paramstyle = "pyformat" +__all__ = [ + # functions + "connect", + "new_bind_param", + "new_bind_params", + "new_multi_binds", + "new_multi_bind", + # objects + "TaosBind", + "TaosConnection", + "TaosCursor", + "TaosResult", + "TaosRows", + "TaosRow", + "TaosStmt", + "PrecisionEnum", +] def connect(*args, **kwargs): - """ Function to return a TDengine connector object + # type: (..., ...) -> TaosConnection + """Function to return a TDengine connector object Current supporting keyword parameters: @dsn: Data source name as string @@ -25,4 +483,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) + return TaosConnection(*args, **kwargs) diff --git a/src/connector/python/taos/bind.py b/src/connector/python/taos/bind.py new file mode 100644 index 0000000000000000000000000000000000000000..ede6381628ae0fd5ff8794ef23db2f5afcfb5f3d --- /dev/null +++ b/src/connector/python/taos/bind.py @@ -0,0 +1,432 @@ +# encoding:UTF-8 +import ctypes +from .constants import FieldType +from .error import * +from .precision import * +from datetime import datetime +from ctypes import * +import sys + +_datetime_epoch = datetime.utcfromtimestamp(0) + +def _is_not_none(obj): + obj != None +class TaosBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_size_t)), + ("is_null", POINTER(c_int)), + ("is_unsigned", c_int), + ("error", POINTER(c_int)), + ("u", c_int64), + ("allocated", c_int), + ] + + def null(self): + self.buffer_type = FieldType.C_NULL + self.is_null = pointer(c_int(1)) + + def bool(self, value): + self.buffer_type = FieldType.C_BOOL + self.buffer = cast(pointer(c_bool(value)), c_void_p) + self.buffer_length = sizeof(c_bool) + + def tinyint(self, value): + self.buffer_type = FieldType.C_TINYINT + self.buffer = cast(pointer(c_int8(value)), c_void_p) + self.buffer_length = sizeof(c_int8) + + def smallint(self, value): + self.buffer_type = FieldType.C_SMALLINT + self.buffer = cast(pointer(c_int16(value)), c_void_p) + self.buffer_length = sizeof(c_int16) + + def int(self, value): + self.buffer_type = FieldType.C_INT + self.buffer = cast(pointer(c_int32(value)), c_void_p) + self.buffer_length = sizeof(c_int32) + + def bigint(self, value): + self.buffer_type = FieldType.C_BIGINT + self.buffer = cast(pointer(c_int64(value)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def float(self, value): + self.buffer_type = FieldType.C_FLOAT + self.buffer = cast(pointer(c_float(value)), c_void_p) + self.buffer_length = sizeof(c_float) + + def double(self, value): + self.buffer_type = FieldType.C_DOUBLE + self.buffer = cast(pointer(c_double(value)), c_void_p) + self.buffer_length = sizeof(c_double) + + def binary(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_BINARY + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def timestamp(self, value, precision=PrecisionEnum.Milliseconds): + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + ts = value + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(pointer(c_int64(ts)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def nchar(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_NCHAR + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def tinyint_unsigned(self, value): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer = cast(pointer(c_uint8(value)), c_void_p) + self.buffer_length = sizeof(c_uint8) + + def smallint_unsigned(self, value): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer = cast(pointer(c_uint16(value)), c_void_p) + self.buffer_length = sizeof(c_uint16) + + def int_unsigned(self, value): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer = cast(pointer(c_uint32(value)), c_void_p) + self.buffer_length = sizeof(c_uint32) + + def bigint_unsigned(self, value): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer = cast(pointer(c_uint64(value)), c_void_p) + self.buffer_length = sizeof(c_uint64) + + +def _datetime_to_timestamp(value, precision): + # type: (datetime | float | int | str | c_int64, PrecisionEnum) -> c_int64 + if value is None: + return FieldType.C_BIGINT_NULL + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + return int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + return c_int64(value) + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif isinstance(value, c_int64): + return value + return FieldType.C_BIGINT_NULL + + +class TaosMultiBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_int32)), + ("is_null", c_char_p), + ("num", c_int), + ] + + def null(self, num): + self.buffer_type = FieldType.C_NULL + self.is_null = cast((c_char * num)(*[1 for _ in range(num)]), c_char_p) + self.buffer = c_void_p(None) + self.num = num + + def bool(self, values): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BOOL_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + self.buffer_type = FieldType.C_BOOL + self.buffer_length = sizeof(c_bool) + + def tinyint(self, values): + self.buffer_type = FieldType.C_TINYINT + self.buffer_length = sizeof(c_int8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint(self, values): + self.buffer_type = FieldType.C_SMALLINT + self.buffer_length = sizeof(c_int16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int(self, values): + self.buffer_type = FieldType.C_INT + self.buffer_length = sizeof(c_int32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint(self, values): + self.buffer_type = FieldType.C_BIGINT + self.buffer_length = sizeof(c_int64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def float(self, values): + self.buffer_type = FieldType.C_FLOAT + self.buffer_length = sizeof(c_float) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_float * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_FLOAT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def double(self, values): + self.buffer_type = FieldType.C_DOUBLE + self.buffer_length = sizeof(c_double) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_double * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_DOUBLE_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def binary(self, values): + self.num = len(values) + self.buffer = cast(c_char_p("".join(filter(_is_not_none, values)).encode("utf-8")), c_void_p) + self.length = (c_int * len(values))(*[len(value) if value is not None else 0 for value in values]) + self.buffer_type = FieldType.C_BINARY + self.is_null = cast((c_byte * self.num)(*[1 if v == None else 0 for v in values]), c_char_p) + + def timestamp(self, values, precision=PrecisionEnum.Milliseconds): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + buffer = buffer_type(*[_datetime_to_timestamp(value, precision) for value in values]) + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(buffer, c_void_p) + self.buffer_length = sizeof(c_int64) + self.num = len(values) + + def nchar(self, values): + # type: (list[str]) -> None + if sys.version_info < (3, 0): + _bytes = [bytes(value) if value is not None else None for value in values] + buffer_length = max(len(b) + 1 for b in _bytes if b is not None) + buffers = [ + create_string_buffer(b, buffer_length) if b is not None else create_string_buffer(buffer_length) + for b in _bytes + ] + buffer_all = b''.join(v[:] for v in buffers) + self.buffer = cast(c_char_p(buffer_all), c_void_p) + else: + _bytes = [value.encode("utf-8") if value is not None else None for value in values] + buffer_length = max(len(b) for b in _bytes if b is not None) + self.buffer = cast( + c_char_p( + b"".join( + [ + create_string_buffer(b, buffer_length) + if b is not None + else create_string_buffer(buffer_length) + for b in _bytes + ] + ) + ), + c_void_p, + ) + self.length = (c_int32 * len(values))(*[len(b) if b is not None else 0 for b in _bytes]) + self.buffer_length = buffer_length + self.num = len(values) + self.is_null = cast((c_byte * self.num)(*[1 if v == None else 0 for v in values]), c_char_p) + self.buffer_type = FieldType.C_NCHAR + + def tinyint_unsigned(self, values): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer_length = sizeof(c_uint8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint_unsigned(self, values): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer_length = sizeof(c_uint16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int_unsigned(self, values): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer_length = sizeof(c_uint32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint_unsigned(self, values): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer_length = sizeof(c_uint64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + +def new_bind_param(): + # type: () -> TaosBind + return TaosBind() + + +def new_bind_params(size): + # type: (int) -> Array[TaosBind] + return (TaosBind * size)() + + +def new_multi_bind(): + # type: () -> TaosMultiBind + return TaosMultiBind() + + +def new_multi_binds(size): + # type: (int) -> Array[TaosMultiBind] + return (TaosMultiBind * size)() diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 660707bfcd04edb9d815b38d8ae806f35d2bfe2b..51e9a8667ddcab3d5d67da8be429f460f33d9eed 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -1,295 +1,839 @@ +# encoding:UTF-8 + import ctypes -from .constants import FieldType -from .error import * -import math -import datetime import platform +import sys +from ctypes import * +try: + from typing import Any +except: + pass + +from .error import * +from .bind import * +from .field import * + + +# stream callback +stream_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p) +stream_callback2_type = CFUNCTYPE(None, c_void_p) + +# C interface class +class TaosOption: + Locale = (0,) + Charset = (1,) + Timezone = (2,) + ConfigDir = (3,) + ShellActivityTimer = (4,) + MaxOptions = (5,) + + +def _load_taos_linux(): + return ctypes.CDLL("libtaos.so") + + +def _load_taos_darwin(): + return ctypes.CDLL("libtaos.dylib") + + +def _load_taos_windows(): + return ctypes.windll.LoadLibrary("taos") -def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=milli/1000.0) +def _load_taos(): + load_func = { + "Linux": _load_taos_linux, + "Darwin": _load_taos_darwin, + "Windows": _load_taos_windows, + } + try: + return load_func[platform.system()]() + except: + sys.exit("unsupported platform to TDengine connector") -def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=micro / 1000000.0) +_libtaos = _load_taos() +_libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) +_libtaos.taos_init.restype = None +_libtaos.taos_connect.restype = ctypes.c_void_p +_libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) +_libtaos.taos_errstr.restype = ctypes.c_char_p +_libtaos.taos_subscribe.restype = ctypes.c_void_p +_libtaos.taos_consume.restype = ctypes.c_void_p +_libtaos.taos_fetch_lengths.restype = ctypes.POINTER(ctypes.c_int) +_libtaos.taos_free_result.restype = None +_libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +try: + _libtaos.taos_stmt_errstr.restype = c_char_p +except AttributeError: + None +finally: + None -def _convert_nanosecond_to_datetime(nanosec): - return nanosec +_libtaos.taos_options.restype = None -def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row + +def taos_options(option, *args): + # type: (TaosOption, Any) -> None + _libtaos.taos_options(option, *args) + + +def taos_init(): + # type: () -> None """ - _timestamp_converter = _convert_millisecond_to_datetime - if precision == FieldType.C_TIMESTAMP_MILLI: - _timestamp_converter = _convert_millisecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_MICRO: - _timestamp_converter = _convert_microsecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_NANO: - _timestamp_converter = _convert_nanosecond_to_datetime - else: - raise DatabaseError("Unknown precision returned from database") + C: taos_init + """ + _libtaos.taos_init() + + +_libtaos.taos_cleanup.restype = None - return [ - None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_int64))[ - :abs(num_of_rows)]] +def taos_cleanup(): + # type: () -> None + """Cleanup workspace.""" + _libtaos.taos_cleanup() -def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row + +_libtaos.taos_get_client_info.restype = c_char_p + + +def taos_get_client_info(): + # type: () -> str + """Get client version info. + 获取客户端版本信息。 """ - return [ - None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_byte))[ - :abs(num_of_rows)]] + return _libtaos.taos_get_client_info().decode() + + +_libtaos.taos_get_server_info.restype = c_char_p +_libtaos.taos_get_server_info.argtypes = (c_void_p,) + + +def taos_get_server_info(connection): + # type: (c_void_p) -> str + return _libtaos.taos_get_server_info(connection).decode() + + +_libtaos.taos_close.restype = None +_libtaos.taos_close.argtypes = (c_void_p,) + + +def taos_close(connection): + # type: (c_void_p) -> None + """Close the TAOS* connection""" + _libtaos.taos_close(connection) + + +_libtaos.taos_connect.restype = c_void_p +_libtaos.taos_connect.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + + +def taos_connect(host=None, user="root", password="taosdata", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p + """Create TDengine database connection. + 创建数据库连接,初始化连接上下文。其中需要用户提供的参数包含: + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - password: user password / 用户密码 + - db: database name (optional) + - port: server port -def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + @rtype: c_void_p, TDengine handle """ - return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # password + try: + _password = c_char_p(password.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") + # port + try: + _port = c_uint16(port) + except TypeError: + raise TypeError("port is expected as an uint16") -def _crow_tinyint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + connection = cast(_libtaos.taos_connect(_host, _user, _password, _db, _port), c_void_p) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_connect_auth.restype = c_void_p +_libtaos.taos_connect_auth.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + + +def taos_connect_auth(host=None, user="root", auth="", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p """ - return [ - None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ubyte))[ - :abs(num_of_rows)]] + 创建数据库连接,初始化连接上下文。其中需要用户提供的参数包含: + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - auth: base64 encoded auth token + - db: database name (optional) + - port: server port -def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row + @rtype: c_void_p, TDengine handle """ - return [ - None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_short))[ - :abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # auth + try: + _auth = c_char_p(auth.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") + + # port + try: + _port = c_int(port) + except TypeError: + raise TypeError("port is expected as an int") + + connection = c_void_p(_libtaos.taos_connect_auth(_host, _user, _auth, _db, _port)) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_query.restype = c_void_p +_libtaos.taos_query.argtypes = c_void_p, c_char_p + + +def taos_query(connection, sql): + # type: (c_void_p, str) -> c_void_p + """Run SQL + + - sql: str, sql string to run + + @return: TAOS_RES*, result pointer -def _crow_smallint_unsigned_to_python( - data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row """ - return [ - None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ushort))[ - :abs(num_of_rows)]] + try: + ptr = c_char_p(sql.encode("utf-8")) + res = c_void_p(_libtaos.taos_query(connection, ptr)) + errno = taos_errno(res) + if errno != 0: + errstr = taos_errstr(res) + taos_free_result(res) + raise ProgrammingError(errstr, errno) + return res + except AttributeError: + raise AttributeError("sql is expected as a string") -def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row +async_query_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_query_a.restype = None +_libtaos.taos_query_a.argtypes = c_void_p, c_char_p, async_query_callback_type, c_void_p + + +def taos_query_a(connection, sql, callback, param): + # type: (c_void_p, str, async_query_callback_type, c_void_p) -> c_void_p + _libtaos.taos_query_a(connection, c_char_p(sql.encode("utf-8")), async_query_callback_type(callback), param) + + +async_fetch_rows_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_fetch_rows_a.restype = None +_libtaos.taos_fetch_rows_a.argtypes = c_void_p, async_fetch_rows_callback_type, c_void_p + + +def taos_fetch_rows_a(result, callback, param): + # type: (c_void_p, async_fetch_rows_callback_type, c_void_p) -> c_void_p + _libtaos.taos_fetch_rows_a(result, async_fetch_rows_callback_type(callback), param) + + +def taos_affected_rows(result): + # type: (c_void_p) -> c_int + """The affected rows after runing query""" + return _libtaos.taos_affected_rows(result) + + +subscribe_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p, c_int) +_libtaos.taos_subscribe.restype = c_void_p +# _libtaos.taos_subscribe.argtypes = c_void_p, c_int, c_char_p, c_char_p, subscribe_callback_type, c_void_p, c_int + + +def taos_subscribe(connection, restart, topic, sql, interval, callback=None, param=None): + # type: (c_void_p, bool, str, str, c_int, subscribe_callback_type, c_void_p | None) -> c_void_p + """Create a subscription + @restart boolean, + @sql string, sql statement for data query, must be a 'select' statement. + @topic string, name of this subscription """ - return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + if callback != None: + callback = subscribe_callback_type(callback) + if param != None: + param = c_void_p(param) + return c_void_p( + _libtaos.taos_subscribe( + connection, + 1 if restart else 0, + c_char_p(topic.encode("utf-8")), + c_char_p(sql.encode("utf-8")), + callback or None, + param, + interval, + ) + ) + + +_libtaos.taos_consume.restype = c_void_p +_libtaos.taos_consume.argstype = c_void_p, + + +def taos_consume(sub): + """Consume data of a subscription""" + return c_void_p(_libtaos.taos_consume(sub)) + + +_libtaos.taos_unsubscribe.restype = None +_libtaos.taos_unsubscribe.argstype = c_void_p, c_int + + +def taos_unsubscribe(sub, keep_progress): + """Cancel a subscription""" + _libtaos.taos_unsubscribe(sub, 1 if keep_progress else 0) + + +def taos_use_result(result): + """Use result after calling self.query, it's just for 1.6.""" + fields = [] + pfields = taos_fetch_fields_raw(result) + for i in range(taos_field_count(result)): + fields.append( + { + "name": pfields[i].name, + "bytes": pfields[i].bytes, + "type": pfields[i].type, + } + ) + + return fields + + +_libtaos.taos_fetch_block.restype = c_int +_libtaos.taos_fetch_block.argtypes = c_void_p, c_void_p + + +def taos_fetch_block_raw(result): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + return pblock, abs(num_of_rows) + + +def taos_fetch_block(result, fields=None, field_count=None): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + precision = taos_result_precision(result) + if fields == None: + fields = taos_fetch_fields(result) + if field_count == None: + field_count = taos_field_count(result) + blocks = [None] * field_count + fieldLen = taos_fetch_lengths(result, field_count) + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]["type"] not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i]["type"]](data, num_of_rows, fieldLen[i], precision) + + return blocks, abs(num_of_rows) + + +_libtaos.taos_fetch_row.restype = c_void_p +_libtaos.taos_fetch_row.argtypes = (c_void_p,) + + +def taos_fetch_row_raw(result): + # type: (c_void_p) -> c_void_p + row = c_void_p(_libtaos.taos_fetch_row(result)) + if row: + return row + return None + + +def taos_fetch_row(result, fields): + # type: (c_void_p, Array[TaosField]) -> tuple(c_void_p, int) + pblock = ctypes.c_void_p(0) + pblock = taos_fetch_row_raw(result) + if pblock: + num_of_rows = 1 + precision = taos_result_precision(result) + field_count = taos_field_count(result) + blocks = [None] * field_count + field_lens = taos_fetch_lengths(result, field_count) + for i in range(field_count): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = [None] + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, num_of_rows, field_lens[i], precision) + else: + return None, 0 + return blocks, abs(num_of_rows) + + +_libtaos.taos_free_result.argtypes = (c_void_p,) -def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row +def taos_free_result(result): + # type: (c_void_p) -> None + if result != None: + _libtaos.taos_free_result(result) + + +_libtaos.taos_field_count.restype = c_int +_libtaos.taos_field_count.argstype = (c_void_p,) + + +def taos_field_count(result): + # type: (c_void_p) -> int + return _libtaos.taos_field_count(result) + + +def taos_num_fields(result): + # type: (c_void_p) -> int + return _libtaos.taos_num_fields(result) + + +_libtaos.taos_fetch_fields.restype = c_void_p +_libtaos.taos_fetch_fields.argstype = (c_void_p,) + + +def taos_fetch_fields_raw(result): + # type: (c_void_p) -> c_void_p + return c_void_p(_libtaos.taos_fetch_fields(result)) + + +def taos_fetch_fields(result): + # type: (c_void_p) -> TaosFields + fields = taos_fetch_fields_raw(result) + count = taos_field_count(result) + return TaosFields(fields, count) + + +def taos_fetch_lengths(result, field_count=None): + # type: (c_void_p, int) -> Array[int] + """Make sure to call taos_fetch_row or taos_fetch_block before fetch_lengths""" + lens = _libtaos.taos_fetch_lengths(result) + if field_count == None: + field_count = taos_field_count(result) + if not lens: + raise OperationalError("field length empty, use taos_fetch_row/block before it") + return lens[:field_count] + + +def taos_result_precision(result): + # type: (c_void_p) -> c_int + return _libtaos.taos_result_precision(result) + + +_libtaos.taos_errno.restype = c_int +_libtaos.taos_errno.argstype = (c_void_p,) + + +def taos_errno(result): + # type: (ctypes.c_void_p) -> c_int + """Return the error number.""" + return _libtaos.taos_errno(result) + + +_libtaos.taos_errstr.restype = c_char_p +_libtaos.taos_errstr.argstype = (c_void_p,) + + +def taos_errstr(result=c_void_p(None)): + # type: (ctypes.c_void_p) -> str + """Return the error styring""" + return _libtaos.taos_errstr(result).decode("utf-8") + + +_libtaos.taos_stop_query.restype = None +_libtaos.taos_stop_query.argstype = (c_void_p,) + + +def taos_stop_query(result): + # type: (ctypes.c_void_p) -> None + """Stop current query""" + return _libtaos.taos_stop_query(result) + + +_libtaos.taos_load_table_info.restype = c_int +_libtaos.taos_load_table_info.argstype = (c_void_p, c_char_p) + + +def taos_load_table_info(connection, tables): + # type: (ctypes.c_void_p, str) -> None + """Stop current query""" + errno = _libtaos.taos_load_table_info(connection, c_char_p(tables.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + raise OperationalError(msg, errno) + + +_libtaos.taos_validate_sql.restype = c_int +_libtaos.taos_validate_sql.argstype = (c_void_p, c_char_p) + + +def taos_validate_sql(connection, sql): + # type: (ctypes.c_void_p, str) -> None | str + """Get taosd server info""" + errno = _libtaos.taos_validate_sql(connection, ctypes.c_char_p(sql.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + return msg + return None + + +_libtaos.taos_print_row.restype = c_int +_libtaos.taos_print_row.argstype = (c_char_p, c_void_p, c_void_p, c_int) + + +def taos_print_row(row, fields, num_fields, buffer_size=4096): + # type: (ctypes.c_void_p, ctypes.c_void_p | TaosFields, int, int) -> str + """Print an row to string""" + p = ctypes.create_string_buffer(buffer_size) + if isinstance(fields, TaosFields): + _libtaos.taos_print_row(p, row, fields.as_ptr(), num_fields) + else: + _libtaos.taos_print_row(p, row, fields, num_fields) + if p: + return p.value.decode("utf-8") + raise OperationalError("taos_print_row failed") + + +_libtaos.taos_select_db.restype = c_int +_libtaos.taos_select_db.argstype = (c_void_p, c_char_p) + + +def taos_select_db(connection, db): + # type: (ctypes.c_void_p, str) -> None + """Select database, eq to sql: use """ + res = _libtaos.taos_select_db(connection, ctypes.c_char_p(db.encode("utf-8"))) + if res != 0: + raise DatabaseError("select database error", res) + + +try: + _libtaos.taos_open_stream.restype = c_void_p + _libtaos.taos_open_stream.argstype = c_void_p, c_char_p, stream_callback_type, c_int64, c_void_p, Any +except: + pass + + +def taos_open_stream(connection, sql, callback, stime=0, param=None, callback2=None): + # type: (ctypes.c_void_p, str, stream_callback_type, c_int64, c_void_p, c_void_p) -> ctypes.pointer + if callback2 != None: + callback2 = stream_callback2_type(callback2) + """Open an stream""" + return c_void_p( + _libtaos.taos_open_stream( + connection, ctypes.c_char_p(sql.encode("utf-8")), stream_callback_type(callback), stime, param, callback2 + ) + ) + + +_libtaos.taos_close_stream.restype = None +_libtaos.taos_close_stream.argstype = (c_void_p,) + + +def taos_close_stream(stream): + # type: (c_void_p) -> None + """Open an stream""" + return _libtaos.taos_close_stream(stream) + + +_libtaos.taos_stmt_init.restype = c_void_p +_libtaos.taos_stmt_init.argstype = (c_void_p,) + + +def taos_stmt_init(connection): + # type: (c_void_p) -> (c_void_p) + """Create a statement query + @param(connection): c_void_p TAOS* + @rtype: c_void_p, *TAOS_STMT + """ + return c_void_p(_libtaos.taos_stmt_init(connection)) + + +_libtaos.taos_stmt_prepare.restype = c_int +_libtaos.taos_stmt_prepare.argstype = (c_void_p, c_char_p, c_int) + + +def taos_stmt_prepare(stmt, sql): + # type: (ctypes.c_void_p, str) -> None + """Prepare a statement query + @stmt: c_void_p TAOS_STMT* """ - return [ - None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint))[ - :abs(num_of_rows)]] + buffer = sql.encode("utf-8") + res = _libtaos.taos_stmt_prepare(stmt, ctypes.c_char_p(buffer), len(buffer)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +_libtaos.taos_stmt_close.restype = c_int +_libtaos.taos_stmt_close.argstype = (c_void_p,) -def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row + +def taos_stmt_close(stmt): + # type: (ctypes.c_void_p) -> None + """Close a statement query + @stmt: c_void_p TAOS_STMT* """ - return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_close(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_errstr.restype = c_char_p + _libtaos.taos_stmt_errstr.argstype = (c_void_p,) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_errstr" % taos_get_client_info()) -def _crow_bigint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row + +def taos_stmt_errstr(stmt): + # type: (ctypes.c_void_p) -> str + """Get error message from stetement query + @stmt: c_void_p TAOS_STMT* """ - return [ - None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint64))[ - :abs(num_of_rows)]] + err = c_char_p(_libtaos.taos_stmt_errstr(stmt)) + if err: + return err.value.decode("utf-8") + +try: + _libtaos.taos_stmt_set_tbname.restype = c_int + _libtaos.taos_stmt_set_tbname.argstype = (c_void_p, c_char_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname" % taos_get_client_info()) -def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C float row to python row + +def taos_stmt_set_tbname(stmt, name): + # type: (ctypes.c_void_p, str) -> None + """Set table name of a statement query if exists. + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_set_tbname(stmt, c_char_p(name.encode("utf-8"))) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_set_tbname_tags.restype = c_int + _libtaos.taos_stmt_set_tbname_tags.argstype = (c_void_p, c_char_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname_tags" % taos_get_client_info()) + + + +def taos_stmt_set_tbname_tags(stmt, name, tags): + # type: (c_void_p, str, c_void_p) -> None + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* + """ + res = _libtaos.taos_stmt_set_tbname_tags(stmt, ctypes.c_char_p(name.encode("utf-8")), tags) + + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C double row to python row +_libtaos.taos_stmt_is_insert.restype = c_int +_libtaos.taos_stmt_is_insert.argstype = (c_void_p, POINTER(c_int)) + + +def taos_stmt_is_insert(stmt): + # type: (ctypes.c_void_p) -> bool + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + is_insert = ctypes.c_int() + res = _libtaos.taos_stmt_is_insert(stmt, ctypes.byref(is_insert)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return is_insert == 0 + + +_libtaos.taos_stmt_num_params.restype = c_int +_libtaos.taos_stmt_num_params.argstype = (c_void_p, POINTER(c_int)) -def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row +def taos_stmt_num_params(stmt): + # type: (ctypes.c_void_p) -> int + """Params number of the current statement query. + @stmt: TAOS_STMT* """ - assert(nbytes is not None) - return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( - 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + num_params = ctypes.c_int() + res = _libtaos.taos_stmt_num_params(stmt, ctypes.byref(num_params)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return num_params.value -def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row +_libtaos.taos_stmt_bind_param.restype = c_int +_libtaos.taos_stmt_bind_param.argstype = (c_void_p, c_void_p) + + +def taos_stmt_bind_param(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - if num_of_rows >= 0: - tmpstr = ctypes.c_char_p(data) - res.append(tmpstr.value.decode()) - else: - res.append((ctypes.cast(data + nbytes * i, - ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) - except ValueError: - res.append(None) + # ptr = ctypes.cast(bind, POINTER(TaosBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_bind_param_batch.restype = c_int + _libtaos.taos_stmt_bind_param_batch.argstype = (c_void_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_param_batch" % taos_get_client_info()) - return res -def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row +def taos_stmt_bind_param_batch(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosMultiBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - rbyte = ctypes.cast( - data + nbytes * i, - ctypes.POINTER( - ctypes.c_short))[ - :1].pop() - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()[0:rbyte]) - except ValueError: - res.append(None) - return res - - -def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row + # ptr = ctypes.cast(bind, POINTER(TaosMultiBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param_batch(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_bind_single_param_batch.restype = c_int + _libtaos.taos_stmt_bind_single_param_batch.argstype = (c_void_p, c_void_p, c_int) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_single_param_batch" % taos_get_client_info()) + + +def taos_stmt_bind_single_param_batch(stmt, bind, col): + # type: (ctypes.c_void_p, Array[TaosMultiBind], c_int) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_MULTI_BIND* + @col: column index """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()) - except ValueError: - res.append(None) - return res - - -_CONVERT_FUNC = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -_CONVERT_FUNC_BLOCK = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python_block, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -# Corresponding TAOS_FIELD structure in C - - -class TaosField(ctypes.Structure): - _fields_ = [('name', ctypes.c_char * 65), - ('type', ctypes.c_char), - ('bytes', ctypes.c_short)] + res = _libtaos.taos_stmt_bind_single_param_batch(stmt, bind, col) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -# C interface class +_libtaos.taos_stmt_add_batch.restype = c_int +_libtaos.taos_stmt_add_batch.argstype = (c_void_p,) -def _load_taos_linux(): - return ctypes.CDLL('libtaos.so') +def taos_stmt_add_batch(stmt): + # type: (ctypes.c_void_p) -> None + """Add current params into batch + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_add_batch(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos_darwin(): - return ctypes.CDLL('libtaos.dylib') +_libtaos.taos_stmt_execute.restype = c_int +_libtaos.taos_stmt_execute.argstype = (c_void_p,) -def _load_taos_windows(): - return ctypes.windll.LoadLibrary('taos') +def taos_stmt_execute(stmt): + # type: (ctypes.c_void_p) -> None + """Execute a statement query + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_execute(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos(): - load_func = { - 'Linux': _load_taos_linux, - 'Darwin': _load_taos_darwin, - 'Windows': _load_taos_windows, - } - try: - return load_func[platform.system()]() - except: - sys.exit('unsupported platform to TDengine connector') +_libtaos.taos_stmt_use_result.restype = c_void_p +_libtaos.taos_stmt_use_result.argstype = (c_void_p,) -class CTaosInterface(object): - libtaos = _load_taos() - - libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) - libtaos.taos_init.restype = None - libtaos.taos_connect.restype = ctypes.c_void_p - # libtaos.taos_use_result.restype = ctypes.c_void_p - libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) - libtaos.taos_errstr.restype = ctypes.c_char_p - libtaos.taos_subscribe.restype = ctypes.c_void_p - libtaos.taos_consume.restype = ctypes.c_void_p - libtaos.taos_fetch_lengths.restype = ctypes.c_void_p - libtaos.taos_free_result.restype = None - libtaos.taos_errno.restype = ctypes.c_int - libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +def taos_stmt_use_result(stmt): + # type: (ctypes.c_void_p) -> None + """Get result of the statement. + @stmt: TAOS_STMT* + """ + result = c_void_p(_libtaos.taos_stmt_use_result(stmt)) + if result == None: + raise StatementError(taos_stmt_errstr(stmt)) + return result + +try: + _libtaos.taos_insert_lines.restype = c_int + _libtaos.taos_insert_lines.argstype = c_void_p, c_void_p, c_int +except AttributeError: + print("WARNING: libtaos(%s) does not support insert_lines" % taos_get_client_info()) + + + +def taos_insert_lines(connection, lines): + # type: (c_void_p, list[str] | tuple(str)) -> None + num_of_lines = len(lines) + lines = (c_char_p(line.encode("utf-8")) for line in lines) + lines_type = ctypes.c_char_p * num_of_lines + p_lines = lines_type(*lines) + errno = _libtaos.taos_insert_lines(connection, p_lines, num_of_lines) + if errno != 0: + raise LinesError("insert lines error", errno) + + +class CTaosInterface(object): def __init__(self, config=None): - ''' + """ Function to initialize the class @host : str, hostname to connect @user : str, username to connect to server @@ -298,304 +842,46 @@ class CTaosInterface(object): @config : str, config directory @rtype : None - ''' + """ if config is None: self._config = ctypes.c_char_p(None) else: try: - self._config = ctypes.c_char_p(config.encode('utf-8')) + self._config = ctypes.c_char_p(config.encode("utf-8")) except AttributeError: raise AttributeError("config is expected as a str") if config is not None: - CTaosInterface.libtaos.taos_options(3, self._config) + taos_options(3, self._config) - CTaosInterface.libtaos.taos_init() + taos_init() @property def config(self): - """ Get current config - """ + """Get current config""" return self._config - def connect( - self, - host=None, - user="root", - password="taosdata", - db=None, - port=0): - ''' + def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + """ Function to connect to server @rtype: c_void_p, TDengine handle - ''' - # host - try: - _host = ctypes.c_char_p(host.encode( - "utf-8")) if host is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("host is expected as a str") - - # user - try: - _user = ctypes.c_char_p(user.encode("utf-8")) - except AttributeError: - raise AttributeError("user is expected as a str") - - # password - try: - _password = ctypes.c_char_p(password.encode("utf-8")) - except AttributeError: - raise AttributeError("password is expected as a str") - - # db - try: - _db = ctypes.c_char_p( - db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("db is expected as a str") - - # port - try: - _port = ctypes.c_int(port) - except TypeError: - raise TypeError("port is expected as an int") - - connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( - _host, _user, _password, _db, _port)) - - if connection.value is None: - print('connect to TDengine failed') - raise ConnectionError("connect to TDengine failed") - # sys.exit(1) - # else: - # print('connect to TDengine success') - - return connection - - @staticmethod - def close(connection): - '''Close the TDengine handle - ''' - CTaosInterface.libtaos.taos_close(connection) - # print('connection is closed') - - @staticmethod - def query(connection, sql): - '''Run SQL - - @sql: str, sql string to run - - @rtype: 0 on success and -1 on failure - ''' - try: - return CTaosInterface.libtaos.taos_query( - connection, ctypes.c_char_p(sql.encode('utf-8'))) - except AttributeError: - raise AttributeError("sql is expected as a string") - # finally: - # CTaosInterface.libtaos.close(connection) - - @staticmethod - def affectedRows(result): - """The affected rows after runing query - """ - return CTaosInterface.libtaos.taos_affected_rows(result) - - @staticmethod - def insertLines(connection, lines): - ''' - insert through lines protocol - @lines: list of str - @rtype: tsdb error codes - ''' - numLines = len(lines) - c_lines_type = ctypes.c_char_p*numLines - c_lines = c_lines_type() - for i in range(numLines): - c_lines[i] = ctypes.c_char_p(lines[i].encode('utf-8')) - return CTaosInterface.libtaos.taos_insert_lines(connection, c_lines, ctypes.c_int(numLines)) - - @staticmethod - def subscribe(connection, restart, topic, sql, interval): - """Create a subscription - @restart boolean, - @sql string, sql statement for data query, must be a 'select' statement. - @topic string, name of this subscription - """ - return ctypes.c_void_p(CTaosInterface.libtaos.taos_subscribe( - connection, - 1 if restart else 0, - ctypes.c_char_p(topic.encode('utf-8')), - ctypes.c_char_p(sql.encode('utf-8')), - None, - None, - interval)) - - @staticmethod - def consume(sub): - """Consume data of a subscription - """ - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_consume(sub)) - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.libtaos.taos_num_fields(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - return result, fields - - @staticmethod - def unsubscribe(sub, keepProgress): - """Cancel a subscription - """ - CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) - - @staticmethod - def useResult(result): - '''Use result after calling self.query - ''' - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - - return fields - - @staticmethod - def fetchBlock(result, fields): - pblock = ctypes.c_void_p(0) - num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - result, ctypes.byref(pblock)) - if num_of_rows == 0: - return None, 0 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: - raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - - return blocks, abs(num_of_rows) - - @staticmethod - def fetchRow(result, fields): - pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock: - num_of_rows = 1 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError( - "Invalid data type returned from database") - if data is None: - blocks[i] = [None] - else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - else: - return None, 0 - return blocks, abs(num_of_rows) - - @staticmethod - def freeResult(result): - CTaosInterface.libtaos.taos_free_result(result) - result.value = None - - @staticmethod - def fieldsCount(result): - return CTaosInterface.libtaos.taos_field_count(result) - - @staticmethod - def fetchFields(result): - return CTaosInterface.libtaos.taos_fetch_fields(result) - - # @staticmethod - # def fetchRow(result, fields): - # l = [] - # row = CTaosInterface.libtaos.taos_fetch_row(result) - # if not row: - # return None - - # for i in range(len(fields)): - # l.append(CTaosInterface.getDataValue( - # row[i], fields[i]['type'], fields[i]['bytes'])) - - # return tuple(l) - - # @staticmethod - # def getDataValue(data, dtype, byte): - # ''' - # ''' - # if not data: - # return None - - # if (dtype == CTaosInterface.TSDB_DATA_TYPE_BOOL): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TINYINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_SMALLINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): - # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): - # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') - - @staticmethod - def errno(result): - """Return the error number. - """ - return CTaosInterface.libtaos.taos_errno(result) - - @staticmethod - def errStr(result): - """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') + return taos_connect(host, user, password, db, port) -if __name__ == '__main__': +if __name__ == "__main__": cinter = CTaosInterface() conn = cinter.connect() - result = cinter.query(conn, 'show databases') + result = cinter.query(conn, "show databases") - print('Query Affected rows: {}'.format(cinter.affectedRows(result))) + print("Query Affected rows: {}".format(cinter.affected_rows(result))) - fields = CTaosInterface.useResult(result) + fields = taos_fetch_fields_raw(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, fields) + data, num_of_rows = taos_fetch_block(result, fields) print(data) - cinter.freeResult(result) + cinter.free_result(result) cinter.close(conn) diff --git a/src/connector/python/taos/connection.py b/src/connector/python/taos/connection.py index 88d06cd7186018788aeb25c982fc205441193cb8..7857c8c706dbe27fd9440e6bf2eb698b6822650e 100644 --- a/src/connector/python/taos/connection.py +++ b/src/connector/python/taos/connection.py @@ -1,11 +1,15 @@ -from .cursor import TDengineCursor -from .subscription import TDengineSubscription -from .cinterface import CTaosInterface +# encoding:UTF-8 +from types import FunctionType +from .cinterface import * +from .cursor import TaosCursor +from .subscription import TaosSubscription +from .statement import TaosStmt +from .stream import TaosStream +from .result import * -class TDengineConnection(object): - """ TDengine connection object - """ +class TaosConnection(object): + """TDengine connection object""" def __init__(self, *args, **kwargs): self._conn = None @@ -21,63 +25,130 @@ class TDengineConnection(object): def config(self, **kwargs): # host - if 'host' in kwargs: - self._host = kwargs['host'] + if "host" in kwargs: + self._host = kwargs["host"] # user - if 'user' in kwargs: - self._user = kwargs['user'] + if "user" in kwargs: + self._user = kwargs["user"] # password - if 'password' in kwargs: - self._password = kwargs['password'] + if "password" in kwargs: + self._password = kwargs["password"] # database - if 'database' in kwargs: - self._database = kwargs['database'] + if "database" in kwargs: + self._database = kwargs["database"] # port - if 'port' in kwargs: - self._port = kwargs['port'] + if "port" in kwargs: + self._port = kwargs["port"] # config - if 'config' in kwargs: - self._config = kwargs['config'] + if "config" in kwargs: + self._config = kwargs["config"] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect( - self._host, - self._user, - self._password, - self._database, - self._port) + self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) def close(self): - """Close current connection. - """ - return CTaosInterface.close(self._conn) - - def subscribe(self, restart, topic, sql, interval): - """Create a subscription. - """ + """Close current connection.""" + if self._conn: + taos_close(self._conn) + self._conn = None + + @property + def client_info(self): + # type: () -> str + return taos_get_client_info() + + @property + def server_info(self): + # type: () -> str + return taos_get_server_info(self._conn) + + def select_db(self, database): + # type: (str) -> None + taos_select_db(self._conn, database) + + def execute(self, sql): + # type: (str) -> None + """Simplely execute sql ignoring the results""" + res = taos_query(self._conn, sql) + taos_free_result(res) + + def query(self, sql): + # type: (str) -> TaosResult + result = taos_query(self._conn, sql) + return TaosResult(result, True, self) + + def query_a(self, sql, callback, param): + # type: (str, async_query_callback_type, c_void_p) -> None + """Asynchronously query a sql with callback function""" + taos_query_a(self._conn, sql, callback, param) + + def subscribe(self, restart, topic, sql, interval, callback=None, param=None): + # type: (bool, str, str, int, subscribe_callback_type, c_void_p) -> TaosSubscription + """Create a subscription.""" if self._conn is None: return None - sub = CTaosInterface.subscribe( - self._conn, restart, topic, sql, interval) - return TDengineSubscription(sub) + sub = taos_subscribe(self._conn, restart, topic, sql, interval, callback, param) + return TaosSubscription(sub, callback != None) - def insertLines(self, lines): - """ - insert lines through line protocol - """ + def statement(self, sql=None): + # type: (str | None) -> TaosStmt if self._conn is None: return None - return CTaosInterface.insertLines(self._conn, lines) - - def cursor(self): - """Return a new Cursor object using the connection. + stmt = taos_stmt_init(self._conn) + if sql != None: + taos_stmt_prepare(stmt, sql) + + return TaosStmt(stmt) + + def load_table_info(self, tables): + # type: (str) -> None + taos_load_table_info(self._conn, tables) + + def stream(self, sql, callback, stime=0, param=None, callback2=None): + # type: (str, Callable[[Any, TaosResult, TaosRows], None], int, Any, c_void_p) -> TaosStream + # cb = cast(callback, stream_callback_type) + # ref = byref(cb) + + stream = taos_open_stream(self._conn, sql, callback, stime, param, callback2) + return TaosStream(stream) + + def insert_lines(self, lines): + # type: (list[str]) -> None + """Line protocol and schemaless support + + ## Example + + ```python + import taos + conn = taos.connect() + conn.exec("drop database if exists test") + conn.select_db("test") + lines = [ + 'ste,t2=5,t3=L"ste" c1=true,c2=4,c3="string" 1626056811855516532', + ] + conn.insert_lines(lines) + ``` + + ## Exception + + ```python + try: + conn.insert_lines(lines) + except SchemalessError as err: + print(err) + ``` """ - return TDengineCursor(self) + return taos_insert_lines(self._conn, lines) + + def cursor(self): + # type: () -> TaosCursor + """Return a new Cursor object using the connection.""" + return TaosCursor(self) def commit(self): """Commit any pending transaction to the database. @@ -87,17 +158,18 @@ class TDengineConnection(object): pass def rollback(self): - """Void functionality - """ + """Void functionality""" pass def clear_result_set(self): - """Clear unused result set on this connection. - """ + """Clear unused result set on this connection.""" pass + def __del__(self): + self.close() + if __name__ == "__main__": - conn = TDengineConnection(host='192.168.1.107') + conn = TaosConnection() conn.close() print("Hello world") diff --git a/src/connector/python/taos/constants.py b/src/connector/python/taos/constants.py index 85689b02db76b0032558b983d8ae8d9297229c42..b500df627c22919e7aab964504ccbe50c573c1c5 100644 --- a/src/connector/python/taos/constants.py +++ b/src/connector/python/taos/constants.py @@ -1,12 +1,11 @@ +# encoding:UTF-8 + """Constants in TDengine python """ -from .dbapi import * - - class FieldType(object): - """TDengine Field Types - """ + """TDengine Field Types""" + # type_code C_NULL = 0 C_BOOL = 1 @@ -34,9 +33,9 @@ class FieldType(object): C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 C_BIGINT_UNSIGNED_NULL = 18446744073709551615 - C_FLOAT_NULL = float('nan') - C_DOUBLE_NULL = float('nan') - C_BINARY_NULL = bytearray([int('0xff', 16)]) + C_FLOAT_NULL = float("nan") + C_DOUBLE_NULL = float("nan") + C_BINARY_NULL = bytearray([int("0xff", 16)]) # Timestamp precision definition C_TIMESTAMP_MILLI = 0 C_TIMESTAMP_MICRO = 1 diff --git a/src/connector/python/taos/cursor.py b/src/connector/python/taos/cursor.py index d443ec95d07bd970da96c6280674dd5c9477a38f..5d21ff95af5d81367e7143d001cc688d90877b67 100644 --- a/src/connector/python/taos/cursor.py +++ b/src/connector/python/taos/cursor.py @@ -1,18 +1,18 @@ -from .cinterface import CTaosInterface +# encoding:UTF-8 +from .cinterface import * from .error import * from .constants import FieldType +from .result import * -# querySeqNum = 0 - -class TDengineCursor(object): +class TaosCursor(object): """Database cursor which is used to manage the context of a fetch operation. Attributes: .description: Read-only attribute consists of 7-item sequences: - > name (mondatory) - > type_code (mondatory) + > name (mandatory) + > type_code (mandatory) > display_size > internal_size > precision @@ -55,8 +55,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow( - self._result, self._fields) + block, self._block_rows = taos_fetch_row(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -69,20 +68,17 @@ class TDengineCursor(object): @property def description(self): - """Return the description of the object. - """ + """Return the description of the object.""" return self._description @property def rowcount(self): - """Return the rowcount of the object - """ + """Return the rowcount of the object""" return self._rowcount @property def affected_rows(self): - """Return the rowcount of insertion - """ + """Return the rowcount of insertion""" return self._affected_rows def callproc(self, procname, *args): @@ -96,8 +92,7 @@ class TDengineCursor(object): self._logfile = logfile def close(self): - """Close the cursor. - """ + """Close the cursor.""" if self._connection is None: return False @@ -107,8 +102,7 @@ class TDengineCursor(object): return True def execute(self, operation, params=None): - """Prepare and execute a database operation (query or command). - """ + """Prepare and execute a database operation (query or command).""" if not operation: return None @@ -124,104 +118,91 @@ class TDengineCursor(object): # global querySeqNum # querySeqNum += 1 - # localSeqNum = querySeqNum # avoid raice condition + # localSeqNum = querySeqNum # avoid race condition # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) - self._result = CTaosInterface.query(self._connection._conn, stmt) + self._result = taos_query(self._connection._conn, stmt) # print(" << Query ({}) Exec Done".format(localSeqNum)) - if (self._logfile): + if self._logfile: with open(self._logfile, "a") as logfile: logfile.write("%s;\n" % operation) - errno = CTaosInterface.libtaos.taos_errno(self._result) - if errno == 0: - if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows( - self._result) - return CTaosInterface.affectedRows(self._result) - else: - self._fields = CTaosInterface.useResult( - self._result) - return self._handle_result() + if taos_field_count(self._result) == 0: + affected_rows = taos_affected_rows(self._result) + self._affected_rows += affected_rows + return affected_rows else: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + self._fields = taos_fetch_fields(self._result) + return self._handle_result() def executemany(self, operation, seq_of_parameters): - """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. - """ + """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.""" pass def fetchone(self): - """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. - """ + """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available.""" pass def fetchmany(self): pass def istype(self, col, dataType): - if (dataType.upper() == "BOOL"): - if (self._description[col][1] == FieldType.C_BOOL): + if dataType.upper() == "BOOL": + if self._description[col][1] == FieldType.C_BOOL: return True - if (dataType.upper() == "TINYINT"): - if (self._description[col][1] == FieldType.C_TINYINT): + if dataType.upper() == "TINYINT": + if self._description[col][1] == FieldType.C_TINYINT: return True - if (dataType.upper() == "TINYINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + if dataType.upper() == "TINYINT UNSIGNED": + if self._description[col][1] == FieldType.C_TINYINT_UNSIGNED: return True - if (dataType.upper() == "SMALLINT"): - if (self._description[col][1] == FieldType.C_SMALLINT): + if dataType.upper() == "SMALLINT": + if self._description[col][1] == FieldType.C_SMALLINT: return True - if (dataType.upper() == "SMALLINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + if dataType.upper() == "SMALLINT UNSIGNED": + if self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED: return True - if (dataType.upper() == "INT"): - if (self._description[col][1] == FieldType.C_INT): + if dataType.upper() == "INT": + if self._description[col][1] == FieldType.C_INT: return True - if (dataType.upper() == "INT UNSIGNED"): - if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + if dataType.upper() == "INT UNSIGNED": + if self._description[col][1] == FieldType.C_INT_UNSIGNED: return True - if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_BIGINT): + if dataType.upper() == "BIGINT": + if self._description[col][1] == FieldType.C_BIGINT: return True - if (dataType.upper() == "BIGINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): + if dataType.upper() == "BIGINT UNSIGNED": + if self._description[col][1] == FieldType.C_BIGINT_UNSIGNED: return True - if (dataType.upper() == "FLOAT"): - if (self._description[col][1] == FieldType.C_FLOAT): + if dataType.upper() == "FLOAT": + if self._description[col][1] == FieldType.C_FLOAT: return True - if (dataType.upper() == "DOUBLE"): - if (self._description[col][1] == FieldType.C_DOUBLE): + if dataType.upper() == "DOUBLE": + if self._description[col][1] == FieldType.C_DOUBLE: return True - if (dataType.upper() == "BINARY"): - if (self._description[col][1] == FieldType.C_BINARY): + if dataType.upper() == "BINARY": + if self._description[col][1] == FieldType.C_BINARY: return True - if (dataType.upper() == "TIMESTAMP"): - if (self._description[col][1] == FieldType.C_TIMESTAMP): + if dataType.upper() == "TIMESTAMP": + if self._description[col][1] == FieldType.C_TIMESTAMP: return True - if (dataType.upper() == "NCHAR"): - if (self._description[col][1] == FieldType.C_NCHAR): + if dataType.upper() == "NCHAR": + if self._description[col][1] == FieldType.C_NCHAR: return True return False def fetchall_row(self): - """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. - """ + """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.""" if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_row(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -230,19 +211,16 @@ class TDengineCursor(object): return list(map(tuple, zip(*buffer))) def fetchall(self): - if self._result is None or self._fields is None: + if self._result is None: raise OperationalError("Invalid use of fetchall") - - buffer = [[] for i in range(len(self._fields))] + fields = self._fields if self._fields is not None else taos_fetch_fields(self._result) + buffer = [[] for i in range(len(fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -250,9 +228,12 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def stop_query(self): + if self._result != None: + taos_stop_query(self._result) + def nextset(self): - """ - """ + """ """ pass def setinputsize(self, sizes): @@ -262,12 +243,11 @@ class TDengineCursor(object): pass def _reset_result(self): - """Reset the result to unused version. - """ + """Reset the result to unused version.""" self._description = [] self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + taos_free_result(self._result) self._result = None self._fields = None self._block = None @@ -276,11 +256,12 @@ class TDengineCursor(object): self._affected_rows = 0 def _handle_result(self): - """Handle the return result from query. - """ + """Handle the return result from query.""" self._description = [] for ele in self._fields: - self._description.append( - (ele['name'], ele['type'], None, None, None, None, False)) + self._description.append((ele["name"], ele["type"], None, None, None, None, False)) return self._result + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/dbapi.py b/src/connector/python/taos/dbapi.py deleted file mode 100644 index 594681ada953abf388e503c23199043cf686e1a3..0000000000000000000000000000000000000000 --- a/src/connector/python/taos/dbapi.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Type Objects and Constructors. -""" - -import time -import datetime - - -class DBAPITypeObject(object): - def __init__(self, *values): - self.values = values - - def __com__(self, other): - if other in self.values: - return 0 - if other < self.values: - return 1 - else: - return -1 - - -Date = datetime.date -Time = datetime.time -Timestamp = datetime.datetime - - -def DataFromTicks(ticks): - return Date(*time.localtime(ticks)[:3]) - - -def TimeFromTicks(ticks): - return Time(*time.localtime(ticks)[3:6]) - - -def TimestampFromTicks(ticks): - return Timestamp(*time.localtime(ticks)[:6]) - - -Binary = bytes - -# STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) -# BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) -# NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) -# DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() diff --git a/src/connector/python/taos/error.py b/src/connector/python/taos/error.py index c584badce8320cd35dc81e8f6b613c56163b1a29..a30adbb162f1c194bdfcf4cca5c43f01107a9776 100644 --- a/src/connector/python/taos/error.py +++ b/src/connector/python/taos/error.py @@ -1,66 +1,86 @@ +# encoding:UTF-8 """Python exceptions """ class Error(Exception): - def __init__(self, msg=None, errno=None): + def __init__(self, msg=None, errno=0xffff): self.msg = msg - self._full_msg = self.msg self.errno = errno + self._full_msg = "[0x%04x]: %s" % (self.errno & 0xffff, self.msg) def __str__(self): return self._full_msg class Warning(Exception): - """Exception raised for important warnings like data truncations while inserting. - """ + """Exception raised for important warnings like data truncations while inserting.""" + pass class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. - """ + """Exception raised for errors that are related to the database interface rather than the database itself.""" + pass class DatabaseError(Error): - """Exception raised for errors that are related to the database. - """ + """Exception raised for errors that are related to the database.""" + pass +class ConnectionError(Error): + """Exceptin raised for connection failed""" + pass class DataError(DatabaseError): - """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. - """ + """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range.""" + pass class OperationalError(DatabaseError): - """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer - """ + """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer""" + pass class IntegrityError(DatabaseError): - """Exception raised when the relational integrity of the database is affected. - """ + """Exception raised when the relational integrity of the database is affected.""" + pass class InternalError(DatabaseError): - """Exception raised when the database encounters an internal error. - """ + """Exception raised when the database encounters an internal error.""" + pass class ProgrammingError(DatabaseError): - """Exception raised for programming errors. - """ + """Exception raised for programming errors.""" + pass class NotSupportedError(DatabaseError): - """Exception raised in case a method or database API was used which is not supported by the database,. - """ + """Exception raised in case a method or database API was used which is not supported by the database,.""" + pass + + +class StatementError(DatabaseError): + """Exception raised in STMT API.""" + + pass + +class ResultError(DatabaseError): + """Result related APIs.""" + + pass + +class LinesError(DatabaseError): + """taos_insert_lines errors.""" + + pass \ No newline at end of file diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py new file mode 100644 index 0000000000000000000000000000000000000000..445cd8afdba6f2512c73be95c9b0dbd8dc00da8a --- /dev/null +++ b/src/connector/python/taos/field.py @@ -0,0 +1,302 @@ +# encoding:UTF-8 +import ctypes +import math +import datetime +from ctypes import * + +from .constants import FieldType +from .error import * + +_datetime_epoch = datetime.datetime.fromtimestamp(0) + +def _convert_millisecond_to_datetime(milli): + return _datetime_epoch + datetime.timedelta(seconds=milli / 1000.0) + + +def _convert_microsecond_to_datetime(micro): + return _datetime_epoch + datetime.timedelta(seconds=micro / 1000000.0) + + +def _convert_nanosecond_to_datetime(nanosec): + return nanosec + + +def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + _timestamp_converter = _convert_millisecond_to_datetime + if precision == FieldType.C_TIMESTAMP_MILLI: + _timestamp_converter = _convert_millisecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_MICRO: + _timestamp_converter = _convert_microsecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_NANO: + _timestamp_converter = _convert_nanosecond_to_datetime + else: + raise DatabaseError("Unknown precision returned from database") + + return [ + None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ubyte))[: abs(num_of_rows)] + ] + + +def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[: abs(num_of_rows)] + ] + + +def _crow_smallint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ushort))[: abs(num_of_rows)] + ] + + +def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[: abs(num_of_rows)] + ] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[: abs(num_of_rows)] + ] + + +def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bigint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint64))[: abs(num_of_rows)] + ] + + +def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C float row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[: abs(num_of_rows)] + ] + + +def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C double row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[: abs(num_of_rows)] + ] + + +def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + return [ + None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode("utf-8") + for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[: abs(num_of_rows)] + ] + + +def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + if num_of_rows >= 0: + tmpstr = ctypes.c_char_p(data) + res.append(tmpstr.value.decode()) + else: + res.append( + ( + ctypes.cast( + data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4)), + ) + )[0].value + ) + except ValueError: + res.append(None) + + return res + + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast(data + nbytes * i, ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + return res + + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) + except ValueError: + res.append(None) + return res + + +CONVERT_FUNC = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +# Corresponding TAOS_FIELD structure in C + + +class TaosField(ctypes.Structure): + _fields_ = [ + ("_name", ctypes.c_char * 65), + ("_type", ctypes.c_uint8), + ("_bytes", ctypes.c_uint16), + ] + + @property + def name(self): + return self._name.decode("utf-8") + + @property + def length(self): + """alias to self.bytes""" + return self._bytes + + @property + def bytes(self): + return self._bytes + + @property + def type(self): + return self._type + + def __dict__(self): + return {"name": self.name, "type": self.type, "bytes": self.length} + + def __str__(self): + return "{name: %s, type: %d, bytes: %d}" % (self.name, self.type, self.length) + + def __getitem__(self, item): + return getattr(self, item) + + +class TaosFields(object): + def __init__(self, fields, count): + if isinstance(fields, c_void_p): + self._fields = cast(fields, POINTER(TaosField)) + if isinstance(fields, POINTER(TaosField)): + self._fields = fields + self._count = count + self._iter = 0 + + def as_ptr(self): + return self._fields + + @property + def count(self): + return self._count + + @property + def fields(self): + return self._fields + + def __next__(self): + return self._next_field() + + def next(self): + return self._next_field() + + def _next_field(self): + if self._iter < self.count: + field = self._fields[self._iter] + self._iter += 1 + return field + else: + raise StopIteration + + def __getitem__(self, item): + return self._fields[item] + + def __iter__(self): + return self + + def __len__(self): + return self.count diff --git a/src/connector/python/taos/precision.py b/src/connector/python/taos/precision.py new file mode 100644 index 0000000000000000000000000000000000000000..d67da592cce6d2121ec8f2eed78a30d6fa0c446b --- /dev/null +++ b/src/connector/python/taos/precision.py @@ -0,0 +1,12 @@ +class PrecisionEnum(object): + """Precision enums""" + + Milliseconds = 0 + Microseconds = 1 + Nanoseconds = 2 + + +class PrecisionError(Exception): + """Python datetime does not support nanoseconds error""" + + pass diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py new file mode 100644 index 0000000000000000000000000000000000000000..81151733615d1b7fdc3318b6e53888ae39d32b14 --- /dev/null +++ b/src/connector/python/taos/result.py @@ -0,0 +1,245 @@ +from .cinterface import * + +# from .connection import TaosConnection +from .error import * + + +class TaosResult(object): + """TDengine result interface""" + + def __init__(self, result, close_after=False, conn=None): + # type: (c_void_p, bool, TaosConnection) -> TaosResult + # to make the __del__ order right + self._conn = conn + self._close_after = close_after + self._result = result + self._fields = None + self._field_count = None + self._precision = None + + self._block = None + self._block_length = None + self._row_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + # fetch next row + return self._next_row() + + def _next_row(self): + if self._result is None or self.fields is None: + raise OperationalError("Invalid use of fetch iterator") + + if self._block == None or self._block_iter >= self._block_length: + self._block, self._block_length = self.fetch_block() + self._block_iter = 0 + # self._row_count += self._block_length + + raw = self._block[self._block_iter] + self._block_iter += 1 + return raw + + @property + def fields(self): + """fields definitions of the current result""" + if self._result is None: + raise ResultError("no result object setted") + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + + return self._fields + + @property + def field_count(self): + """Field count of the current result, eq to taos_field_count(result)""" + return self.fields.count + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._row_count + + @property + def precision(self): + if self._precision == None: + self._precision = taos_result_precision(self._result) + return self._precision + + @property + def affected_rows(self): + return taos_affected_rows(self._result) + + # @property + def field_lengths(self): + return taos_fetch_lengths(self._result, self.field_count) + + def rows_iter(self, num_of_rows=None): + return TaosRows(self, num_of_rows) + + def blocks_iter(self): + return TaosBlocks(self) + + def fetch_block(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + + block, length = taos_fetch_block_raw(self._result) + if length == 0: + raise StopIteration + precision = self.precision + field_count = self.field_count + fields = self.fields + blocks = [None] * field_count + lengths = self.field_lengths() + for i in range(field_count): + data = ctypes.cast(block, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i].type](data, length, lengths[i], precision) + + return list(map(tuple, zip(*blocks))), length + + def fetch_all(self): + if self._result is None: + raise OperationalError("Invalid use of fetchall") + + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + buffer = [[] for i in range(len(self._fields))] + self._row_count = 0 + while True: + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) + if errno != 0: + raise ProgrammingError(taos_errstr(self._result), errno) + if num_of_fields == 0: + break + self._row_count += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetch_rows_a(self, callback, param): + taos_fetch_rows_a(self._result, callback, param) + + def stop_query(self): + return taos_stop_query(self._result) + + def errno(self): + """**DO NOT** use this directly unless you know what you are doing""" + return taos_errno(self._result) + + def errstr(self): + return taos_errstr(self._result) + + def check_error(self, errno=None, close=True): + if errno == None: + errno = self.errno() + if errno != 0: + msg = self.errstr() + self.close() + raise OperationalError(msg, errno) + + def close(self): + """free result object.""" + if self._result != None and self._close_after: + taos_free_result(self._result) + self._result = None + self._fields = None + self._field_count = None + self._field_lengths = None + + def __del__(self): + self.close() + + +class TaosRows: + """TDengine result rows iterator""" + + def __init__(self, result, num_of_rows=None): + self._result = result + self._num_of_rows = num_of_rows + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + return self._next_row() + + def _next_row(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + if self._num_of_rows != None and self._num_of_rows <= self._result._row_count: + raise StopIteration + + row = taos_fetch_row_raw(self._result._result) + if not row: + raise StopIteration + self._result._row_count += 1 + return TaosRow(self._result, row) + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._result._row_count + + +class TaosRow: + def __init__(self, result, row): + self._result = result + self._row = row + + def __str__(self): + return taos_print_row(self._row, self._result.fields, self._result.field_count) + + def __call__(self): + return self.as_tuple() + + def _astuple(self): + return self.as_tuple() + + def __iter__(self): + return self.as_tuple() + + def as_ptr(self): + return self._row + + def as_tuple(self): + precision = self._result.precision + field_count = self._result.field_count + blocks = [None] * field_count + fields = self._result.fields + field_lens = self._result.field_lengths() + for i in range(field_count): + data = ctypes.cast(self._row, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = None + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, 1, field_lens[i], precision)[0] + return tuple(blocks) + + +class TaosBlocks: + """TDengine result blocks iterator""" + + def __init__(self, result): + self._result = result + + def __iter__(self): + return self + + def __next__(self): + return self._result.fetch_block() + + def next(self): + return self._result.fetch_block() diff --git a/src/connector/python/taos/statement.py b/src/connector/python/taos/statement.py new file mode 100644 index 0000000000000000000000000000000000000000..155e98173b7f920640aa84d0fcda618d2669bb1e --- /dev/null +++ b/src/connector/python/taos/statement.py @@ -0,0 +1,85 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStmt(object): + """TDengine STMT interface""" + + def __init__(self, stmt, conn = None): + self._conn = conn + self._stmt = stmt + + def set_tbname(self, name): + """Set table name if needed. + + Note that the set_tbname* method should only used in insert statement + """ + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname(self._stmt, name) + + def prepare(self, sql): + # type: (str) -> None + taos_stmt_prepare(self._stmt, sql) + + def set_tbname_tags(self, name, tags): + # type: (str, Array[TaosBind]) -> None + """Set table name with tags, tags is array of BindParams""" + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname_tags(self._stmt, name, tags) + + def bind_param(self, params, add_batch=True): + # type: (Array[TaosBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param(self._stmt, params) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def bind_param_batch(self, binds, add_batch=True): + # type: (Array[TaosMultiBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param_batch(self._stmt, binds) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def add_batch(self): + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_add_batch(self._stmt) + + def execute(self): + if self._stmt is None: + raise StatementError("Invalid use of execute") + taos_stmt_execute(self._stmt) + + def use_result(self): + result = taos_stmt_use_result(self._stmt) + return TaosResult(result) + + def close(self): + """Close stmt.""" + if self._stmt is None: + return + taos_stmt_close(self._stmt) + self._stmt = None + + def __del__(self): + self.close() + + +if __name__ == "__main__": + from taos.connection import TaosConnection + + conn = TaosConnection() + + stmt = conn.statement("select * from log.log limit 10") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + stmt.close() + conn.close() diff --git a/src/connector/python/taos/stream.py b/src/connector/python/taos/stream.py new file mode 100644 index 0000000000000000000000000000000000000000..fe3c8c85e3279511972293882224bf20c30dfa64 --- /dev/null +++ b/src/connector/python/taos/stream.py @@ -0,0 +1,22 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStream(object): + """TDengine Stream interface""" + + def __init__(self, stream): + self._raw = stream + + def as_ptr(self): + return self._raw + + def close(self): + """Close stmt.""" + if self._raw is not None: + taos_close_stream(self._raw) + self._raw = None + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/subscription.py b/src/connector/python/taos/subscription.py index 270d9de09217fc58a389981a3542698dd1c0428a..3c6958b6f8d55791b9753a84a4bbd7653bdae780 100644 --- a/src/connector/python/taos/subscription.py +++ b/src/connector/python/taos/subscription.py @@ -1,49 +1,41 @@ -from .cinterface import CTaosInterface +from taos.result import TaosResult +from .cinterface import * from .error import * -class TDengineSubscription(object): - """TDengine subscription object - """ +class TaosSubscription(object): + """TDengine subscription object""" - def __init__(self, sub): + def __init__(self, sub, with_callback = False): self._sub = sub + self._with_callback = with_callback def consume(self): - """Consume rows of a subscription - """ + """Consume rows of a subscription""" if self._sub is None: raise OperationalError("Invalid use of consume") - - result, fields = CTaosInterface.consume(self._sub) - buffer = [[] for i in range(len(fields))] - while True: - block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: - break - for i in range(len(fields)): - buffer[i].extend(block[i]) - - self.fields = fields - return list(map(tuple, zip(*buffer))) + if self._with_callback: + raise OperationalError("DONOT use consume method in an subscription with callback") + result = taos_consume(self._sub) + return TaosResult(result) def close(self, keepProgress=True): - """Close the Subscription. - """ + """Close the Subscription.""" if self._sub is None: return False - CTaosInterface.unsubscribe(self._sub, keepProgress) + taos_unsubscribe(self._sub, keepProgress) + self._sub = None return True + + def __del__(self): + self.close() + +if __name__ == "__main__": + from .connection import TaosConnection -if __name__ == '__main__': - from .connection import TDengineConnection - conn = TDengineConnection( - host="127.0.0.1", - user="root", - password="taosdata", - database="test") + conn = TaosConnection(host="127.0.0.1", user="root", password="taosdata", database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) diff --git a/src/connector/python/taos/timestamp.py b/src/connector/python/taos/timestamp.py new file mode 100644 index 0000000000000000000000000000000000000000..ab5679fdf12e2942aa94f76716ff98e6d2a88d69 --- /dev/null +++ b/src/connector/python/taos/timestamp.py @@ -0,0 +1,17 @@ + +class TimestampType(object): + """Choose which type that parsing TDengine timestamp data to + + - DATETIME: use python datetime.datetime, note that it does not support nanosecond precision, + and python taos will use raw c_int64 as a fallback for nanosecond results. + - NUMPY: use numpy.datetime64 type. + - RAW: use raw c_int64. + - TAOS: use taos' TaosTimestamp. + """ + DATETIME = 0, + NUMPY = 1, + RAW = 2, + TAOS = 3, + +class TaosTimestamp: + pass diff --git a/src/connector/python/tests/test_ctaos.py b/src/connector/python/tests/test_ctaos.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9566931f2b29dcbdc8646d2f087ebf40e716cc --- /dev/null +++ b/src/connector/python/tests/test_ctaos.py @@ -0,0 +1,162 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * + +import time +import datetime +import pytest + +@pytest.fixture +def conn(): + return CTaosInterface().connect() + + +def test_simple(conn, caplog): + dbname = "pytest_ctaos_simple" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, level tinyint, content binary(100), ipaddr binary(134))", + ) + taos_free_result(res) + + res = taos_query(conn, "insert into log values(now, 1, 'hello', 'test')") + taos_free_result(res) + + res = taos_query(conn, "select level,content,ipaddr from log limit 1") + + fields = taos_fetch_fields_raw(res) + field_count = taos_field_count(res) + + fields = taos_fetch_fields(res) + for field in fields: + print(field) + + # field_lengths = taos_fetch_lengths(res, field_count) + # if not field_lengths: + # raise "fetch lengths error" + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, field_count) + assert rowstr == "1 hello test" + + row, num = taos_fetch_row(res, fields) + print(row) + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + + +def test_stmt(conn, caplog): + dbname = "pytest_ctaos_stmt" + try: + res = taos_query(conn, "drop database if exists %s" % dbname) + taos_free_result(res) + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100))", + ) + taos_free_result(res) + + stmt = taos_stmt_init(conn) + + taos_stmt_prepare(stmt, "insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + + params = new_bind_params(14) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].null() + params[2].tinyint(2) + params[3].smallint(3) + params[4].int(4) + params[5].bigint(5) + params[6].tinyint_unsigned(6) + params[7].smallint_unsigned(7) + params[8].int_unsigned(8) + params[9].bigint_unsigned(9) + params[10].float(10.1) + params[11].double(10.11) + params[12].binary("hello") + params[13].nchar("stmt") + taos_stmt_bind_param(stmt, params) + taos_stmt_add_batch(stmt) + taos_stmt_execute(stmt) + + res = taos_query(conn, "select * from log limit 1") + + fields = taos_fetch_fields(res) + filed_count = taos_field_count(res) + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, filed_count, 100) + + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + + assert rowstr == "1626861392589 NULL 2 3 4 5 6 7 8 9 10.100000 10.110000 hello stmt" + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + +def stream_callback(param, result, row): + # type: (c_void_p, c_void_p, c_void_p) -> None + try: + if result == None or row == None: + return + result = c_void_p(result) + row = c_void_p(row) + fields = taos_fetch_fields_raw(result) + num_fields = taos_field_count(result) + s = taos_print_row(row, fields, num_fields) + print(s) + taos_stop_query(result) + except Exception as err: + print(err) + +def test_stream(conn, caplog): + dbname = "pytest_ctaos_stream" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, n int)", + ) + taos_free_result(res) + + res = taos_query(conn, "select count(*) from log interval(5s)") + cc = taos_num_fields(res) + assert cc == 2 + + stream = taos_open_stream(conn, "select count(*) from log interval(5s)", stream_callback, 0, None, None) + print("waiting for data") + time.sleep(1) + + for i in range(0, 2): + res = taos_query(conn, "insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + taos_free_result(res) + time.sleep(2) + taos_close_stream(stream) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err diff --git a/src/connector/python/tests/test_info.py b/src/connector/python/tests/test_info.py new file mode 100644 index 0000000000000000000000000000000000000000..bddfec7ef9ddbc203adfcadd262839048466592c --- /dev/null +++ b/src/connector/python/tests/test_info.py @@ -0,0 +1,23 @@ +from taos.cinterface import * + +from taos import * + +import pytest + +@pytest.fixture +def conn(): + return connect() + +def test_client_info(): + print(taos_get_client_info()) + None + +def test_server_info(conn): + # type: (TaosConnection) -> None + print(conn.client_info) + print(conn.server_info) + None + +if __name__ == "__main__": + test_client_info() + test_server_info(connect()) diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9d2cdb39d6f4f2612581ce7284c057c456ef91 --- /dev/null +++ b/src/connector/python/tests/test_lines.py @@ -0,0 +1,57 @@ +from taos.error import OperationalError +from taos import connect, new_bind_params, PrecisionEnum +from taos import * + +from ctypes import * +import taos +import pytest + + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + + +def test_insert_lines(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_insert_lines" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + + lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + result = conn.query("select * from st") + print(*result.fields) + all = result.rows_iter() + for row in all: + print(row) + result.close() + print(result.row_count) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err + + +if __name__ == "__main__": + test_insert_lines(connect()) diff --git a/src/connector/python/tests/test_query.py b/src/connector/python/tests/test_query.py new file mode 100644 index 0000000000000000000000000000000000000000..f4e139b1f14df29e8b6304dd2ca03519ea274f43 --- /dev/null +++ b/src/connector/python/tests/test_query.py @@ -0,0 +1,43 @@ +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + return taos.connect() + +def test_query(conn): + """This test will use fetch_block for rows fetching, significantly faster than rows_iter""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result: + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +def test_query_row_iter(conn): + """This test will use fetch_row for each row fetching, this is the only way in async callback""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result.rows_iter(): + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +if __name__ == "__main__": + test_query(taos.connect(database = "log")) + test_query_row_iter(taos.connect(database = "log")) diff --git a/src/connector/python/tests/test_query_a.py b/src/connector/python/tests/test_query_a.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4be5695a87f1fd1017435b13983df7c4f70f06 --- /dev/null +++ b/src/connector/python/tests/test_query_a.py @@ -0,0 +1,66 @@ +from taos import * +from ctypes import * +import taos +import pytest +import time + + +@pytest.fixture +def conn(): + return taos.connect() + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(taos.connect()) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py new file mode 100644 index 0000000000000000000000000000000000000000..938ba10eb3d2377a63f7972deb99dbd47f7de1b2 --- /dev/null +++ b/src/connector/python/tests/test_stmt.py @@ -0,0 +1,149 @@ +from taos import * + +from ctypes import * +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + +def test_stmt_insert(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + + stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + params = new_bind_params(16) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].bool(True) + params[2].null() + params[3].tinyint(2) + params[4].smallint(3) + params[5].int(4) + params[6].bigint(5) + params[7].tinyint_unsigned(6) + params[8].smallint_unsigned(7) + params[9].int_unsigned(8) + params[10].bigint_unsigned(9) + params[11].float(10.1) + params[12].double(10.11) + params[13].binary("hello") + params[14].nchar("stmt") + params[15].timestamp(1626861392589, PrecisionEnum.Milliseconds) + + stmt.bind_param(params) + stmt.execute() + + result = stmt.use_result() + assert result.affected_rows == 1 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + row = result.next() + print(row) + assert row[2] == None + for i in range(3, 11): + assert row[i] == i - 1 + #float == may not work as expected + # assert row[10] == c_float(10.1) + assert row[12] == 10.11 + assert row[13] == "hello" + assert row[14] == "stmt" + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + +def test_stmt_insert_multi(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt_multi" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + start = datetime.now() + stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + + params = new_multi_binds(16) + params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) + params[1].bool((True, None, False)) + params[2].tinyint([-128, -128, None]) # -128 is tinyint null + params[3].tinyint([0, 127, None]) + params[4].smallint([3, None, 2]) + params[5].int([3, 4, None]) + params[6].bigint([3, 4, None]) + params[7].tinyint_unsigned([3, 4, None]) + params[8].smallint_unsigned([3, 4, None]) + params[9].int_unsigned([3, 4, None]) + params[10].bigint_unsigned([3, 4, None]) + params[11].float([3, None, 1]) + params[12].double([3, None, 1.2]) + params[13].binary(["abc", "dddafadfadfadfadfa", None]) + params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) + params[15].timestamp([None, None, 1626861392591]) + stmt.bind_param_batch(params) + + stmt.execute() + end = datetime.now() + print("elapsed time: ", end - start) + result = stmt.use_result() + assert result.affected_rows == 3 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + result.close() + + stmt.close() + + # start = datetime.now() + # conn.query("insert into log values(1626861392660, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392661, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392662, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)") + + # end = datetime.now() + # print("elapsed time: ", end - start) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err +if __name__ == "__main__": + test_stmt_insert(connect()) + test_stmt_insert_multi(connect()) \ No newline at end of file diff --git a/src/connector/python/tests/test_stream.py b/src/connector/python/tests/test_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..de6e20928b176e51bc6d350fb01268459f4e7f95 --- /dev/null +++ b/src/connector/python/tests/test_stream.py @@ -0,0 +1,70 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * +from taos import * +from ctypes import * +import time +import pytest + + +@pytest.fixture +def conn(): + return connect() + + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.execute("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) diff --git a/src/connector/python/tests/test_subscribe.py b/src/connector/python/tests/test_subscribe.py new file mode 100644 index 0000000000000000000000000000000000000000..99fe5b263625c63200f416ec98fcb561773becd8 --- /dev/null +++ b/src/connector/python/tests/test_subscribe.py @@ -0,0 +1,100 @@ +from taos.subscription import TaosSubscription +from taos import * +from ctypes import * +import taos +import pytest +import time +from random import random + + +@pytest.fixture +def conn(): + return taos.connect() + + +def test_subscribe(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + + sub = conn.subscribe(True, "test", "select * from log", 1000) + print("# consume from begin") + for ts, n in sub.consume(): + print(ts, n) + + print("# consume new data") + for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + + print("# consume with a stop condition") + for i in range(10): + conn.execute("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe(taos.connect()) + test_subscribe_callback(taos.connect()) diff --git a/tests/pytest/insert/line_insert.py b/tests/pytest/insert/line_insert.py index 53eaa55aa50a1369b4aff9c49421263788205038..92fdd0f28e612994df414ea1b560152a3f2001a8 100644 --- a/tests/pytest/insert/line_insert.py +++ b/tests/pytest/insert/line_insert.py @@ -42,18 +42,18 @@ class TDTestCase: "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines(lines) + print("insert_lines result {}".format(code)) lines2 = [ "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns" ] - code = self._conn.insertLines([ lines2[0] ]) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines([ lines2[0] ]) + print("insert_lines result {}".format(code)) - self._conn.insertLines([ lines2[1] ]) - print("insertLines result {}".format(code)) + self._conn.insert_lines([ lines2[1] ]) + print("insert_lines result {}".format(code)) tdSql.query("select * from st") tdSql.checkRows(4) @@ -73,7 +73,7 @@ class TDTestCase: tdSql.query("describe stf") tdSql.checkData(2, 2, 14) - self._conn.insertLines([ + self._conn.insert_lines([ "sth,t1=4i64,t2=5f64,t4=5f64,ID=\"childtable\" c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641ms", "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654ms" ]) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 88abea477a9f1b458dc17c93c27d68934f5af03f..49c32235883483c9d709b6d7a41dc9419dd95461 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -11,8 +11,10 @@ # -*- coding: utf-8 -*- +import traceback import random import string +from taos.error import LinesError import time from copy import deepcopy import numpy as np @@ -292,7 +294,7 @@ class TDTestCase: def resCmp(self, input_sql, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None): expect_list = self.inputHandle(input_sql) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) query_sql = f"{query_sql} {stb_name} {condition}" res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) if ts == 0: @@ -312,7 +314,9 @@ class TDTestCase: expect_list[0].pop(j) tdSql.checkEqual(res_row_list[0], expect_list[0]) tdSql.checkEqual(res_field_list_without_ts, expect_list[1]) - tdSql.checkEqual(res_type_list, expect_list[2]) + for i in range(len(res_type_list)): + tdSql.checkEqual(res_type_list[i], expect_list[2][i]) + # tdSql.checkEqual(res_type_list, expect_list[2]) def cleanStb(self): query_sql = "show stables" @@ -405,13 +409,14 @@ class TDTestCase: """ for input_sql in [self.genLongSql(128, 1)[0], self.genLongSql(1, 4094)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) - + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass + def idIllegalNameCheckCase(self): """ test illegal id name @@ -421,8 +426,10 @@ class TDTestCase: rstr = list("`~!@#$¥%^&*()-+={}|[]、「」【】\:;《》<>?") for i in rstr: input_sql = self.genFullTypeSql(tb_name=f"\"aaa{i}bbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def idStartWithNumCheckCase(self): """ @@ -430,8 +437,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(tb_name=f"\"1aaabbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def nowTsCheckCase(self): """ @@ -439,8 +448,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="now")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def dateFormatTsCheckCase(self): """ @@ -448,8 +459,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="2021-07-21\ 19:01:46.920")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def illegalTsCheckCase(self): """ @@ -457,8 +470,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="16260068336390us19")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def tagValueLengthCheckCase(self): """ @@ -471,8 +486,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(t1=t1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i16 for t2 in ["-32767i16", "32767i16"]: @@ -480,8 +497,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(t2=t2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i32 for t3 in ["-2147483647i32", "2147483647i32"]: @@ -489,8 +508,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(t3=t3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i64 for t4 in ["-9223372036854775807i64", "9223372036854775807i64"]: @@ -498,8 +519,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(t4=t4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass # f32 for t5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -508,8 +531,12 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for t5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(t5=t5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) + # f64 for t6 in [f'{-1.79769*(10**308)}f64', f'{-1.79769*(10**308)}f64']: @@ -518,27 +545,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16375, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + pass # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4094, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def colValueLengthCheckCase(self): """ @@ -552,16 +588,22 @@ class TDTestCase: for c1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(c1=c1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i16 for c2 in ["-32767i16"]: input_sql, stb_name = self.genFullTypeSql(c2=c2) self.resCmp(input_sql, stb_name) for c2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(c2=c2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i32 for c3 in ["-2147483647i32"]: @@ -569,8 +611,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(c3=c3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i64 for c4 in ["-9223372036854775807i64"]: @@ -578,8 +623,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(c4=c4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f32 for c5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -588,8 +636,11 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for c5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(c5=c5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f64 for c6 in [f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64', f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64']: @@ -598,27 +649,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16375, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4094, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def tagColIllegalValueCheckCase(self): @@ -629,11 +689,17 @@ class TDTestCase: # bool for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: input_sql1 = self.genFullTypeSql(t0=i)[0] - code = self._conn.insertLines([input_sql1]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql1]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql2 = self.genFullTypeSql(c0=i)[0] - code = self._conn.insertLines([input_sql2]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql2]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i8 i16 i32 i64 f32 f64 for input_sql in [ @@ -651,8 +717,11 @@ class TDTestCase: self.genFullTypeSql(c6="11.1s45f64")[0], self.genFullTypeSql(c9="1s1u64")[0] ]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check binary and nchar blank stb_name = self.getLongName(7, "letters") @@ -661,18 +730,19 @@ class TDTestCase: input_sql3 = f'{stb_name},t0=t,t1="abc aaa" c0=f 1626006833639000000ns' input_sql4 = f'{stb_name},t0=t,t1=L"abc aaa" c0=f 1626006833639000000ns' for input_sql in [input_sql1, input_sql2, input_sql3, input_sql4]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check accepted binary and nchar symbols # # * ~!@#$¥%^&*()-+={}|[]、「」:; for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): input_sql1 = f'{stb_name},t0=t c0=f,c1="abc{symbol}aaa" 1626006833639000000ns' input_sql2 = f'{stb_name},t0=t,t1="abc{symbol}aaa" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql1]) - tdSql.checkEqual(code, 0) - code = self._conn.insertLines([input_sql2]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) def duplicateIdTagColInsertCheckCase(self): @@ -681,23 +751,35 @@ class TDTestCase: """ self.cleanStb() input_sql_id = self.genFullTypeSql(id_double_tag=True)[0] - code = self._conn.insertLines([input_sql_id]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_id]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_tag = input_sql.replace("t5", "t6") - code = self._conn.insertLines([input_sql_tag]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_tag]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "c6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "C6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) ##### stb exist ##### def noIdStbExistCheckCase(self): @@ -720,8 +802,7 @@ class TDTestCase: self.cleanStb() input_sql, stb_name = self.genFullTypeSql() self.resCmp(input_sql, stb_name) - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) self.resCmp(input_sql, stb_name) def tagColBinaryNcharLengthCheckCase(self): @@ -788,7 +869,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(tb_name1, tb_name2) input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True, ct_add_tag=True) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) tb_name3 = self.getNoIdTbName(stb_name) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) @@ -803,29 +884,35 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(5, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(6, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError: + pass tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) # # * check col,col+ts max in describe ---> 16143 input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(12, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(13, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -838,28 +925,32 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + code = self._conn.insert_lines([input_sql]) # * legal nchar could not be larger than 16374/4 input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(1, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(2, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(4, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(5, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -880,8 +971,7 @@ class TDTestCase: "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns", "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(lines) def multiInsertCheckCase(self, count): """ @@ -894,8 +984,7 @@ class TDTestCase: for i in range(count): input_sql = self.genFullTypeSql(stb_name=stb_name, t7=f'"{self.getLongName(8, "letters")}"', c7=f'"{self.getLongName(8, "letters")}"', id_noexist_tag=True)[0] sql_list.append(input_sql) - code = self._conn.insertLines(sql_list) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(sql_list) def batchErrorInsertCheckCase(self): """ @@ -905,8 +994,11 @@ class TDTestCase: stb_name = self.getLongName(8, "letters") lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", f"{stb_name},t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"] - code = self._conn.insertLines(lines) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines(lines) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def genSqlList(self, count=5, stb_name="", tb_name=""): """ @@ -957,7 +1049,7 @@ class TDTestCase: def genMultiThreadSeq(self, sql_list): tlist = list() for insert_sql in sql_list: - t = threading.Thread(target=self._conn.insertLines,args=([insert_sql[0]],)) + t = threading.Thread(target=self._conn.insert_lines,args=([insert_sql[0]],)) tlist.append(t) return tlist @@ -1155,16 +1247,18 @@ class TDTestCase: def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" input_sql2 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64 c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64 1626006933640000000ns" - code = self._conn.insertLines([input_sql1]) - code = self._conn.insertLines([input_sql2]) - print(code) - # self._conn.insertLines([input_sql2]) + try: + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) + except LinesError as err: + print(err.errno) + # self._conn.insert_lines([input_sql2]) # input_sql3 = f'abcd,id="cc¥Ec",t0=True,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="ndsfdrum",t8=L"ncharTagValue" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="igwoehkm",c8=L"ncharColValue",c9=7u64 0' # print(input_sql3) # input_sql4 = 'hmemeb,id="kilrcrldgf",t0=F,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="fysodjql",t8=L"ncharTagValue" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="waszbfvc",c8=L"ncharColValue",c9=7u64 0' - # code = self._conn.insertLines([input_sql3]) + # code = self._conn.insert_lines([input_sql3]) # print(code) - # self._conn.insertLines([input_sql4]) + # self._conn.insert_lines([input_sql4]) def runAll(self): self.initCheckCase() @@ -1222,7 +1316,11 @@ class TDTestCase: def run(self): print("running {}".format(__file__)) self.createDb() - self.runAll() + try: + self.runAll() + except Exception as err: + print(''.join(traceback.format_exception(None, err, err.__traceback__))) + raise err # self.tagColIllegalValueCheckCase() # self.test() diff --git a/tests/pytest/util/sub.py b/tests/pytest/util/sub.py index 2e3c2a96b7312c176c25bd35e109e146e5c4593f..664d830b86290e81e8ac1726f92193380d8c7715 100644 --- a/tests/pytest/util/sub.py +++ b/tests/pytest/util/sub.py @@ -29,9 +29,10 @@ class TDSub: self.sub.close(keepProgress) def consume(self): - self.data = self.sub.consume() - self.consumedRows = len(self.data) - self.consumedCols = len(self.sub.fields) + self.result = self.sub.consume() + self.result.fetch_all() + self.consumedRows = self.result.row_count + self.consumedCols = self.result.field_count return self.consumedRows def checkRows(self, expectRows):