diff --git a/.github/workflows/client-python.yml b/.github/workflows/client-python.yml index 2a96963edd99629050d9686813a552cbd4dd84ac..19a12e6432b582ad7ff86b251511e2f265c3dfac 100644 --- a/.github/workflows/client-python.yml +++ b/.github/workflows/client-python.yml @@ -48,6 +48,9 @@ jobs: docker images - name: Install IoTDB python client requirements run: pip3 install -r iotdb-client/client-py/requirements_dev.txt + - name: Check code style + shell: bash + run: black iotdb-client/client-py/ --check --diff - name: Integration test shell: bash run: | diff --git a/.gitignore b/.gitignore index 7eff5899cd5541abe174b7a34b07d6a9cc0773cf..812db1ac019626c0bd0b40b2395739c69b6afafe 100644 --- a/.gitignore +++ b/.gitignore @@ -119,14 +119,9 @@ classes/ Makefile **/CMakeFiles/ -### cluster test data -node1/ -node2/ -node3/ - # Exclude copied license -/client-py/LICENSE -/mlnode/LICENSE +iotdb-client/client-py/LICENSE +iotdb-core/mlnode/LICENSE # ANTLR iotdb-core/antlr/gen/ diff --git a/iotdb-client/client-py/.gitignore b/iotdb-client/client-py/.gitignore index 0778bd56f17c9a7026e30193fa11a9204881a63f..c2e6bdffad39ee8a046475c3c70666e143438f23 100644 --- a/iotdb-client/client-py/.gitignore +++ b/iotdb-client/client-py/.gitignore @@ -3,3 +3,5 @@ /build/ /dist/ /apache_iotdb.egg-info +/setup.py +!/resources/setup/py diff --git a/iotdb-client/client-py/SessionPoolExample.py b/iotdb-client/client-py/SessionPoolExample.py index 29bf8b160d615b442b6828f58d272d335b654012..9230634ba6f90eceac9c52ddcada59c49236fdc0 100644 --- a/iotdb-client/client-py/SessionPoolExample.py +++ b/iotdb-client/client-py/SessionPoolExample.py @@ -45,13 +45,13 @@ def prepare_data(): "root.test.d1.s0", "root.test.d1.s1", "root.test.d1.s2", - "root.test.d1.s3" + "root.test.d1.s3", ] data_type_lst = [ TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64, - TSDataType.FLOAT + TSDataType.FLOAT, ] encoding_lst = [TSEncoding.PLAIN for _ in range(len(data_type_lst))] compressor_lst = [Compressor.SNAPPY for _ in range(len(data_type_lst))] @@ -76,11 +76,7 @@ def insert_data(num: int): device = ["root.test.d" + str(num)] measurements = ["s0", "s1", "s2"] values = [False, 10, 11] - data_types = [ - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64 - ] + data_types = [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64] # Get a session from the pool session = session_pool.get_session() session.insert_records(device, [1], [measurements], [data_types], [values]) @@ -119,9 +115,14 @@ ip = "127.0.0.1" port = "6667" username = "root" password = "root" -pool_config = PoolConfig(node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], user_name=username, - password=password, fetch_size=1024, - time_zone="UTC+8", max_retry=3) +pool_config = PoolConfig( + node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], + user_name=username, + password=password, + fetch_size=1024, + time_zone="UTC+8", + max_retry=3, +) max_pool_size = 5 wait_timeout_in_ms = 3000 diff --git a/iotdb-client/client-py/iotdb/IoTDBContainer.py b/iotdb-client/client-py/iotdb/IoTDBContainer.py index 2f52f17a9306628609d4be1372a60e8be52c1a3e..51ac52c05ed147d04261c9b4a7a40fa02296c813 100644 --- a/iotdb-client/client-py/iotdb/IoTDBContainer.py +++ b/iotdb-client/client-py/iotdb/IoTDBContainer.py @@ -42,11 +42,12 @@ class IoTDBContainer(DockerContainer): self.get_container_host_ip(), self.get_exposed_port(6667), "root", "root" ) session.open(False) - with session.execute_statement( - "SHOW CLUSTER" - ) as session_data_set: + with session.execute_statement("SHOW CLUSTER") as session_data_set: while session_data_set.has_next(): - if session_data_set.next().get_fields()[2].get_string_value() != "Running": + if ( + session_data_set.next().get_fields()[2].get_string_value() + != "Running" + ): raise ContainerStartException("IoTDB is not started") session.close() diff --git a/iotdb-client/client-py/iotdb/SessionPool.py b/iotdb-client/client-py/iotdb/SessionPool.py index 4a7bdc3ae39cb68fffe412558bc48fb34b2ee996..6f1d7580796915eff0083ce32ede625c1495e54b 100644 --- a/iotdb-client/client-py/iotdb/SessionPool.py +++ b/iotdb-client/client-py/iotdb/SessionPool.py @@ -31,15 +31,25 @@ logger = logging.getLogger("IoTDB") class PoolConfig(object): - def __init__(self, host: str = None, port: str = None, user_name: str = None, password: str = None, - node_urls: list = None, - fetch_size: int = DEFAULT_FETCH_SIZE, time_zone: str = DEFAULT_TIME_ZONE, - max_retry: int = DEFAULT_MAX_RETRY, enable_compression: bool = False): + def __init__( + self, + host: str = None, + port: str = None, + user_name: str = None, + password: str = None, + node_urls: list = None, + fetch_size: int = DEFAULT_FETCH_SIZE, + time_zone: str = DEFAULT_TIME_ZONE, + max_retry: int = DEFAULT_MAX_RETRY, + enable_compression: bool = False, + ): self.host = host self.port = port if node_urls is None: if host is None or port is None: - raise ValueError("(host,port) and node_urls cannot be None at the same time.") + raise ValueError( + "(host,port) and node_urls cannot be None at the same time." + ) node_urls = [] self.node_urls = node_urls self.user_name = user_name @@ -51,8 +61,9 @@ class PoolConfig(object): class SessionPool(object): - - def __init__(self, pool_config: PoolConfig, max_pool_size: int, wait_timeout_in_ms: int): + def __init__( + self, pool_config: PoolConfig, max_pool_size: int, wait_timeout_in_ms: int + ): self.__pool_config = pool_config self.__max_pool_size = max_pool_size self.__wait_timeout_in_ms = wait_timeout_in_ms / 1000 @@ -63,13 +74,23 @@ class SessionPool(object): def __construct_session(self) -> Session: if len(self.__pool_config.node_urls) > 0: - session = Session.init_from_node_urls(self.__pool_config.node_urls, self.__pool_config.user_name, - self.__pool_config.password, self.__pool_config.fetch_size, - self.__pool_config.time_zone) + session = Session.init_from_node_urls( + self.__pool_config.node_urls, + self.__pool_config.user_name, + self.__pool_config.password, + self.__pool_config.fetch_size, + self.__pool_config.time_zone, + ) else: - session = Session(self.__pool_config.host, self.__pool_config.port, self.__pool_config.user_name, - self.__pool_config.password, self.__pool_config.fetch_size, self.__pool_config.time_zone) + session = Session( + self.__pool_config.host, + self.__pool_config.port, + self.__pool_config.user_name, + self.__pool_config.password, + self.__pool_config.fetch_size, + self.__pool_config.time_zone, + ) session.open(self.__pool_config.enable_compression) return session @@ -97,8 +118,11 @@ class SessionPool(object): break else: if time.time() - start > self.__wait_timeout_in_ms: - raise TimeoutError("Wait to get session timeout in SessionPool, current pool size: {0}" - .format(self.__max_pool_size)) + raise TimeoutError( + "Wait to get session timeout in SessionPool, current pool size: {0}".format( + self.__max_pool_size + ) + ) time.sleep(1) session = self.__poll_session() @@ -115,7 +139,9 @@ class SessionPool(object): def put_back(self, session: Session): if self.__closed: - raise ConnectionError("SessionPool has already been closed, please close the session manually.") + raise ConnectionError( + "SessionPool has already been closed, please close the session manually." + ) if session.is_open(): self.__queue.put(session) @@ -134,7 +160,9 @@ class SessionPool(object): logger.info("SessionPool has been closed successfully.") -def create_session_pool(pool_config: PoolConfig, max_pool_size: int, wait_timeout_in_ms: int) -> SessionPool: +def create_session_pool( + pool_config: PoolConfig, max_pool_size: int, wait_timeout_in_ms: int +) -> SessionPool: if max_pool_size <= 0: max_pool_size = multiprocessing.cpu_count() * DEFAULT_MULTIPIE return SessionPool(pool_config, max_pool_size, wait_timeout_in_ms) diff --git a/iotdb-client/client-py/pom.xml b/iotdb-client/client-py/pom.xml index 74c6309fd5309bc482fc781a3715cc71692506eb..7e87085a57fdf9ddc988486afac853ce920a8c91 100644 --- a/iotdb-client/client-py/pom.xml +++ b/iotdb-client/client-py/pom.xml @@ -38,11 +38,10 @@ true - + org.apache.maven.plugins maven-clean-plugin - 2.4.1 @@ -52,6 +51,15 @@ false + + apache_iotdb.egg-info + + + build + + + dist + ./ @@ -114,9 +122,59 @@ + + copy-setup-file-resources + + generate-sources + + copy-resources + + + utf-8 + ${basedir}/ + + + ${basedir}/target/classes + + setup.py + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + write-python-version + + regex-property + + generate-resources + + python_version + -SNAPSHOT + ${project.version} + \.dev0 + false + + + + + ${basedir}/resources + + setup.py + + true + + diff --git a/iotdb-client/client-py/release.sh b/iotdb-client/client-py/release.sh index 2d9ca299795058917af171f9219e6b411f55dab0..4a3175d42a3b11e138707a317b88f3ac577f07d5 100755 --- a/iotdb-client/client-py/release.sh +++ b/iotdb-client/client-py/release.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # # # Licensed to the Apache Software Foundation (ASF) under one @@ -20,21 +20,22 @@ # # the python version must be python3. -python --version +python3 --version rm -Rf build rm -Rf dist rm -Rf iotdb_session.egg_info # (Re-)build generated code -(cd ..; mvn clean generate-sources -pl client-py -am) - -# Run Linting -flake8 +(cd ../..; mvn clean package -pl iotdb-client/client-py -am) # Run unit tests -pytest . +if [ "$1" == "test" ]; then + pytest . +fi # See https://packaging.python.org/tutorials/packaging-projects/ -python setup.py sdist bdist_wheel -twine upload --repository pypi dist/* \ No newline at end of file +python3 setup.py sdist bdist_wheel +if [ "$1" == "release" ]; then + python3 -m twine upload dist/* +fi diff --git a/iotdb-client/client-py/requirements_dev.txt b/iotdb-client/client-py/requirements_dev.txt index df21910441f946dc8e921fde466399f1c032a315..db282583dc88ef3d86ee69a3147873f621bad3f6 100644 --- a/iotdb-client/client-py/requirements_dev.txt +++ b/iotdb-client/client-py/requirements_dev.txt @@ -20,7 +20,7 @@ # Pytest to run tests pytest==7.2.0 flake8==3.9.0 -black==20.8b1 +black==22.3.0 # Testcontainer testcontainers==3.4.2 # For releases diff --git a/iotdb-client/client-py/setup.py b/iotdb-client/client-py/resources/setup.py similarity index 98% rename from iotdb-client/client-py/setup.py rename to iotdb-client/client-py/resources/setup.py index 364f42e5ae6ad23623b8432d3dd465441fed2d1b..3af2705af2653ee64a0977a7e40c818a3e48f1e3 100644 --- a/iotdb-client/client-py/setup.py +++ b/iotdb-client/client-py/resources/setup.py @@ -31,7 +31,7 @@ print(long_description) setuptools.setup( name="apache-iotdb", # Replace with your own username - version="1.0.0", + version="${python_version}", author=" Apache Software Foundation", author_email="dev@iotdb.apache.org", description="Apache IoTDB client API", diff --git a/iotdb-client/client-py/tests/test_session.py b/iotdb-client/client-py/tests/test_session.py index 5118fe90658f4a33d56b1bbb29a11a4f317cebf3..197b9b9de522bea0ccbc80f9af8b54adb19bc158 100644 --- a/iotdb-client/client-py/tests/test_session.py +++ b/iotdb-client/client-py/tests/test_session.py @@ -59,8 +59,16 @@ def session_test(use_session_pool=False): db: IoTDBContainer if use_session_pool: - pool_config = PoolConfig(db.get_container_host_ip(), db.get_exposed_port(6667), "root", "root", None, 1024, - "Asia/Shanghai", 3) + pool_config = PoolConfig( + db.get_container_host_ip(), + db.get_exposed_port(6667), + "root", + "root", + None, + 1024, + "Asia/Shanghai", + 3, + ) session_pool = create_session_pool(pool_config, 1, 3000) session = session_pool.get_session() else: @@ -170,14 +178,14 @@ def session_test(use_session_pool=False): # delete time series if ( - session.delete_time_series( - [ - "root.sg_test_01.d_01.s_07", - "root.sg_test_01.d_01.s_08", - "root.sg_test_01.d_01.s_09", - ] - ) - < 0 + session.delete_time_series( + [ + "root.sg_test_01.d_01.s_07", + "root.sg_test_01.d_01.s_08", + "root.sg_test_01.d_01.s_09", + ] + ) + < 0 ): test_fail() print_message("delete time series failed") @@ -213,10 +221,10 @@ def session_test(use_session_pool=False): TSDataType.TEXT, ] if ( - session.insert_record( - "root.sg_test_01.d_01", 1, measurements_, data_types_, values_ - ) - < 0 + session.insert_record( + "root.sg_test_01.d_01", 1, measurements_, data_types_, values_ + ) + < 0 ): test_fail() print_message("insert record failed") @@ -233,10 +241,10 @@ def session_test(use_session_pool=False): data_type_list_ = [data_types_, data_types_] device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_02"] if ( - session.insert_records( - device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ - ) - < 0 + session.insert_records( + device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ + ) + < 0 ): test_fail() print_message("insert records failed") @@ -353,14 +361,14 @@ def session_test(use_session_pool=False): values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] if ( - session.insert_records_of_one_device( - "root.sg_test_01.d_01", - time_list, - measurements_list, - data_types_list, - values_list, - ) - < 0 + session.insert_records_of_one_device( + "root.sg_test_01.d_01", + time_list, + measurements_list, + data_types_list, + values_list, + ) + < 0 ): test_fail() print_message("insert records of one device failed") diff --git a/iotdb-client/client-py/tests/test_session_pool.py b/iotdb-client/client-py/tests/test_session_pool.py index fc71882fadc7f3ff2e448b5b0720bf7665222385..43351be251690a1a1e9dc0ff839fd295d8e3b721 100644 --- a/iotdb-client/client-py/tests/test_session_pool.py +++ b/iotdb-client/client-py/tests/test_session_pool.py @@ -27,8 +27,16 @@ def test_session_pool(): with IoTDBContainer(CONTAINER_NAME) as db: db: IoTDBContainer max_pool_size = 2 - pool_config = PoolConfig(db.get_container_host_ip(), db.get_exposed_port(6667), "root", "root", - [], 1024, "Asia/Shanghai", 3) + pool_config = PoolConfig( + db.get_container_host_ip(), + db.get_exposed_port(6667), + "root", + "root", + [], + 1024, + "Asia/Shanghai", + 3, + ) session_pool = create_session_pool(pool_config, max_pool_size, 3000) session = session_pool.get_session() assert session.is_open() is True @@ -39,7 +47,11 @@ def test_session_pool(): session_pool.get_session() except TimeoutError as e: timeout = True - assert str(e) == "Wait to get session timeout in SessionPool, current pool size: " + str(max_pool_size) + assert str( + e + ) == "Wait to get session timeout in SessionPool, current pool size: " + str( + max_pool_size + ) assert timeout is True Thread(target=lambda: session_pool.put_back(session2)).start() @@ -60,7 +72,10 @@ def test_session_pool(): session_pool.put_back(session3) except ConnectionError as e: is_closed = True - assert str(e) == "SessionPool has already been closed, please close the session manually." + assert ( + str(e) + == "SessionPool has already been closed, please close the session manually." + ) assert is_closed is True @@ -69,7 +84,9 @@ def test_session_pool_by_node_urls(): db: IoTDBContainer node_url = db.get_container_host_ip() + ":" + str(db.get_exposed_port(6667)) max_pool_size = 1 - pool_config = PoolConfig(node_urls=[node_url], user_name="root", password="root") + pool_config = PoolConfig( + node_urls=[node_url], user_name="root", password="root" + ) session_pool = create_session_pool(pool_config, max_pool_size, 3000) session = session_pool.get_session() @@ -79,7 +96,11 @@ def test_session_pool_by_node_urls(): session_pool.get_session() except TimeoutError as e: timeout = True - assert str(e) == "Wait to get session timeout in SessionPool, current pool size: " + str(max_pool_size) + assert str( + e + ) == "Wait to get session timeout in SessionPool, current pool size: " + str( + max_pool_size + ) assert timeout is True