diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index fb8324ebcddbf1caf26847a1fb0ce519b210becd..3ab10f7d2e58cba433abd8481537edb17073c0ab 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -50,6 +50,8 @@ jobs: python-version: 3.7 - name: Install Development Dependences run: pip install -r requirements_dev.txt + - name: Run Isort Checking + run: isort --check . - name: Run Black Checking run: black --check . - name: Run Flake8 Checking diff --git a/dolphinscheduler-python/pydolphinscheduler/.isort.cfg b/dolphinscheduler-python/pydolphinscheduler/.isort.cfg new file mode 100644 index 0000000000000000000000000000000000000000..70fa2e05bd4ecf0c17a13f7e856ca63d7e3ebd90 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/.isort.cfg @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[settings] +profile=black diff --git a/dolphinscheduler-python/pydolphinscheduler/README.md b/dolphinscheduler-python/pydolphinscheduler/README.md index a0cb7486b4f4d573bd4d241875ebf660d330214d..b06632cc005902be96407d0caea93db1840c7a00 100644 --- a/dolphinscheduler-python/pydolphinscheduler/README.md +++ b/dolphinscheduler-python/pydolphinscheduler/README.md @@ -21,6 +21,7 @@ [![GitHub Build][ga-py-test]][ga] [![Code style: black][black-shield]][black-gh] +[![Imports: isort][isort-shield]][isort-gh] pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition your workflow by python code, aka workflow-as-codes. @@ -104,16 +105,20 @@ and would be implemented in the further. ### Code Style -We use [Black][black] for code formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm] -or [IntelliJ IDEA][idea], maybe you could follow [Black-integration][black-editor] to configure them in your environment. +We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code +formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea], +maybe you could follow [Black-integration][black-editor] to configure them in your environment. -Our Python API CI would automatically run unittest when you submit pull request in GitHub, you could also run -static check locally. +Our Python API CI would automatically run code style checker and unittest when you submit pull request in +GitHub, you could also run static check locally. ```shell -# We recommend you run Black before Flake8, because Black could auto fix some code style issue +# We recommend you run isort and Black before Flake8, because Black could auto fix some code style issue # but Flake8 just hint when code style not match pep8 +# Run Isort +isort . + # Run Black black . @@ -158,8 +163,11 @@ this command output. [flake8]: https://flake8.pycqa.org/en/latest/index.html [black-editor]: https://black.readthedocs.io/en/stable/integrations/editors.html#pycharm-intellij-idea [coverage]: https://coverage.readthedocs.io/en/stable/ +[isort]: https://pycqa.github.io/isort/index.html [ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-ci.yml/badge.svg?branch=dev [ga]: https://github.com/apache/dolphinscheduler/actions [black-shield]: https://img.shields.io/badge/code%20style-black-000000.svg [black-gh]: https://github.com/psf/black +[isort-shield]: https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336 +[isort-gh]: https://pycqa.github.io/isort/ diff --git a/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py index 451bb75b227faa3433d7526f94bc7c9e5eb7c32f..d58bd753b00aa5ab2f7a098f69fd5938ff28eeaa 100644 --- a/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py @@ -33,7 +33,6 @@ it will instantiate and run all the task it have. from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.tasks.shell import Shell - with ProcessDefinition( name="tutorial", schedule="0 0 0 * * ? *", diff --git a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt index fa40e3caa81b5ecd4ea708df77a2b4a162dce4a3..d31a56dc44fcae592b48789589a676dcb24feb5f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt +++ b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt @@ -24,3 +24,4 @@ coverage flake8 flake8-docstrings flake8-black +isort diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py index ce71a4a0649eeed141c2433660eb0b05b75343da..d636d512e171a7fdd7fcaed4d3f9234aee1b8442 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py @@ -17,7 +17,7 @@ """DolphinScheduler Base object.""" -from typing import Optional, Dict +from typing import Dict, Optional # from pydolphinscheduler.side.user import User from pydolphinscheduler.utils.string import attr2camel diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 152119bf5d35ec4e361f8a5a4a5e043564e8c9de..e5769f518d63e67f78a8ded5dd6ac37d0c7646d9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -19,16 +19,16 @@ import json from datetime import datetime -from typing import Optional, List, Dict, Set, Any +from typing import Any, Dict, List, Optional, Set from pydolphinscheduler.constants import ( - ProcessDefinitionReleaseState, ProcessDefinitionDefault, + ProcessDefinitionReleaseState, ) from pydolphinscheduler.core.base import Base from pydolphinscheduler.java_gateway import launch_gateway -from pydolphinscheduler.side import Tenant, Project, User -from pydolphinscheduler.utils.date import conv_from_str, conv_to_schedule, MAX_DATETIME +from pydolphinscheduler.side import Project, Tenant, User +from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule class ProcessDefinitionContext: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 6f9e454ef002723762ecb4cd05763b57e7625082..c8eb54ad437a092754b0c34d125a08349f77ebc5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -17,19 +17,21 @@ """DolphinScheduler ObjectJsonBase, TaskParams and Task object.""" -from typing import Optional, List, Dict, Set, Union, Sequence, Tuple +from typing import Dict, List, Optional, Sequence, Set, Tuple, Union from pydolphinscheduler.constants import ( - TaskPriority, ProcessDefinitionDefault, TaskFlag, + TaskPriority, TaskTimeoutFlag, ) from pydolphinscheduler.core.base import Base -from pydolphinscheduler.core.process_definition import ProcessDefinition -from pydolphinscheduler.core.process_definition import ProcessDefinitionContext +from pydolphinscheduler.core.process_definition import ( + ProcessDefinition, + ProcessDefinitionContext, +) from pydolphinscheduler.java_gateway import launch_gateway -from pydolphinscheduler.utils.string import snake2camel, class_name2camel +from pydolphinscheduler.utils.string import class_name2camel, snake2camel class ObjectJsonBase: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py index 027ca94bc4c875e10d59c94956dd379d48817301..f40bb363a628c8870c7decdc2caf79ac9afedb58 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py @@ -20,7 +20,7 @@ from typing import Any, Optional from py4j.java_collections import JavaMap -from py4j.java_gateway import JavaGateway, GatewayParameters +from py4j.java_gateway import GatewayParameters, JavaGateway from pydolphinscheduler.constants import JavaGatewayDefault diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py index 96051a2941868c9688bc77e005fe41939ac829bd..02382cadc564ba9e9bbba7c48bc62f2530607a63 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py @@ -19,8 +19,8 @@ from typing import Optional -from pydolphinscheduler.core.base_side import BaseSide from pydolphinscheduler.constants import ProcessDefinitionDefault +from pydolphinscheduler.core.base_side import BaseSide from pydolphinscheduler.java_gateway import launch_gateway diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py index 720135186c0b485eb3ed09ede27293ba9744d3e1..9d6664e14b005d868a26735c424926660277a527 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py @@ -21,7 +21,7 @@ from typing import Optional from pydolphinscheduler.constants import ProcessDefinitionDefault from pydolphinscheduler.core.base_side import BaseSide -from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker +from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway class Queue(BaseSide): diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py index e2a4cd1cd4429d5137b1b723b795e597a695dcfb..18cf93e3181fb991bf2765f04768187ecdcc8d95 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py @@ -18,6 +18,7 @@ """Date util function collections.""" from datetime import datetime + from pydolphinscheduler.constants import Delimiter, Time LEN_SUPPORT_DATETIME = ( diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index 930909a4fcdb9196a0ab215bccb772b9658b4929..a678649041cd1c2dede14baa1ebb599e76d18ad4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -20,9 +20,8 @@ from datetime import datetime from typing import Any -from pydolphinscheduler.utils.date import conv_to_schedule - import pytest +from freezegun import freeze_time from pydolphinscheduler.constants import ( ProcessDefinitionDefault, @@ -30,9 +29,9 @@ from pydolphinscheduler.constants import ( ) from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import TaskParams -from pydolphinscheduler.side import Tenant, Project, User +from pydolphinscheduler.side import Project, Tenant, User +from pydolphinscheduler.utils.date import conv_to_schedule from tests.testing.task import Task -from freezegun import freeze_time TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 46103375e7b1815b76aee301b9dcd85ad1d96da0..63d6496672f20cd6be30f946986474fa61740d5b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -18,9 +18,10 @@ """Test Task class function.""" from unittest.mock import patch + import pytest -from pydolphinscheduler.core.task import TaskParams, TaskRelation, Task +from pydolphinscheduler.core.task import Task, TaskParams, TaskRelation from tests.testing.task import Task as testTask diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py index d0456a6af26600371c27450a73b708adababcab1..3c8831e16d78935d153551ccd07cee0c030839d4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py @@ -18,7 +18,7 @@ """Test pydolphinscheduler java gateway.""" -from py4j.java_gateway import java_import, JavaGateway +from py4j.java_gateway import JavaGateway, java_import def test_gateway_connect(): diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py index 648f2c40a8b3477320200cc375db4dcbfdf0430d..bb204c9294e4d9b6d6a0de06aedd5e6c42075a9b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py @@ -17,13 +17,11 @@ """Test utils.date module.""" -import pytest from datetime import datetime -from pydolphinscheduler.utils.date import ( - conv_from_str, - conv_to_schedule, - FMT_STD, -) + +import pytest + +from pydolphinscheduler.utils.date import FMT_STD, conv_from_str, conv_to_schedule curr_date = datetime.now() diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py index 6942d7e75ec5001e615c8bbd812f90a33c68ee8a..2ccd206df18af2687e668dd3d0f8fee9a3aa77d5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py @@ -17,9 +17,10 @@ """Test utils.string module.""" -from pydolphinscheduler.utils.string import attr2camel, snake2camel, class_name2camel import pytest +from pydolphinscheduler.utils.string import attr2camel, class_name2camel, snake2camel + @pytest.mark.parametrize( "snake, expect",