[python] Refactor structure to avoid cycle import (#11167)

Currently, our core and side module dependent on each others. and will cause
cycle import in our codebase, especially in issue #10905, we try to refactor
to solve this problem.

This patch do the following change:

* Rename module `side` to `models`
* Move `core/base` and `core/sidebase` to dir `modules`
* Move `configuration` and `default_config.yaml` to the root of pydolphinscheduler

dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@@ -34,7 +34,7 @@ Now, we should install all dependence to make sure we could run test or check co
 
 ```shell
 cd dolphinscheduler/dolphinscheduler-python/pydolphinscheduler
-python -m pip install .[dev]
+python -m pip install -e '.[dev]'
 ```
 
 Next, we have to open pydolphinscheduler project in you editor. We recommend you use [pycharm][pycharm]

dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst

@@ -27,7 +27,7 @@ Core
 Sides
 -----
 
-.. automodule:: pydolphinscheduler.side
+.. automodule:: pydolphinscheduler.models
   :inherited-members:
 
 Tasks

dolphinscheduler-python/pydolphinscheduler/docs/source/config.rst

@@ -200,7 +200,7 @@ All Configurations in File
 
 Here are all our configurations for pydolphinscheduler.
 
-.. literalinclude:: ../../src/pydolphinscheduler/core/default_config.yaml
+.. literalinclude:: ../../src/pydolphinscheduler/default_config.yaml
    :language: yaml
    :lines: 18-
 

dolphinscheduler-python/pydolphinscheduler/setup.py

@@ -147,7 +147,7 @@ setup(
     package_dir={"": "src"},
     include_package_data=True,
     package_data={
-        "pydolphinscheduler": ["core/default_config.yaml"],
+        "pydolphinscheduler": ["default_config.yaml"],
     },
     platforms=["any"],
     classifiers=[

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/cli/commands.py

@@ -21,7 +21,7 @@ import click
 from click import echo
 
 from pydolphinscheduler import __version__
-from pydolphinscheduler.core.configuration import (
+from pydolphinscheduler.configuration import (
     get_single_config,
     init_config_file,
     set_single_config,

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/__init__.py

@@ -23,8 +23,8 @@ from pydolphinscheduler.core.process_definition import ProcessDefinition
 from pydolphinscheduler.core.task import Task
 
 __all__ = [
+    "Database",
     "Engine",
     "ProcessDefinition",
     "Task",
-    "Database",
 ]

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py

@@ -21,12 +21,11 @@ import json
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Set
 
+from pydolphinscheduler import configuration
 from pydolphinscheduler.constants import TaskType
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base import Base
 from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException
 from pydolphinscheduler.java_gateway import launch_gateway
-from pydolphinscheduler.side import Project, Tenant, User
+from pydolphinscheduler.models import Base, Project, Tenant, User
 from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule
 
 
@@ -170,7 +169,7 @@ class ProcessDefinition(Base):
     def user(self) -> User:
         """Get user object.
 
-        For now we just get from python side but not from java gateway side, so it may not correct.
+        For now we just get from python models but not from java gateway models, so it may not correct.
         """
         return User(name=self._user, tenant=self._tenant)
 
@@ -358,10 +357,10 @@ class ProcessDefinition(Base):
         self.start()
 
     def _ensure_side_model_exists(self):
-        """Ensure process definition side model exists.
+        """Ensure process definition models model exists.
 
-        For now, side object including :class:`pydolphinscheduler.side.project.Project`,
-        :class:`pydolphinscheduler.side.tenant.Tenant`, :class:`pydolphinscheduler.side.user.User`.
+        For now, models object including :class:`pydolphinscheduler.models.project.Project`,
+        :class:`pydolphinscheduler.models.tenant.Tenant`, :class:`pydolphinscheduler.models.user.User`.
         If these model not exists, would create default value in
         :class:`pydolphinscheduler.constants.ProcessDefinitionDefault`.
         """

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py

@@ -19,7 +19,7 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core.base import Base
+from pydolphinscheduler.models import Base
 
 
 class Resource(Base):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py

@@ -20,6 +20,7 @@
 from logging import getLogger
 from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
+from pydolphinscheduler import configuration
 from pydolphinscheduler.constants import (
     Delimiter,
     ResourceKey,
@@ -27,13 +28,12 @@ from pydolphinscheduler.constants import (
     TaskPriority,
     TaskTimeoutFlag,
 )
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base import Base
 from pydolphinscheduler.core.process_definition import (
     ProcessDefinition,
     ProcessDefinitionContext,
 )
 from pydolphinscheduler.java_gateway import launch_gateway
+from pydolphinscheduler.models import Base
 
 logger = getLogger(__name__)
 

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/default_config.yaml → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/default_config.yaml

@@ -22,7 +22,7 @@ java_gateway:
   address: 127.0.0.1
 
   # The port of Python gateway server start. Define which port you could connect to Python gateway server from
-  # Python API side.
+  # Python API models.
   port: 25333
 
   # Whether automatically convert Python objects to Java Objects. Default value is ``True``. There is some

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py

@@ -35,7 +35,7 @@ task_dependent:
 
 task_dependent(this task dependent on task_dependent_external.task_1 and task_dependent_external.task_2).
 """
-from pydolphinscheduler.core import configuration
+from pydolphinscheduler import configuration
 from pydolphinscheduler.core.process_definition import ProcessDefinition
 from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or
 from pydolphinscheduler.tasks.shell import Shell

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@@ -22,8 +22,8 @@ from typing import Any, Optional
 from py4j.java_collections import JavaMap
 from py4j.java_gateway import GatewayParameters, JavaGateway
 
+from pydolphinscheduler import configuration
 from pydolphinscheduler.constants import JavaGatewayDefault
-from pydolphinscheduler.core import configuration
 from pydolphinscheduler.exceptions import PyDSJavaGatewayException
 
 

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py

@@ -17,13 +17,17 @@
 
 """Init Side package, Side package keep object related to DolphinScheduler but not in the Core part."""
 
-from pydolphinscheduler.side.project import Project
-from pydolphinscheduler.side.queue import Queue
-from pydolphinscheduler.side.tenant import Tenant
-from pydolphinscheduler.side.user import User
-from pydolphinscheduler.side.worker_group import WorkerGroup
+from pydolphinscheduler.models.base import Base
+from pydolphinscheduler.models.base_side import BaseSide
+from pydolphinscheduler.models.project import Project
+from pydolphinscheduler.models.queue import Queue
+from pydolphinscheduler.models.tenant import Tenant
+from pydolphinscheduler.models.user import User
+from pydolphinscheduler.models.worker_group import WorkerGroup
 
 __all__ = [
+    "Base",
+    "BaseSide",
     "Project",
     "Tenant",
     "User",

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base.py

@@ -19,7 +19,7 @@
 
 from typing import Dict, Optional
 
-# from pydolphinscheduler.side.user import User
+# from pydolphinscheduler.models.user import User
 from pydolphinscheduler.utils.string import attr2camel
 
 

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base_side.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py

@@ -15,16 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Module for side object."""
+"""Module for models object."""
 
 from typing import Optional
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base import Base
+from pydolphinscheduler import configuration
+from pydolphinscheduler.models import Base
 
 
 class BaseSide(Base):
-    """Base class for side object, it declare base behavior for them."""
+    """Base class for models object, it declare base behavior for them."""
 
     def __init__(self, name: str, description: Optional[str] = None):
         super().__init__(name, description)

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py

@@ -19,9 +19,9 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base_side import BaseSide
+from pydolphinscheduler import configuration
 from pydolphinscheduler.java_gateway import launch_gateway
+from pydolphinscheduler.models import BaseSide
 
 
 class Project(BaseSide):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py

@@ -19,9 +19,9 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base_side import BaseSide
+from pydolphinscheduler import configuration
 from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway
+from pydolphinscheduler.models import BaseSide
 
 
 class Queue(BaseSide):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/tenant.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py

@@ -19,9 +19,9 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base_side import BaseSide
+from pydolphinscheduler import configuration
 from pydolphinscheduler.java_gateway import launch_gateway
+from pydolphinscheduler.models import BaseSide
 
 
 class Tenant(BaseSide):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/user.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py

@@ -19,10 +19,9 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.base_side import BaseSide
+from pydolphinscheduler import configuration
 from pydolphinscheduler.java_gateway import launch_gateway
-from pydolphinscheduler.side.tenant import Tenant
+from pydolphinscheduler.models import BaseSide, Tenant
 
 
 class User(BaseSide):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/worker_group.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/worker_group.py

@@ -19,7 +19,7 @@
 
 from typing import Optional
 
-from pydolphinscheduler.core.base_side import BaseSide
+from pydolphinscheduler.models import BaseSide
 
 
 class WorkerGroup(BaseSide):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/condition.py

@@ -20,9 +20,9 @@
 from typing import Dict, List
 
 from pydolphinscheduler.constants import TaskType
-from pydolphinscheduler.core.base import Base
 from pydolphinscheduler.core.task import Task
 from pydolphinscheduler.exceptions import PyDSParamException
+from pydolphinscheduler.models.base import Base
 
 
 class Status(Base):

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/dependent.py

@@ -20,10 +20,10 @@
 from typing import Dict, Optional, Tuple
 
 from pydolphinscheduler.constants import TaskType
-from pydolphinscheduler.core.base import Base
 from pydolphinscheduler.core.task import Task
 from pydolphinscheduler.exceptions import PyDSJavaGatewayException, PyDSParamException
 from pydolphinscheduler.java_gateway import launch_gateway
+from pydolphinscheduler.models.base import Base
 
 DEPENDENT_ALL_TASK_IN_WORKFLOW = "0"
 
@@ -31,8 +31,8 @@ DEPENDENT_ALL_TASK_IN_WORKFLOW = "0"
 class DependentDate(str):
     """Constant of Dependent date value.
 
-    These values set according to Java server side, if you want to add and change it,
-    please change Java server side first.
+    These values set according to Java server models, if you want to add and change it,
+    please change Java server models first.
     """
 
     # TODO Maybe we should add parent level to DependentDate for easy to use, such as

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py

@@ -20,9 +20,9 @@
 from typing import Dict, Optional
 
 from pydolphinscheduler.constants import TaskType
-from pydolphinscheduler.core.base import Base
 from pydolphinscheduler.core.task import Task
 from pydolphinscheduler.exceptions import PyDSParamException
+from pydolphinscheduler.models.base import Base
 
 
 class SwitchBranch(Base):

dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py

@@ -23,7 +23,7 @@ from pathlib import Path
 import pytest
 
 from pydolphinscheduler.cli.commands import cli
-from pydolphinscheduler.core.configuration import BUILD_IN_CONFIG_PATH, config_path
+from pydolphinscheduler.configuration import BUILD_IN_CONFIG_PATH, config_path
 from tests.testing.cli import CliTestWrapper
 from tests.testing.constants import DEV_MODE, ENV_PYDS_HOME
 from tests.testing.file import get_file_content

dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py

@@ -24,8 +24,8 @@ from typing import Any
 
 import pytest
 
-from pydolphinscheduler.core import configuration
-from pydolphinscheduler.core.configuration import (
+from pydolphinscheduler import configuration
+from pydolphinscheduler.configuration import (
     BUILD_IN_CONFIG_PATH,
     config_path,
     get_single_config,

dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py

@@ -24,11 +24,11 @@ from unittest.mock import patch
 import pytest
 from freezegun import freeze_time
 
-from pydolphinscheduler.core import configuration
+from pydolphinscheduler import configuration
 from pydolphinscheduler.core.process_definition import ProcessDefinition
 from pydolphinscheduler.core.resource import Resource
 from pydolphinscheduler.exceptions import PyDSParamException
-from pydolphinscheduler.side import Project, Tenant, User
+from pydolphinscheduler.models import Project, Tenant, User
 from pydolphinscheduler.tasks.switch import Branch, Default, Switch, SwitchCondition
 from pydolphinscheduler.utils.date import conv_to_schedule
 from tests.testing.task import Task

dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

@@ -26,7 +26,7 @@ path_code_tasks = project_root.joinpath("src", "pydolphinscheduler", "tasks")
 path_example = project_root.joinpath("src", "pydolphinscheduler", "examples")
 path_doc_tasks = project_root.joinpath("docs", "source", "tasks")
 path_default_config_yaml = project_root.joinpath(
-    "src", "pydolphinscheduler", "core", "default_config.yaml"
+    "src", "pydolphinscheduler", "default_config.yaml"
 )