[python] Add multiple version of document (#11391)

* Using sphinx-multiversion to support multiple version
of python api document
* change python api sidebar link
* fix whitelist_externals deprecated and add git

close: #8227

.github/workflows/py-ci.yml

@@ -97,6 +97,10 @@ jobs:
     timeout-minutes: 15
     needs: lint
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env-list: [doc-build, doc-build-multi]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
@@ -106,9 +110,9 @@ jobs:
       - name: Install Dependences
         run: |
           python -m pip install --upgrade ${{ env.DEPENDENCES }}
-      - name: Run Tests Build Docs
+      - name: Run Build Docs Tests ${{ matrix.env-list }}
         run: |
-          python -m tox -vv -e doc-build-test
+          python -m tox -vv -e ${{ matrix.env-list }}
   local-ci:
     name: Local CI
     timeout-minutes: 15

docs/configs/docsdev.js

@@ -426,7 +426,7 @@ export default {
                     },
                     {
                         title: 'PyDolphinScheduler',
-                        link: '/python/index.html',
+                        link: '/python/dev/index.html',
                     },
                     {
                         title: 'Health Check',
@@ -1005,7 +1005,7 @@ export default {
                     },
                     {
                         title: 'PyDolphinScheduler',
-                        link: '/python/index.html',
+                        link: '/python/dev/index.html',
                     },
                     {
                         title: '健康检查',

dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@@ -146,10 +146,24 @@ python -m flake8
 
 #### Testing
 
-## Build Docs
+## Build Document
 
 We use [sphinx][sphinx] to build docs. Dolphinscheduler Python API CI would automatically build docs when you submit pull request in 
-GitHub. You may locally ensure docs could be built suceessfully in case the failure blocks CI.
+GitHub. You may locally ensure docs could be built successfully in case the failure blocks CI, you can build by tox or manual.
+
+### Build Document Automatically with tox
+
+We integrated document build process into tox, you can build the latest document and all document(including history documents) via
+single command
+
+```shell
+# Build the latest document in dev branch
+tox -e doc-build
+# Build all documents, which including the latest and all history documents
+tox -e doc-build-multi
+```
+
+### Build Document Manually
 
 To build docs locally, install sphinx and related python modules first via:
 
@@ -157,13 +171,16 @@ To build docs locally, install sphinx and related python modules first via:
 python -m pip install '.[doc]'
 ``` 
 
-Then 
+Then go to document directory and execute the build command
 
 ```shell
 cd pydolphinscheduler/docs/
 make clean && make html
 ```
 
+> NOTE: We support build multiple versions of documents with [sphinx-multiversion](https://holzhaus.github.io/sphinx-multiversion/master/index.html),
+> you can build with command `git fetch --tags && make clean && make multiversion`
+
 ## Testing
 
 pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create

dolphinscheduler-python/pydolphinscheduler/docs/Makefile

@@ -24,6 +24,7 @@
 # Add opts `turn warnings into errors` strict sphinx-build behavior
 SPHINXOPTS    ?= -W
 SPHINXBUILD   ?= sphinx-build
+SPHINXMULTIVERSION   ?= sphinx-multiversion
 SOURCEDIR     = source
 BUILDDIR      = build
 
@@ -37,3 +38,7 @@ help:
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Create multiple version of docs
+multiversion:
+	@$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html"

dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html

+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+#}
+
+{% if versions %}
+<h3>{{ _('Versions') }}</h3>
+<ul>
+  {%- for item in versions %}
+  <li><a href="{{ item.url }}">{{ item.name }}</a></li>
+  {%- endfor %}
+</ul>
+{% endif %}

dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html

+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+#}
+
+{%- if current_version %}
+<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    <span class="fa fa-book"> Other Versions</span>
+    v: {{ current_version.name }}
+    <span class="fa fa-caret-down"></span>
+  </span>
+  <div class="rst-other-versions">
+    {%- if versions.tags %}
+    <dl>
+      <dt>Tags</dt>
+      {%- for item in versions.tags %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+    {%- if versions.branches %}
+    <dl>
+      <dt>Branches</dt>
+      {%- for item in versions.branches %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+  </div>
+</div>
+{%- endif %}

dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst

@@ -24,8 +24,8 @@ Core
 .. automodule:: pydolphinscheduler.core
   :inherited-members:
 
-Sides
------
+Models
+------
 
 .. automodule:: pydolphinscheduler.models
   :inherited-members:

dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py

@@ -26,17 +26,31 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+
+import os
+import sys
+from pathlib import Path
+
+# For sphinx-multiversion, we need to build API docs of the corresponding package version, related issue:
+# https://github.com/Holzhaus/sphinx-multiversion/issues/42
+pkg_src_dir = (
+    Path(os.environ.get("SPHINX_MULTIVERSION_SOURCEDIR", default="."))
+    .joinpath("../../src")
+    .resolve()
+)
+sys.path.insert(0, str(pkg_src_dir))
+# Debug to uncomment this to see the source path
+# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=")
+# print(pkg_src_dir)
+# [print(p) for p in sys.path]
+# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=")
 
 
 # -- Project information -----------------------------------------------------
 
 project = "pydolphinscheduler"
 copyright = "2022, apache"
-author = "apache"
+author = "apache dolphinscheduler contributors"
 
 # The full version, including alpha/beta/rc tags
 release = "0.0.1"
@@ -60,11 +74,25 @@ extensions = [
     # Add inline tabbed content
     "sphinx_inline_tabs",
     "sphinx_copybutton",
+    "sphinx_multiversion",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
+# sphinx_multiversion configuration
+html_sidebars = {
+    "**": [
+        "versioning.html",
+    ],
+}
+# Match all exists tag for pydolphinscheduler expect version 2.0.4(not release apache dolphinscheduler)
+smv_tag_whitelist = r"^(?!2.0.4)\d+\.\d+\.\d+$"
+smv_branch_whitelist = "dev"
+smv_remote_whitelist = r"^(origin|upstream)$"
+smv_released_pattern = "^refs/tags/.*$"
+smv_outputdir_format = "versions/{ref.name}"
+
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.

dolphinscheduler-python/pydolphinscheduler/setup.py

@@ -53,6 +53,10 @@ doc = [
     "sphinx-click>=3.0",
     "sphinx-inline-tabs",
     "sphinx-copybutton>=0.4.0",
+    # Unreleased package have a feature we want(use correct version package for API ref), so we install from
+    # GitHub directly, see also:
+    # https://github.com/Holzhaus/sphinx-multiversion/issues/42#issuecomment-1210539786
+    "sphinx-multiversion @ git+https://github.com/Holzhaus/sphinx-multiversion#egg=sphinx-multiversion",
 ]
 
 test = [

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part."""
+"""Init Models package, keeping object related to DolphinScheduler covert from Java Gateway Service."""
 
 from pydolphinscheduler.models.base import Base
 from pydolphinscheduler.models.base_side import BaseSide

dolphinscheduler-python/pydolphinscheduler/tox.ini

@@ -16,10 +16,12 @@
 # under the License.
 
 [tox]
-envlist = local-ci, auto-lint, lint, doc-build-test, code-test, integrate-test, local-integrate-test, py{36,37,38,39}
+envlist = local-ci, auto-lint, lint, doc-build, doc-build-multi, code-test, integrate-test, local-integrate-test, py{36,37,38,39}
 
 [testenv]
-whitelist_externals = make
+allowlist_externals = 
+    make
+    git
 
 [testenv:auto-lint]
 extras = style
@@ -42,12 +44,20 @@ extras = test
 commands =
     python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/
 
-[testenv:doc-build-test]
+[testenv:doc-build]
 extras = doc
 commands =
     make -C {toxinidir}/docs clean
     make -C {toxinidir}/docs html
 
+[testenv:doc-build-multi]
+extras = doc
+commands =
+    # Get all tags for `multiversion` subcommand
+    git fetch --tags
+    make -C {toxinidir}/docs clean
+    make -C {toxinidir}/docs multiversion
+
 [testenv:integrate-test]
 extras = test
 commands =
@@ -65,4 +75,4 @@ extras = dev
 commands =
     {[testenv:lint]commands}
     {[testenv:code-test]commands}
-    {[testenv:doc-build-test]commands}
+    {[testenv:doc-build]commands}