This is an automated email from the ASF dual-hosted git repository. zhongjiajie pushed a commit to branch 3.0.1-prepare in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 82deeab2fb37813754bbfe9f00ef8327eca03723 Author: Jiajie Zhong <[email protected]> AuthorDate: Thu Aug 11 12:02:29 2022 +0800 [python] Add multiple version of document (#11391) * Using sphinx-multiversion to support multiple version of python api document * change python api sidebar link * fix whitelist_externals deprecated and add git close: #8227 (cherry picked from commit 26afd655cb26226aa5853355937e00880f259ee1) --- .github/workflows/py-ci.yml | 8 +++- .../pydolphinscheduler/DEVELOP.md | 23 +++++++++-- .../pydolphinscheduler/docs/Makefile | 5 +++ .../docs/source/_templates/versioning.html | 27 +++++++++++++ .../docs/source/_templates/versions.html | 46 ++++++++++++++++++++++ .../pydolphinscheduler/docs/source/api.rst | 4 +- .../pydolphinscheduler/docs/source/conf.py | 38 +++++++++++++++--- .../pydolphinscheduler/setup.py | 4 ++ .../src/pydolphinscheduler/side/__init__.py | 2 +- dolphinscheduler-python/pydolphinscheduler/tox.ini | 18 +++++++-- 10 files changed, 158 insertions(+), 17 deletions(-) diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index 2c8e578a7a..b0e042c4e7 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -97,6 +97,10 @@ jobs: timeout-minutes: 15 needs: lint runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env-list: [doc-build, doc-build-multi] steps: - uses: actions/checkout@v2 - name: Set up Python 3.7 @@ -106,9 +110,9 @@ jobs: - name: Install Dependences run: | python -m pip install --upgrade ${{ env.DEPENDENCES }} - - name: Run Tests Build Docs + - name: Run Build Docs Tests ${{ matrix.env-list }} run: | - python -m tox -vv -e doc-build-test + python -m tox -vv -e ${{ matrix.env-list }} local-ci: name: Local CI timeout-minutes: 15 diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index bdd0416675..926a467282 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -146,10 +146,24 @@ python -m flake8 #### Testing -## Build Docs +## Build Document We use [sphinx][sphinx] to build docs. Dolphinscheduler Python API CI would automatically build docs when you submit pull request in -GitHub. You may locally ensure docs could be built suceessfully in case the failure blocks CI. +GitHub. You may locally ensure docs could be built successfully in case the failure blocks CI, you can build by tox or manual. + +### Build Document Automatically with tox + +We integrated document build process into tox, you can build the latest document and all document(including history documents) via +single command + +```shell +# Build the latest document in dev branch +tox -e doc-build +# Build all documents, which including the latest and all history documents +tox -e doc-build-multi +``` + +### Build Document Manually To build docs locally, install sphinx and related python modules first via: @@ -157,13 +171,16 @@ To build docs locally, install sphinx and related python modules first via: python -m pip install '.[doc]' ``` -Then +Then go to document directory and execute the build command ```shell cd pydolphinscheduler/docs/ make clean && make html ``` +> NOTE: We support build multiple versions of documents with [sphinx-multiversion](https://holzhaus.github.io/sphinx-multiversion/master/index.html), +> you can build with command `git fetch --tags && make clean && make multiversion` + ## Testing pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/Makefile b/dolphinscheduler-python/pydolphinscheduler/docs/Makefile index 985198a779..ff2c4ebb44 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/Makefile +++ b/dolphinscheduler-python/pydolphinscheduler/docs/Makefile @@ -24,6 +24,7 @@ # Add opts `turn warnings into errors` strict sphinx-build behavior SPHINXOPTS ?= -W SPHINXBUILD ?= sphinx-build +SPHINXMULTIVERSION ?= sphinx-multiversion SOURCEDIR = source BUILDDIR = build @@ -37,3 +38,7 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Create multiple version of docs +multiversion: + @$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html" diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html new file mode 100644 index 0000000000..47136c45cf --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html @@ -0,0 +1,27 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{% if versions %} +<h3>{{ _('Versions') }}</h3> +<ul> + {%- for item in versions %} + <li><a href="{{ item.url }}">{{ item.name }}</a></li> + {%- endfor %} +</ul> +{% endif %} diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html new file mode 100644 index 0000000000..51b7271e9c --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html @@ -0,0 +1,46 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{%- if current_version %} +<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> + <span class="rst-current-version" data-toggle="rst-current-version"> + <span class="fa fa-book"> Other Versions</span> + v: {{ current_version.name }} + <span class="fa fa-caret-down"></span> + </span> + <div class="rst-other-versions"> + {%- if versions.tags %} + <dl> + <dt>Tags</dt> + {%- for item in versions.tags %} + <dd><a href="{{ item.url }}">{{ item.name }}</a></dd> + {%- endfor %} + </dl> + {%- endif %} + {%- if versions.branches %} + <dl> + <dt>Branches</dt> + {%- for item in versions.branches %} + <dd><a href="{{ item.url }}">{{ item.name }}</a></dd> + {%- endfor %} + </dl> + {%- endif %} + </div> +</div> +{%- endif %} diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst index 8e55ea5583..f2dd4e5c57 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst @@ -24,8 +24,8 @@ Core .. automodule:: pydolphinscheduler.core :inherited-members: -Sides ------ +Models +------ .. automodule:: pydolphinscheduler.side :inherited-members: diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py b/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py index b162e0cc50..23fc117fb7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py @@ -26,17 +26,31 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) + +import os +import sys +from pathlib import Path + +# For sphinx-multiversion, we need to build API docs of the corresponding package version, related issue: +# https://github.com/Holzhaus/sphinx-multiversion/issues/42 +pkg_src_dir = ( + Path(os.environ.get("SPHINX_MULTIVERSION_SOURCEDIR", default=".")) + .joinpath("../../src") + .resolve() +) +sys.path.insert(0, str(pkg_src_dir)) +# Debug to uncomment this to see the source path +# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") +# print(pkg_src_dir) +# [print(p) for p in sys.path] +# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") # -- Project information ----------------------------------------------------- project = "pydolphinscheduler" copyright = "2022, apache" -author = "apache" +author = "apache dolphinscheduler contributors" # The full version, including alpha/beta/rc tags release = "0.0.1" @@ -60,11 +74,25 @@ extensions = [ # Add inline tabbed content "sphinx_inline_tabs", "sphinx_copybutton", + "sphinx_multiversion", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] +# sphinx_multiversion configuration +html_sidebars = { + "**": [ + "versioning.html", + ], +} +# Match all exists tag for pydolphinscheduler expect version 2.0.4(not release apache dolphinscheduler) +smv_tag_whitelist = r"^(?!2.0.4)\d+\.\d+\.\d+$" +smv_branch_whitelist = "dev" +smv_remote_whitelist = r"^(origin|upstream)$" +smv_released_pattern = "^refs/tags/.*$" +smv_outputdir_format = "versions/{ref.name}" + # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index b1e3742310..fe36fa1018 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -53,6 +53,10 @@ doc = [ "sphinx-click>=3.0", "sphinx-inline-tabs", "sphinx-copybutton>=0.4.0", + # Unreleased package have a feature we want(use correct version package for API ref), so we install from + # GitHub directly, see also: + # https://github.com/Holzhaus/sphinx-multiversion/issues/42#issuecomment-1210539786 + "sphinx-multiversion @ git+https://github.com/Holzhaus/sphinx-multiversion#egg=sphinx-multiversion";, ] test = [ diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py index c4479ddf24..014da11b84 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part.""" +"""Init Models package, keeping object related to DolphinScheduler covert from Java Gateway Service.""" from pydolphinscheduler.side.project import Project from pydolphinscheduler.side.queue import Queue diff --git a/dolphinscheduler-python/pydolphinscheduler/tox.ini b/dolphinscheduler-python/pydolphinscheduler/tox.ini index 8e9280f057..186fe456d4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tox.ini +++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini @@ -16,10 +16,12 @@ # under the License. [tox] -envlist = local-ci, auto-lint, lint, doc-build-test, code-test, integrate-test, py{36,37,38,39} +envlist = local-ci, auto-lint, lint, doc-build, doc-build-multi, code-test, integrate-test, local-integrate-test, py{36,37,38,39} [testenv] -whitelist_externals = make +allowlist_externals = + make + git [testenv:auto-lint] extras = style @@ -42,12 +44,20 @@ extras = test commands = python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/ -[testenv:doc-build-test] +[testenv:doc-build] extras = doc commands = make -C {toxinidir}/docs clean make -C {toxinidir}/docs html +[testenv:doc-build-multi] +extras = doc +commands = + # Get all tags for `multiversion` subcommand + git fetch --tags + make -C {toxinidir}/docs clean + make -C {toxinidir}/docs multiversion + [testenv:integrate-test] extras = test commands = @@ -58,4 +68,4 @@ extras = dev commands = {[testenv:lint]commands} {[testenv:code-test]commands} - {[testenv:doc-build-test]commands} + {[testenv:doc-build]commands}
