This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 935fb3e2fd minor: simplify docs build process & pin pip package
versions (#17816)
935fb3e2fd is described below
commit 935fb3e2fd8cf3574af6b7ddecfece2c7e515f56
Author: Jeffrey Vo <[email protected]>
AuthorDate: Wed Oct 1 00:01:06 2025 +1000
minor: simplify docs build process & pin pip package versions (#17816)
---
.github/dependabot.yml | 5 +++++
docs/.gitignore | 5 +++--
docs/Makefile | 2 +-
docs/README.md | 20 ++++++++++++++++---
docs/build.sh | 9 +--------
docs/make.bat | 3 ++-
docs/requirements.txt | 12 ++++++------
docs/rustdoc_trim.py | 37 +++++++++++------------------------
docs/source/conf.py | 52 +++++++++++++++++++++++++-------------------------
9 files changed, 72 insertions(+), 73 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index ec3368da9a..9d1d77d44c 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -50,3 +50,8 @@ updates:
interval: "daily"
open-pull-requests-limit: 10
labels: [auto-dependencies]
+ - package-ecosystem: "pip"
+ directory: "/docs"
+ schedule:
+ interval: "weekly"
+ labels: [auto-dependencies]
diff --git a/docs/.gitignore b/docs/.gitignore
index e2a54c053e..a3adddc690 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-build
-temp
+temp/
+build/
venv/
.python-version
+__pycache__/
diff --git a/docs/Makefile b/docs/Makefile
index 6bce19911d..20ccd822f5 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -21,7 +21,7 @@
# You can set these variables from the command line, and also
# from the environment for the first two.
-SPHINXOPTS ?=
+SPHINXOPTS ?= -W
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
diff --git a/docs/README.md b/docs/README.md
index acf3cb754c..c3d87ee8e8 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -28,22 +28,36 @@ https://datafusion.apache.org/ as part of the release
process.
It's recommended to install build dependencies and build the documentation
inside a Python virtualenv.
-- Python
-- `pip install -r requirements.txt`
+```sh
+python3 -m venv venv
+pip install -r requirements.txt
+```
+
+If using [uv](https://docs.astral.sh/uv/) the script can be run like so without
+needing to create a virtual environment:
+
+```sh
+uv run --with-requirements requirements.txt bash build.sh
+```
## Build & Preview
Run the provided script to build the HTML pages.
```bash
+# If using venv, ensure you have activated it
./build.sh
```
-The HTML will be generated into a `build` directory.
+The HTML will be generated into a `build` directory. Open
`build/html/index.html`
+in your preferred browser, e.g.
Preview the site on Linux by running this command.
```bash
+# On macOS
+open build/html/index.html
+# On Linux with Firefox
firefox build/html/index.html
```
diff --git a/docs/build.sh b/docs/build.sh
index 73516e8e9c..9e4a118580 100755
--- a/docs/build.sh
+++ b/docs/build.sh
@@ -20,12 +20,5 @@
set -e
rm -rf build 2> /dev/null
-rm -rf temp 2> /dev/null
-mkdir temp
-cp -rf source/* temp/
-# replace relative URLs with absolute URLs
-sed -i -e
's/\.\.\/\.\.\/\.\.\//https:\/\/github.com\/apache\/arrow-datafusion\/blob\/main\//g'
temp/contributor-guide/index.md
-python rustdoc_trim.py
-
-make SOURCEDIR=`pwd`/temp SPHINXOPTS=-W html
+make html
diff --git a/docs/make.bat b/docs/make.bat
index ded5b4a3e2..33e25e4ee4 100644
--- a/docs/make.bat
+++ b/docs/make.bat
@@ -23,7 +23,8 @@ REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
-)
+)
+set SPHINXOPTS=-W
set SOURCEDIR=source
set BUILDDIR=build
diff --git a/docs/requirements.txt b/docs/requirements.txt
index b1cb76a3cc..2e3263c3b5 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -15,10 +15,10 @@
# specific language governing permissions and limitations
# under the License.
-sphinx
-sphinx-reredirects
+sphinx==8.2.3
+sphinx-reredirects==1.0.0
pydata-sphinx-theme==0.8.0
-myst-parser
-maturin
-jinja2
-setuptools>=48.0.0
+myst-parser==4.0.1
+maturin==1.9.4
+jinja2==3.1.6
+setuptools==80.9.0
diff --git a/docs/rustdoc_trim.py b/docs/rustdoc_trim.py
index 7ea96dbb44..70becc45ee 100644
--- a/docs/rustdoc_trim.py
+++ b/docs/rustdoc_trim.py
@@ -16,8 +16,7 @@
# under the License.
import re
-
-from pathlib import Path
+from sphinx.application import Sphinx
# Regex pattern to match Rust code blocks in Markdown
RUST_CODE_BLOCK_PATTERN = re.compile(r"```rust\s*(.*?)```", re.DOTALL)
@@ -46,30 +45,16 @@ def remove_hashtag_lines_in_rust_blocks(markdown_content):
return RUST_CODE_BLOCK_PATTERN.sub(_process_code_block, markdown_content)
-# Example usage
-def process_markdown_file(file_path):
- # Read the Markdown file
- with open(file_path, "r", encoding="utf-8") as file:
- markdown_content = file.read()
-
+def process_source_file(app: Sphinx, docname: str, source: list[str]):
+ original_content = source[0]
# Remove lines starting with '#' in Rust code blocks
- updated_markdown_content =
remove_hashtag_lines_in_rust_blocks(markdown_content)
-
- # Write the updated content back to the Markdown file
- with open(file_path, "w", encoding="utf-8") as file:
- file.write(updated_markdown_content)
-
- print(f"Done processing file: {file_path}")
-
-
-root_directory = Path("./temp/library-user-guide")
-for file_path in root_directory.rglob("*.md"):
- print(f"Processing file: {file_path}")
- process_markdown_file(file_path)
+ modified_content = remove_hashtag_lines_in_rust_blocks(original_content)
+ source[0] = modified_content
-root_directory = Path("./temp/user-guide")
-for file_path in root_directory.rglob("*.md"):
- print(f"Processing file: {file_path}")
- process_markdown_file(file_path)
-print("All Markdown files processed.")
+def setup(app: Sphinx):
+ app.connect("source-read", process_source_file)
+ return dict(
+ parallel_read_safe=True,
+ parallel_write_safe=True,
+ )
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5e31864e9a..36556e74e6 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -26,16 +26,17 @@
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+
+# To pickup rustdoc_trim.py
+sys.path.insert(0, os.path.abspath(".."))
# -- Project information -----------------------------------------------------
-project = 'Apache DataFusion'
-copyright = '2019-2025, Apache Software Foundation'
-author = 'Apache Software Foundation'
+project = "Apache DataFusion"
+copyright = "2019-2025, Apache Software Foundation"
+author = "Apache Software Foundation"
# -- General configuration ---------------------------------------------------
@@ -44,24 +45,25 @@ author = 'Apache Software Foundation'
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx.ext.autosummary',
- 'sphinx.ext.doctest',
- 'sphinx.ext.ifconfig',
- 'sphinx.ext.mathjax',
- 'sphinx.ext.viewcode',
- 'sphinx.ext.napoleon',
- 'myst_parser',
- 'sphinx_reredirects',
+ "sphinx.ext.autodoc",
+ "sphinx.ext.autosummary",
+ "sphinx.ext.doctest",
+ "sphinx.ext.ifconfig",
+ "sphinx.ext.mathjax",
+ "sphinx.ext.viewcode",
+ "sphinx.ext.napoleon",
+ "myst_parser",
+ "sphinx_reredirects",
+ "rustdoc_trim",
]
source_suffix = {
- '.rst': 'restructuredtext',
- '.md': 'markdown',
+ ".rst": "restructuredtext",
+ ".md": "markdown",
}
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
@@ -83,7 +85,7 @@ autosummary_generate = True
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = 'pydata_sphinx_theme'
+html_theme = "pydata_sphinx_theme"
html_theme_options = {
"use_edit_page_button": True,
@@ -99,13 +101,11 @@ html_context = {
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
html_logo = "_static/images/2x_bgwhite_original.png"
-html_css_files = [
- "theme_overrides.css"
-]
+html_css_files = ["theme_overrides.css"]
html_sidebars = {
"**": ["docs-sidebar.html"],
@@ -121,9 +121,9 @@ myst_enable_extensions = ["colon_fence", "deflist",
"tasklist"]
# presence of some special characters like: 🚀, å, {,... But this isn’t a major
# issue for our documentation. So, suppress these warnings to keep our build
# log cleaner.
-suppress_warnings = ['misc.highlighting_failure']
+suppress_warnings = ["misc.highlighting_failure"]
redirects = {
"library-user-guide/adding-udfs": "functions/index.html",
"user-guide/runtime_configs": "configs.html",
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
