This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch lh-fix-sync-content in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
commit d931aad291351e6841d255a232ec72cb1cd08dd7 Author: Lari Hotari <[email protected]> AuthorDate: Mon May 18 10:32:05 2026 +0300 Fix sync-content workflow for apache/pulsar gradle build apache/pulsar master switched to a Gradle build, so the previous Maven install in the sync-content composite action no longer produces the jars, classpath, and bin scripts that the docs generators need. Maintenance branches still ship Maven, and the same Python entry points are reused for release-time doc generation there. - Add a pulsar_build helper that detects gradle vs maven from the pulsar checkout layout, exposes the build-system-specific classpath and swagger output dirs, and runs gradle assemble + :distribution:pulsar-{server,shell}-distribution:exportClasspath when the artifacts are missing on a gradle checkout. - Route all five reference/CLI doc generators and swagger_generator through the detector so the same scripts work on master (gradle) and maintenance branches (maven, unchanged). - swagger_generator skips with a warning on gradle since apache/pulsar master has no swagger task yet; the maven mvn -Pswagger fallback is preserved for maintenance branches. - Make site_uploader.execute()'s head_sha optional. site-updater.py never passed it after the .publish-ref change, which left the sync workflow latently broken; head_sha=None now skips the .publish-ref marker that is only meaningful for the asf-site-next branch. - Rewrite .github/actions/sync-content/action.yml to use gradle/actions/setup-gradle and ./gradlew assemble exportClasspath in place of the old mvn install step. - Drop PULSARBOT_TOKEN from .github/workflows/ci-sync-content.yml and grant the default GITHUB_TOKEN contents:write so the workflow can push to main. --- .github/actions/sync-content/action.yml | 27 ++++---- .github/workflows/ci-sync-content.yml | 4 +- tools/pytools/lib/execute/config_doc_generator.py | 7 +- .../lib/execute/pulsar_admin_clidoc_generator.py | 4 ++ tools/pytools/lib/execute/pulsar_build.py | 81 ++++++++++++++++++++++ .../pytools/lib/execute/pulsar_clidoc_generator.py | 4 ++ .../lib/execute/pulsar_client_clidoc_generator.py | 4 ++ .../lib/execute/pulsar_perf_clidoc_generator.py | 4 ++ tools/pytools/lib/execute/site_uploader.py | 11 ++- tools/pytools/lib/execute/swagger_generator.py | 23 ++++-- 10 files changed, 143 insertions(+), 26 deletions(-) diff --git a/.github/actions/sync-content/action.yml b/.github/actions/sync-content/action.yml index 7f02101a01e..0c1b602c771 100644 --- a/.github/actions/sync-content/action.yml +++ b/.github/actions/sync-content/action.yml @@ -31,27 +31,24 @@ runs: with: python-version: '3.12' cache: 'poetry' - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 21 uses: actions/setup-java@v5 with: distribution: corretto java-version: 21 - - name: Run install by skip tests + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v5 + with: + build-root-directory: tmp/pulsar + - name: Build pulsar artifacts and export classpath working-directory: tmp/pulsar - env: - MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 - run: mvn -B -ntp install -Pcore-modules,swagger,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true + run: | + ./gradlew \ + assemble \ + :distribution:pulsar-server-distribution:exportClasspath \ + :distribution:pulsar-shell-distribution:exportClasspath \ + --no-configuration-cache \ + --no-daemon shell: bash - name: Update generated docs working-directory: tools/pytools diff --git a/.github/workflows/ci-sync-content.yml b/.github/workflows/ci-sync-content.yml index cf04526e6d6..d53bf3616b7 100644 --- a/.github/workflows/ci-sync-content.yml +++ b/.github/workflows/ci-sync-content.yml @@ -27,9 +27,9 @@ jobs: name: Synchronize Content from Main Repo runs-on: ubuntu-latest timeout-minutes: 180 + permissions: + contents: write steps: - uses: actions/checkout@v6 - with: - token: ${{ secrets.PULSARBOT_TOKEN }} - name: Sync content and push uses: ./.github/actions/sync-content diff --git a/tools/pytools/lib/execute/config_doc_generator.py b/tools/pytools/lib/execute/config_doc_generator.py index d34497af455..a83ef5de7ba 100644 --- a/tools/pytools/lib/execute/config_doc_generator.py +++ b/tools/pytools/lib/execute/config_doc_generator.py @@ -22,6 +22,7 @@ from pathlib import Path import semver from command import find_command, run from constant import site_path +from execute import pulsar_build @dataclass @@ -35,9 +36,11 @@ class Settings: def execute(master: Path, version: str): java = find_command('java', msg='java is required') + build = pulsar_build.detect(master) + pulsar_build.ensure_built(master, build) + reference = site_path() / 'static' / 'reference' / version - classpath = master / 'distribution' / 'server' / 'target' / 'classpath.txt' - classpath = classpath.read_text() + classpath = pulsar_build.server_classpath_file(master, build).read_text() broker_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' client_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' diff --git a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py index ba61cf99498..533919e7c53 100644 --- a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py @@ -20,9 +20,13 @@ from pathlib import Path from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + admin = basedir / 'bin' / 'pulsar-admin' reference = site_path() / 'static' / 'reference' / version / 'pulsar-admin' diff --git a/tools/pytools/lib/execute/pulsar_build.py b/tools/pytools/lib/execute/pulsar_build.py new file mode 100644 index 00000000000..24c1f6f90f6 --- /dev/null +++ b/tools/pytools/lib/execute/pulsar_build.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import enum +from pathlib import Path + +from command import run + + +class BuildSystem(enum.Enum): + gradle = 'gradle' + maven = 'maven' + + +# Apache Pulsar master switched to a Gradle build (no more `pom.xml` at the +# repo root, classpath/jars land under `build/`). Maintenance branches keep +# the legacy Maven layout (`pom.xml`, `target/`). Detect once per call site so +# the same generator code path works for both checkouts. +def detect(master: Path) -> BuildSystem: + if (master / 'gradlew').exists() and ( + (master / 'build.gradle.kts').exists() or (master / 'build.gradle').exists() + ): + return BuildSystem.gradle + if (master / 'pom.xml').exists(): + return BuildSystem.maven + raise RuntimeError( + f'Cannot determine pulsar build system in {master}: ' + f'no gradlew/build.gradle(.kts) or pom.xml found.' + ) + + +def server_classpath_file(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'distribution' / 'server' / 'build' / 'classpath.txt' + return master / 'distribution' / 'server' / 'target' / 'classpath.txt' + + +def swagger_output_dir(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'pulsar-broker' / 'build' / 'docs' + return master / 'pulsar-broker' / 'target' / 'docs' + + +# Pre-build the artifacts needed by the reference/CLI doc generators when the +# checkout uses Gradle. Maven branches are still primed by the GitHub Action +# step that runs `mvn install`, so this is a no-op there. +def ensure_built(master: Path, build: BuildSystem) -> None: + if build != BuildSystem.gradle: + return + + classpath_file = server_classpath_file(master, build) + if classpath_file.exists(): + return + + gradlew = master / 'gradlew' + if not gradlew.exists(): + raise RuntimeError(f'gradlew not found at {gradlew}') + + run( + str(gradlew.absolute()), + 'assemble', + ':distribution:pulsar-server-distribution:exportClasspath', + ':distribution:pulsar-shell-distribution:exportClasspath', + '--no-configuration-cache', + '--no-daemon', + cwd=master, + ) diff --git a/tools/pytools/lib/execute/pulsar_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_clidoc_generator.py index e76c4bc9fb6..74efa5f3e43 100644 --- a/tools/pytools/lib/execute/pulsar_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_clidoc_generator.py @@ -20,9 +20,13 @@ from pathlib import Path from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + pulsar = basedir / 'bin' / 'pulsar' reference = site_path() / 'static' / 'reference' / version / 'pulsar' diff --git a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py index c1621f829db..10df4513495 100644 --- a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py @@ -20,9 +20,13 @@ from pathlib import Path from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + client = basedir / 'bin' / 'pulsar-client' reference = site_path() / 'static' / 'reference' / version / 'pulsar-client' diff --git a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py index 15f25b615db..f85824ff767 100644 --- a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py @@ -20,9 +20,13 @@ from pathlib import Path from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + perf = basedir / 'bin' / 'pulsar-perf' reference = site_path() / 'static' / 'reference' / version / 'pulsar-perf' diff --git a/tools/pytools/lib/execute/site_uploader.py b/tools/pytools/lib/execute/site_uploader.py index 692e44cd19e..0ce733b3120 100755 --- a/tools/pytools/lib/execute/site_uploader.py +++ b/tools/pytools/lib/execute/site_uploader.py @@ -21,6 +21,7 @@ import enum import os import tempfile from pathlib import Path +from typing import Optional from command import run, find_command, run_pipe @@ -44,7 +45,7 @@ def _should_push(mode: Mode) -> bool: return result -def _do_push(msg: str, site: Path, branch: str, head_sha: str): +def _do_push(msg: str, site: Path, branch: str, head_sha: Optional[str]): git = find_command('git', msg="git is required") # Persist the source-repo SHA we just published so the next run can compute @@ -52,7 +53,11 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): # it lands in the same commit as the published content. If `git push` later # fails, this local file is discarded along with the unpushed commit — the # next CI run re-clones asf-site-next and reads the previous .publish-ref. - (site / '.publish-ref').write_text(head_sha + '\n') + # Only relevant when publishing to the build output branch (asf-site-next); + # callers that push generated content into other branches (e.g. site-updater + # syncing into `main`) pass head_sha=None to skip the marker. + if head_sha is not None: + (site / '.publish-ref').write_text(head_sha + '\n') run(git, 'add', '-A', '.', cwd=site) changed = run(git, 'diff-index', '--quiet', 'HEAD', codes={0, 1}, cwd=site).returncode @@ -75,7 +80,7 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): run(git, 'push', 'origin', branch, cwd=site) -def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: str): +def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: Optional[str] = None): if _should_push(mode): _do_push(msg, site, branch, head_sha) else: # show changes diff --git a/tools/pytools/lib/execute/swagger_generator.py b/tools/pytools/lib/execute/swagger_generator.py index b813c80dc5e..6ec002bfd57 100644 --- a/tools/pytools/lib/execute/swagger_generator.py +++ b/tools/pytools/lib/execute/swagger_generator.py @@ -17,18 +17,33 @@ import json import os +import sys from pathlib import Path from command import find_command, run from constant import site_path +from execute import pulsar_build def execute(master: Path, version: str): - master_swaggers = master / 'pulsar-broker' / 'target' / 'docs' + build = pulsar_build.detect(master) + master_swaggers = pulsar_build.swagger_output_dir(master, build) - if not master_swaggers.exists(): # generate master swaggers - mvn = find_command('mvn', msg="mvn is required") - run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + if not master_swaggers.exists(): + if build == pulsar_build.BuildSystem.maven: + mvn = find_command('mvn', msg="mvn is required") + run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + else: + # Gradle build on apache/pulsar master does not yet have a task + # that regenerates the Swagger JSONs (the old `mvn -Pswagger` + # invocation has no Gradle equivalent). Skip rather than fail so + # the rest of the docs sync still produces useful output. + print( + f'[swagger_generator] Skipping Swagger generation: Gradle build at {master} ' + f'has no swagger task; expected output dir {master_swaggers} is missing.', + file=sys.stderr, + ) + return os.makedirs(site_path() / 'static' / 'swagger' / version, exist_ok=True) for f in master_swaggers.glob('*.json'):
