This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fdec8f86b3 HDDS-10273. Intermittent build failure while downloading
nodejs (#6664)
fdec8f86b3 is described below
commit fdec8f86b3735d62ca2de53cdf21c5f4cf96c096
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue May 14 08:26:20 2024 +0200
HDDS-10273. Intermittent build failure while downloading nodejs (#6664)
---
.github/workflows/populate-cache.yml | 26 ++++++++++----
dev-support/ci/populate-cache.sh | 70 ++++++++++++++++++++++++++++++++++++
hadoop-ozone/recon/pom.xml | 2 +-
pom.xml | 3 ++
4 files changed, 94 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/populate-cache.yml
b/.github/workflows/populate-cache.yml
index d4c9cd8120..787410c7a1 100644
--- a/.github/workflows/populate-cache.yml
+++ b/.github/workflows/populate-cache.yml
@@ -28,6 +28,7 @@ on:
- '.github/workflows/populate-cache.yml'
schedule:
- cron: '20 3 * * *'
+ workflow_dispatch:
jobs:
build:
@@ -45,27 +46,40 @@ jobs:
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ - name: Check for partial cache
+ id: verify-cache
+ run: |
+ cache_ok='true'
+ if [[ '${{ steps.restore-cache.outputs.cache-hit }}' != 'true' ]];
then
+ cache_ok='false'
+ elif [[ -e ~/.m2/repository/.cache/org-apache-ozone/partial-cache
]]; then
+ cache_ok='false'
+ fi
+ echo "cache-ok=$cache_ok" >> $GITHUB_OUTPUT
+
- name: Setup Java
- if: steps.restore-cache.outputs.cache-hit != 'true'
+ if: steps.verify-cache.outputs.cache-ok != 'true'
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 8
- name: Fetch dependencies
- if: steps.restore-cache.outputs.cache-hit != 'true'
- run: mvn --batch-mode --fail-never --no-transfer-progress
--show-version -Pgo-offline -Pdist clean verify
+ if: steps.verify-cache.outputs.cache-ok != 'true'
+ run: |
+ mkdir -p ~/.m2/repository
+ dev-support/ci/populate-cache.sh
+ continue-on-error: true
- name: Delete Ozone jars from repo
- if: steps.restore-cache.outputs.cache-hit != 'true'
+ if: steps.verify-cache.outputs.cache-ok != 'true'
run: rm -fr ~/.m2/repository/org/apache/ozone
- name: List repo contents
- if: steps.restore-cache.outputs.cache-hit != 'true'
run: find ~/.m2/repository -type f | sort | xargs ls -lh
- name: Save cache for Maven dependencies
- if: steps.restore-cache.outputs.cache-hit != 'true'
+ if: steps.verify-cache.outputs.cache-ok != 'true'
uses: actions/cache/save@v4
with:
path: |
diff --git a/dev-support/ci/populate-cache.sh b/dev-support/ci/populate-cache.sh
new file mode 100755
index 0000000000..107c1d2e13
--- /dev/null
+++ b/dev-support/ci/populate-cache.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# This script gets a list of the changed files from git, and matches
+# that list against a set of regexes, each of which corresponds to a
+# test group. For each set of matches, a flag is set to let github
+# actions know to run that test group.
+
+# Populates local Maven repository (similar to go-offline, but using actual
build).
+# Downloads Node.js for multiple platforms (Linux, Mac)
+
+set -u -o pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "${DIR}/../.." || exit 1
+
+: ${REPO_DIR:="${HOME}/.m2/repository"}
+: ${MARKER_FILE:="${REPO_DIR}/.cache/org-apache-ozone/partial-cache"}
+
+declare -i rc=0
+
+if [[ ! -d "${REPO_DIR}" ]]; then
+ echo "Error: repository dir ${REPO_DIR} does not exist"
+ exit 1
+fi
+
+# Download Node.js manually
+NODEJS_VERSION=$(mvn help:evaluate -Dexpression=nodejs.version -q
-DforceStdout)
+if [[ -n "${NODEJS_VERSION}" ]]; then
+ for platform in darwin-x64 linux-x64; do
+
output="${REPO_DIR}/com/github/eirslett/node/${NODEJS_VERSION}/node-${NODEJS_VERSION}-${platform}.tar.gz"
+
url="https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-${platform}.tar.gz";
+ mkdir -pv "$(dirname "${output}")"
+ curl --retry 3 --location --continue-at - -o "${output}" "${url}"
+ irc=$?
+
+ if [[ ${rc} -eq 0 ]]; then
+ rc=${irc}
+ fi
+ done
+fi
+
+MAVEN_OPTIONS="--batch-mode --fail-at-end --no-transfer-progress
--show-version -Pgo-offline -Pdist"
+if ! mvn ${MAVEN_OPTIONS} clean verify; then
+ rc=1
+fi
+
+if [[ ${rc} -eq 0 ]]; then
+ rm -fv "${MARKER_FILE}"
+else
+ # Create Marker file if build fails to indicate cache is partial, needs to
be rebuilt next time
+ mkdir -pv "$(dirname "${MARKER_FILE}")"
+ touch "${MARKER_FILE}"
+fi
+
+exit ${rc}
diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml
index 2895840aaf..917a691c54 100644
--- a/hadoop-ozone/recon/pom.xml
+++ b/hadoop-ozone/recon/pom.xml
@@ -103,7 +103,7 @@
<goal>install-node-and-npm</goal>
</goals>
<configuration>
- <nodeVersion>v16.14.2</nodeVersion>
+ <nodeVersion>v${nodejs.version}</nodeVersion>
</configuration>
</execution>
<execution>
diff --git a/pom.xml b/pom.xml
index e0f97d5f6d..e7a976f88f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -295,6 +295,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<spring.version>5.3.34</spring.version>
<jooq.version>3.11.10</jooq.version>
+ <!-- Node.js version number (without the v prefix required by
frontend-maven-plugin) -->
+ <nodejs.version>16.14.2</nodejs.version>
+
<vault.driver.version>5.1.0</vault.driver.version>
<native.lib.tmp.dir></native.lib.tmp.dir>
<properties.maven.plugin.version>1.2.1</properties.maven.plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
