This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4372a62d5bf Improve RAT licence check (#48119)
4372a62d5bf is described below
commit 4372a62d5bf22b9c2347beb914398dc8e37fbac9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Mar 23 17:52:58 2025 +0100
Improve RAT licence check (#48119)
RAT licence check had not been updated for 3 years and started to
show its age.
With this PR, we are upgrading RAT to latest version (0.16.1), we
use non-deprecated and fresh official eclipse-temurin OpenJDK image
as base and we build the image as multi-architecture image that
supports both ARM and AMD64.
We also fix the check to only be run when LICENSE file is changed in
the root of Airflow, which basically means that it should NEVER be
run locally unless you specify `--all-files` - this way it is
always run in CI but you need to use `--all-files` to trigger the
check locally.
---
.pre-commit-config.yaml | 3 +--
.rat-excludes | 1 +
scripts/ci/dockerfiles/apache-rat/Dockerfile | 21 ++++++++-------------
scripts/ci/dockerfiles/apache-rat/build_and_push.sh | 13 +++++++------
scripts/ci/pre_commit/check_license.py | 2 +-
5 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7f899ca2fd4..53314ec11e4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -755,8 +755,7 @@ repos:
name: Check if licenses are OK for Apache
entry: ./scripts/ci/pre_commit/check_license.py
language: python
- files: ^.*LICENSE.*$|^.*LICENCE.*$
- exclude: ^providers/.*/src/.*/LICENSE$|.*/dist/.*
+ files: ^LICENSE$
pass_filenames: false
- id: check-aiobotocore-optional
name: Check if aiobotocore is an optional dependency only
diff --git a/.rat-excludes b/.rat-excludes
index 2c730134a76..d9bc6ac0e0a 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -44,6 +44,7 @@ venv
clients/*
files/*
dags/*
+generated/*
.gitmodules
prod_image_installed_providers.txt
airflow_pre_installed_providers.txt
diff --git a/scripts/ci/dockerfiles/apache-rat/Dockerfile
b/scripts/ci/dockerfiles/apache-rat/Dockerfile
index d9d5be19071..2cdf1aa0a0d 100644
--- a/scripts/ci/dockerfiles/apache-rat/Dockerfile
+++ b/scripts/ci/dockerfiles/apache-rat/Dockerfile
@@ -15,11 +15,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-FROM ibmjava:sfj-alpine
+FROM eclipse-temurin:21-jre-ubi9-minimal
ARG AIRFLOW_APACHERAT_VERSION
ARG APACHERAT_VERSION
-ARG APACHE_MIRROR="https://www-us.apache.org/dist";
+ARG APACHE_DOWNLOAD="https://downloads.apache.org";
+ARG COMMIT_SHA
+ENV dist_name="apache-rat-${APACHERAT_VERSION}"
+ENV rat_tgz="${dist_name}-bin.tar.gz"
SHELL ["/bin/sh", "-e", "-x", "-c"]
@@ -29,21 +32,13 @@ SHELL ["/bin/sh", "-e", "-x", "-c"]
# Those are build deps only but still we want the latest versions of those
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
-RUN apk add -U --no-cache --virtual .build-deps \
- gnupg \
- unzip \
- && wget -qT 30 "${APACHE_MIRROR}/creadur/KEYS" \
- && gpg --import KEYS \
- && rm KEYS \
- && dist_name="apache-rat-${APACHERAT_VERSION}" \
- && rat_tgz="${dist_name}-bin.tar.gz" \
- && wget -qT 30 "${APACHE_MIRROR}/creadur/${dist_name}/${rat_tgz}.asc" \
- && wget -qT 30 "${APACHE_MIRROR}/creadur/${dist_name}/${rat_tgz}" \
+RUN wget -qT 30 "${APACHE_DOWNLOAD}/creadur/KEYS" && gpg --import KEYS && rm
KEYS \
+ && wget -qT 30 "${APACHE_DOWNLOAD}/creadur/${dist_name}/${rat_tgz}.asc" \
+ && wget -qT 30 "${APACHE_DOWNLOAD}/creadur/${dist_name}/${rat_tgz}" \
&& gpg --verify "${rat_tgz}.asc" "${rat_tgz}" \
&& tar --extract --gzip --file "${rat_tgz}" --strip-components=1
"${dist_name}/${dist_name}.jar" "${dist_name}/LICENSE" \
&& rm -vrf "${rat_tgz}" "${rat_tgz}.asc" \
&& ln -s "${dist_name}.jar" "apache-rat.jar" \
- && apk del .build-deps \
&& rm -rf ~/.gnupg
LABEL org.apache.airflow.component="apache-rat"
diff --git a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
index 228b3014566..ddd004d9337 100755
--- a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
+++ b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
@@ -19,10 +19,10 @@ set -euo pipefail
GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
readonly GITHUB_REPOSITORY
-APACHERAT_VERSION="0.13"
+APACHERAT_VERSION="0.16.1"
readonly APACHERAT_VERSION
-AIRFLOW_APACHERAT_VERSION="2021.07.04"
+AIRFLOW_APACHERAT_VERSION="2024.03.23"
readonly AIRFLOW_APACHERAT_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -33,12 +33,13 @@ cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
TAG="ghcr.io/${GITHUB_REPOSITORY}-apache-rat:${APACHERAT_VERSION}-${AIRFLOW_APACHERAT_VERSION}"
readonly TAG
-docker build . \
+docker buildx build . \
--pull \
+ --builder "airflow_cache" \
--build-arg "APACHERAT_VERSION=${APACHERAT_VERSION}" \
--build-arg "AIRFLOW_APACHERAT_VERSION=${AIRFLOW_APACHERAT_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
--label
"org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}"; \
- --tag "${TAG}"
-
-docker push "${TAG}"
+ --platform "linux/amd64,linux/arm64" \
+ --tag "${TAG}" \
+ --push
diff --git a/scripts/ci/pre_commit/check_license.py
b/scripts/ci/pre_commit/check_license.py
index c90bc262deb..f08b496d538 100755
--- a/scripts/ci/pre_commit/check_license.py
+++ b/scripts/ci/pre_commit/check_license.py
@@ -39,7 +39,7 @@ cmd = [
"--rm",
"--platform",
"linux/amd64",
- "ghcr.io/apache/airflow-apache-rat:0.13-2021.07.04",
+
"ghcr.io/apache/airflow-apache-rat:0.16.1-2024.03.23@sha256:83c4d2610ec4a439d1809a67fadbdc9a1df089ab130b32209351bdd4527a3f02",
"-d",
"/opt/airflow",
"--exclude-file",
