This is an automated email from the ASF dual-hosted git repository.
jorgecarleitao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new d97fc91 Fix RAT check (#652)
d97fc91 is described below
commit d97fc9145bfc41969bd30dee3d33337ac3932aa5
Author: Andrew Lamb <[email protected]>
AuthorDate: Fri Jul 2 23:28:33 2021 -0400
Fix RAT check (#652)
---
.github/workflows/dev.yml | 21 ++++++++++----
dev/release/check-rat-report.py | 59 +++++++++++++++++++++++++++++++++++++++
dev/release/rat_exclude_files.txt | 16 +++++++++++
dev/release/run-rat.sh | 43 ++++++++++++++++++++++++++++
4 files changed, 133 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index a7e574e..8bb35f1 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -16,15 +16,11 @@
# under the License.
name: Dev
-
-on:
- # always trigger
- push:
- pull_request:
+on: [push, pull_request]
jobs:
lint:
- name: Lint C++, Python, R, Rust, Docker, RAT
+ name: Lint C++, Python, R, Rust, Docker
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
@@ -42,6 +38,19 @@ jobs:
- name: Lint
run: archery lint --rat
+ rat:
+ name: Release Audit Tool (RAT)
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v1
+ with:
+ python-version: 3.8
+ - name: Audit licenses
+ run: ./dev/release/run-rat.sh .
+
prettier:
name: Use prettier to check formatting of documents
runs-on: ubuntu-latest
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
new file mode 100644
index 0000000..e30d72b
--- /dev/null
+++ b/dev/release/check-rat-report.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+import fnmatch
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) != 3:
+ sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
+ sys.argv[0])
+ sys.exit(1)
+
+exclude_globs_filename = sys.argv[1]
+xml_filename = sys.argv[2]
+
+globs = [line.strip() for line in open(exclude_globs_filename, "r")]
+
+tree = ET.parse(xml_filename)
+root = tree.getroot()
+resources = root.findall('resource')
+
+all_ok = True
+for r in resources:
+ approvals = r.findall('license-approval')
+ if not approvals or approvals[0].attrib['name'] == 'true':
+ continue
+ clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+ excluded = False
+ for g in globs:
+ if fnmatch.fnmatch(clean_name, g):
+ excluded = True
+ break
+ if not excluded:
+ sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
+ clean_name, r.attrib['name'], approvals[0].attrib['name']))
+ all_ok = False
+
+if not all_ok:
+ sys.exit(1)
+
+print('OK')
+sys.exit(0)
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index 96beccd..5a7d351 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -106,3 +106,19 @@ ballista/rust/scheduler/testdata/*
ballista/ui/scheduler/yarn.lock
python/rust-toolchain
python/requirements*.txt
+**/testdata/*
+benchmarks/queries/*
+benchmarks/data/*
+ci/*
+**/*.svg
+**/*.csv
+**/*.json
+venv/*
+testing/*
+target/*
+**/target/*
+Cargo.lock
+**/Cargo.lock
+.history
+parquet-testing/*
+*rat.txt
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
new file mode 100755
index 0000000..94fa55f
--- /dev/null
+++ b/dev/release/run-rat.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+RAT_VERSION=0.13
+
+# download apache rat
+if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
+ curl -s
https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar
> apache-rat-${RAT_VERSION}.jar
+fi
+
+RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
+
+RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+
+# generate the rat report
+$RAT $1 > rat.txt
+python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt
rat.txt > filtered_rat.txt
+cat filtered_rat.txt
+UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l`
+
+if [ "0" -eq "${UNAPPROVED}" ]; then
+ echo "No unapproved licenses"
+else
+ echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
+ exit 1
+fi
