This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new 0fa32b63a [Chore] Add dependency license check (#3878)
0fa32b63a is described below
commit 0fa32b63a1527ba5a9c65d79d47aceeab55a5187
Author: xiangzihao <[email protected]>
AuthorDate: Wed Jul 17 17:07:04 2024 +0800
[Chore] Add dependency license check (#3878)
* add dependency license check
---
.github/workflows/backend.yml | 4 +-
.gitignore | 1 +
.licenserc.yaml | 1 +
tools/dependencies/check-LICENSE.sh | 51 ++++
tools/dependencies/known-dependencies.txt | 413 ++++++++++++++++++++++++++++++
5 files changed, 469 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 09b3b126a..ef1a83a6e 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -96,7 +96,9 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
restore-keys: ${{ runner.os }}-maven-
- name: Backend Build with Maven
- run: ./mvnw -B clean install -Pshaded -DskipTests
+ run: ./mvnw -B clean install -Pshaded,webapp,dist -DskipTests
+ - name: Check dependency license
+ run: tools/dependencies/check-LICENSE.sh
result:
name: Backend - Result
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index 22572595b..a460e3033 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,3 +45,4 @@ hs_err_pid*
*Spec-output/
**/node
dist/
+dist-license-check/
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 1c6652248..5949df8a5 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -81,6 +81,7 @@ header:
- 'streampark-console/streampark-console-webapp/src/views/base/exception'
- 'streampark-console/streampark-console-webapp/src/views/base/lock'
- 'streampark-console/streampark-console-webapp/src/views/base/redirect'
+ - 'tools/dependencies/known-dependencies.txt'
comment: on-failure
diff --git a/tools/dependencies/check-LICENSE.sh
b/tools/dependencies/check-LICENSE.sh
new file mode 100755
index 000000000..58ca164db
--- /dev/null
+++ b/tools/dependencies/check-LICENSE.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+check_path=dist-license-check
+
+if [ -d "$check_path" ];then
+rm -rf $check_path
+fi
+mkdir $check_path || true
+
+tar -zxf dist/apache-streampark*-bin.tar.gz --strip=1 -C $check_path
+
+# List all modules(jars) that belong to the streampark itself, these will be
ignored when checking the dependency
+# licenses
+echo '=== Self modules: ' && ./mvnw --batch-mode --quiet
-Dexec.executable='echo'
-Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee
$check_path/self-modules.txt
+
+echo '=== dist-license-checkributed dependencies: ' && find dist-license-check
-name "*.jar" -exec basename {} \; | sort | uniq | tee
$check_path/all-dependencies.txt
+
+# Exclude all self modules(jars) to generate all third-party dependencies
+echo '=== Third party dependencies: ' && grep -vf $check_path/self-modules.txt
$check_path/all-dependencies.txt | sort | uniq | tee
$check_path/third-party-dependencies.txt
+
+# 1. Compare the third-party dependencies with known dependencies, expect that
all third-party dependencies are KNOWN
+# and the exit code of the command is 0, otherwise we should add its license
to LICENSE file
+# add the dependency to known-dependencies.txt.
+#
+# 2. Unify the `sort` behaviour: here we'll sort them again in case that the
behaviour of `sort` command in
+# target OS is different from what we used to sort the file
`known-dependencies.txt`, i.e. "sort the two file
+# using the same command (and default arguments)"
+
+diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort <
$check_path/third-party-dependencies.txt)
+
+if [ $? -ne 0 ]; then
+ echo "Third-party dependencies are not all known, please add the license to
LICENSE file and add the dependency to
tools/dependencies/known-dependencies.txt"
+ exit 1
+fi
diff --git a/tools/dependencies/known-dependencies.txt
b/tools/dependencies/known-dependencies.txt
new file mode 100644
index 000000000..056652a8c
--- /dev/null
+++ b/tools/dependencies/known-dependencies.txt
@@ -0,0 +1,413 @@
+HikariCP-3.4.5.jar
+JavaEWAH-1.1.13.jar
+RoaringBitmap-0.9.0.jar
+accessors-smart-2.4.9.jar
+activation-1.1.jar
+aether-api-1.1.0.jar
+aether-connector-basic-1.1.0.jar
+aether-impl-1.1.0.jar
+aether-spi-1.1.0.jar
+aether-transport-file-1.1.0.jar
+aether-transport-http-1.1.0.jar
+aether-util-1.1.0.jar
+aircompressor-0.10.jar
+annotations-13.0.jar
+antlr4-runtime-4.8-1.jar
+aopalliance-repackaged-2.6.1.jar
+arrow-format-2.0.0.jar
+arrow-memory-core-2.0.0.jar
+arrow-memory-netty-2.0.0.jar
+arrow-vector-2.0.0.jar
+asm-8.0.jar
+asm-analysis-8.0.jar
+asm-commons-8.0.jar
+asm-tree-8.0.jar
+asm-util-8.0.jar
+aspectjweaver-1.9.7.jar
+audience-annotations-0.5.0.jar
+avro-1.8.2.jar
+avro-ipc-1.8.2.jar
+avro-mapred-1.8.2-hadoop2.jar
+bcpkix-jdk18on-1.76.jar
+bcprov-jdk18on-1.76.jar
+bcutil-jdk18on-1.76.jar
+buji-pac4j-5.0.1.jar
+caffeine-2.8.6.jar
+checker-qual-3.5.0.jar
+chill-java-0.9.5.jar
+chill_2.12-0.9.5.jar
+classgraph-4.8.147.jar
+commons-beanutils-1.9.3.jar
+commons-cli-1.5.0.jar
+commons-codec-1.15.jar
+commons-collections-3.2.2.jar
+commons-compiler-3.1.9.jar
+commons-compress-1.21.jar
+commons-configuration2-2.1.1.jar
+commons-crypto-1.1.0.jar
+commons-email-1.5.jar
+commons-io-2.6.jar
+commons-lang-2.6.jar
+commons-lang3-3.8.1.jar
+commons-logging-1.1.3.jar
+commons-math3-3.4.1.jar
+commons-net-3.9.0.jar
+commons-text-1.6.jar
+compress-lzf-1.0.3.jar
+config-1.4.2.jar
+content-type-2.1.jar
+curator-client-2.12.0.jar
+curator-framework-2.13.0.jar
+curator-recipes-2.13.0.jar
+dnsjava-2.1.7.jar
+docker-java-api-3.3.6.jar
+docker-java-core-3.3.6.jar
+docker-java-transport-3.3.6.jar
+docker-java-transport-httpclient5-3.3.6.jar
+encoder-1.2.3.jar
+enumeratum-macros_2.12-1.6.1.jar
+enumeratum_2.12-1.6.1.jar
+failureaccess-1.0.1.jar
+flatbuffers-java-1.9.0.jar
+flink-annotations-1.14.4.jar
+flink-clients_2.12-1.14.4.jar
+flink-connector-base-1.14.4.jar
+flink-connector-files-1.14.4.jar
+flink-core-1.14.4.jar
+flink-file-sink-common-1.14.4.jar
+flink-hadoop-fs-1.14.4.jar
+flink-java-1.14.4.jar
+flink-kubernetes_2.12-1.14.4.jar
+flink-metrics-core-1.14.4.jar
+flink-optimizer-1.14.4.jar
+flink-rpc-akka-loader-1.14.4.jar
+flink-rpc-core-1.14.4.jar
+flink-runtime-1.14.4.jar
+flink-scala_2.12-1.14.4.jar
+flink-shaded-asm-7-7.1-14.0.jar
+flink-shaded-force-shading-14.0.jar
+flink-shaded-guava-30.1.1-jre-14.0.jar
+flink-shaded-jackson-2.12.4-14.0.jar
+flink-shaded-netty-4.1.65.Final-14.0.jar
+flink-streaming-java_2.12-1.14.4.jar
+flink-table-common-1.14.4.jar
+freemarker-2.3.30.jar
+gson-2.9.1.jar
+gson-fire-1.8.5.jar
+guava-30.0-jre.jar
+h2-2.1.214.jar
+hadoop-annotations-3.2.0.jar
+hadoop-auth-3.2.0.jar
+hadoop-client-3.2.0.jar
+hadoop-client-api-3.3.4.jar
+hadoop-client-runtime-3.3.4.jar
+hadoop-common-3.2.0.jar
+hadoop-hdfs-client-3.2.0.jar
+hadoop-mapreduce-client-common-3.2.0.jar
+hadoop-mapreduce-client-core-3.2.0.jar
+hadoop-mapreduce-client-jobclient-3.2.0.jar
+hadoop-yarn-api-3.2.0.jar
+hadoop-yarn-client-3.2.0.jar
+hadoop-yarn-common-3.2.0.jar
+hive-storage-api-2.7.2.jar
+hk2-api-2.6.1.jar
+hk2-locator-2.6.1.jar
+hk2-utils-2.6.1.jar
+htrace-core4-4.1.0-incubating.jar
+httpclient-4.5.14.jar
+httpclient5-5.1.jar
+httpclient5-fluent-5.1.jar
+httpcore-4.4.16.jar
+httpcore5-5.1.5.jar
+httpcore5-h2-5.1.5.jar
+ivy-2.4.0.jar
+j2objc-annotations-1.3.jar
+jackson-annotations-2.13.5.jar
+jackson-core-2.13.5.jar
+jackson-core-asl-1.9.13.jar
+jackson-databind-2.13.5.jar
+jackson-databind-nullable-0.2.4.jar
+jackson-dataformat-yaml-2.13.5.jar
+jackson-datatype-jdk8-2.13.5.jar
+jackson-datatype-jsr310-2.13.5.jar
+jackson-jaxrs-base-2.13.5.jar
+jackson-jaxrs-json-provider-2.13.5.jar
+jackson-mapper-asl-1.9.13.jar
+jackson-module-jaxb-annotations-2.13.5.jar
+jackson-module-parameter-names-2.13.5.jar
+jackson-module-scala_2.12-2.13.5.jar
+jakarta.activation-api-1.2.2.jar
+jakarta.annotation-api-1.3.5.jar
+jakarta.inject-2.6.1.jar
+jakarta.servlet-api-4.0.4.jar
+jakarta.validation-api-2.0.2.jar
+jakarta.websocket-api-1.1.2.jar
+jakarta.ws.rs-api-2.1.6.jar
+jakarta.xml.bind-api-2.3.3.jar
+janino-3.1.9.jar
+java-jwt-4.0.0.jar
+javassist-3.24.0-GA.jar
+javax.activation-api-1.2.0.jar
+javax.mail-1.5.6.jar
+javax.ws.rs-api-2.1.1.jar
+jaxb-api-2.3.1.jar
+jboss-logging-3.4.3.Final.jar
+jboss-threads-3.1.0.Final.jar
+jcip-annotations-1.0-1.jar
+jcl-over-slf4j-1.7.36.jar
+jdependency-2.4.0.jar
+jersey-client-2.35.jar
+jersey-common-2.35.jar
+jersey-container-servlet-2.35.jar
+jersey-container-servlet-core-2.35.jar
+jersey-hk2-2.35.jar
+jersey-server-2.35.jar
+jetty-servlet-9.4.51.v20230217.jar
+jetty-util-9.4.51.v20230217.jar
+jetty-util-ajax-9.4.51.v20230217.jar
+jna-5.13.0.jar
+jsch-0.2.11.jar
+json-smart-2.4.10.jar
+json4s-ast_2.12-4.0.6.jar
+json4s-core_2.12-4.0.6.jar
+json4s-jackson-core_2.12-4.0.6.jar
+json4s-jackson_2.12-4.0.6.jar
+json4s-scalap_2.12-4.0.6.jar
+jsp-api-2.1.jar
+jsqlparser-4.4.jar
+jsr305-3.0.2.jar
+jsr311-api-1.1.1.jar
+jul-to-slf4j-1.7.36.jar
+kafka-clients-3.1.2.jar
+kerb-admin-1.0.1.jar
+kerb-client-1.0.1.jar
+kerb-common-1.0.1.jar
+kerb-core-1.0.1.jar
+kerb-crypto-1.0.1.jar
+kerb-identity-1.0.1.jar
+kerb-server-1.0.1.jar
+kerb-simplekdc-1.0.1.jar
+kerb-util-1.0.1.jar
+kerby-asn1-1.0.1.jar
+kerby-config-1.0.1.jar
+kerby-pkix-1.0.1.jar
+kerby-util-1.0.1.jar
+kerby-xdr-1.0.1.jar
+knife4j-core-4.0.0.jar
+knife4j-openapi3-spring-boot-starter-4.0.0.jar
+knife4j-openapi3-ui-4.0.0.jar
+kotlin-stdlib-1.6.21.jar
+kotlin-stdlib-common-1.6.21.jar
+kotlin-stdlib-jdk7-1.6.21.jar
+kotlin-stdlib-jdk8-1.6.21.jar
+kryo-2.24.0.jar
+kryo-shaded-4.0.2.jar
+lang-tag-1.4.4.jar
+leveldbjni-all-1.8.jar
+listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
+log4j-1.2.17.jar
+log4j-api-2.17.2.jar
+log4j-over-slf4j-1.7.32.jar
+log4j-to-slf4j-2.17.2.jar
+logback-classic-1.2.11.jar
+logback-core-1.2.12.jar
+logging-interceptor-4.9.3.jar
+lz4-java-1.7.1.jar
+mail-1.4.7.jar
+maven-aether-provider-3.3.9.jar
+maven-artifact-3.0.jar
+maven-artifact-transfer-0.12.0.jar
+maven-builder-support-3.3.9.jar
+maven-common-artifact-filters-3.0.1.jar
+maven-core-3.0.jar
+maven-dependency-tree-3.0.1.jar
+maven-model-3.0.jar
+maven-model-builder-3.3.9.jar
+maven-plugin-api-3.0.jar
+maven-repository-metadata-3.3.9.jar
+maven-settings-3.0.jar
+maven-settings-builder-3.0.jar
+maven-shade-plugin-3.2.4.jar
+maven-shared-utils-3.1.0.jar
+maven-wrapper.jar
+metrics-core-4.2.18.jar
+metrics-graphite-4.2.18.jar
+metrics-jmx-4.2.18.jar
+metrics-json-4.2.18.jar
+metrics-jvm-4.2.18.jar
+minlog-1.2.jar
+minlog-1.3.0.jar
+mybatis-3.5.10.jar
+mybatis-plus-3.5.3.1.jar
+mybatis-plus-annotation-3.5.3.1.jar
+mybatis-plus-boot-starter-3.5.3.1.jar
+mybatis-plus-core-3.5.3.1.jar
+mybatis-plus-extension-3.5.3.1.jar
+mybatis-spring-2.0.7.jar
+netty-all-4.1.91.Final.jar
+netty-buffer-4.1.91.Final.jar
+netty-codec-4.1.91.Final.jar
+netty-codec-dns-4.1.91.Final.jar
+netty-codec-haproxy-4.1.91.Final.jar
+netty-codec-http-4.1.91.Final.jar
+netty-codec-http2-4.1.91.Final.jar
+netty-codec-memcache-4.1.91.Final.jar
+netty-codec-mqtt-4.1.91.Final.jar
+netty-codec-redis-4.1.91.Final.jar
+netty-codec-smtp-4.1.91.Final.jar
+netty-codec-socks-4.1.91.Final.jar
+netty-codec-stomp-4.1.91.Final.jar
+netty-codec-xml-4.1.91.Final.jar
+netty-common-4.1.91.Final.jar
+netty-handler-4.1.91.Final.jar
+netty-handler-proxy-4.1.91.Final.jar
+netty-handler-ssl-ocsp-4.1.91.Final.jar
+netty-resolver-4.1.91.Final.jar
+netty-resolver-dns-4.1.91.Final.jar
+netty-resolver-dns-classes-macos-4.1.91.Final.jar
+netty-resolver-dns-native-macos-4.1.91.Final-osx-aarch_64.jar
+netty-resolver-dns-native-macos-4.1.91.Final-osx-x86_64.jar
+netty-transport-4.1.91.Final.jar
+netty-transport-classes-epoll-4.1.91.Final.jar
+netty-transport-classes-kqueue-4.1.91.Final.jar
+netty-transport-native-epoll-4.1.91.Final-linux-aarch_64.jar
+netty-transport-native-epoll-4.1.91.Final-linux-x86_64.jar
+netty-transport-native-kqueue-4.1.91.Final-osx-aarch_64.jar
+netty-transport-native-kqueue-4.1.91.Final-osx-x86_64.jar
+netty-transport-native-unix-common-4.1.91.Final.jar
+netty-transport-rxtx-4.1.91.Final.jar
+netty-transport-sctp-4.1.91.Final.jar
+netty-transport-udt-4.1.91.Final.jar
+nimbus-jose-jwt-8.22.1.jar
+oauth2-oidc-sdk-8.22.jar
+objenesis-3.2.jar
+okhttp-2.7.5.jar
+okhttp-4.9.3.jar
+okio-2.8.0.jar
+orc-core-1.5.12.jar
+orc-mapreduce-1.5.12.jar
+orc-shims-1.5.12.jar
+org.eclipse.jgit-5.13.1.202206130422-r.jar
+org.eclipse.jgit.ssh.jsch-5.13.1.202206130422-r.jar
+oro-2.0.8.jar
+osgi-resource-locator-1.0.3.jar
+p6spy-3.9.1.jar
+pac4j-config-4.5.7.jar
+pac4j-core-4.5.7.jar
+pac4j-oauth-4.5.7.jar
+pac4j-oidc-4.5.7.jar
+pac4j-springboot-4.5.7.jar
+paranamer-2.8.jar
+parquet-column-1.10.1.jar
+parquet-common-1.10.1.jar
+parquet-encoding-1.10.1.jar
+parquet-format-2.4.0.jar
+parquet-hadoop-1.10.1.jar
+parquet-jackson-1.10.1.jar
+plexus-cipher-1.4.jar
+plexus-classworlds-2.2.3.jar
+plexus-component-annotations-1.6.jar
+plexus-interpolation-1.14.jar
+plexus-sec-dispatcher-1.3.jar
+plexus-utils-3.3.0.jar
+postgresql-42.5.1.jar
+protobuf-java-3.11.4.jar
+py4j-0.10.9.jar
+pyrolite-4.30.jar
+quartz-2.3.2.jar
+re2j-1.1.jar
+scala-compiler-2.12.17.jar
+scala-library-2.12.17.jar
+scala-parser-combinators_2.12-1.1.2.jar
+scala-reflect-2.12.17.jar
+scala-xml_2.12-1.3.0.jar
+scribejava-apis-7.1.1.jar
+scribejava-core-7.1.1.jar
+shims-0.9.0.jar
+shiro-cache-1.10.0.jar
+shiro-config-core-1.10.0.jar
+shiro-config-ogdl-1.10.0.jar
+shiro-core-1.10.0.jar
+shiro-crypto-cipher-1.10.0.jar
+shiro-crypto-core-1.10.0.jar
+shiro-crypto-hash-1.10.0.jar
+shiro-event-1.10.0.jar
+shiro-lang-1.10.0.jar
+shiro-spring-1.10.0.jar
+shiro-web-1.10.0.jar
+sisu-guice-2.1.7-noaop.jar
+sisu-inject-bean-1.4.2.jar
+sisu-inject-plexus-1.4.2.jar
+slf4j-api-1.7.32.jar
+snakeyaml-2.0.jar
+snappy-java-1.1.8.2.jar
+spark-catalyst_2.12-3.1.2.jar
+spark-core_2.12-3.1.2.jar
+spark-kvstore_2.12-3.1.2.jar
+spark-launcher_2.12-3.1.2.jar
+spark-network-common_2.12-3.1.2.jar
+spark-network-shuffle_2.12-3.1.2.jar
+spark-sketch_2.12-3.1.2.jar
+spark-sql_2.12-3.1.2.jar
+spark-streaming-kafka-0-10_2.12-3.1.2.jar
+spark-streaming_2.12-3.1.2.jar
+spark-tags_2.12-3.1.2.jar
+spark-token-provider-kafka-0-10_2.12-3.1.2.jar
+spark-unsafe_2.12-3.1.2.jar
+spring-aop-5.3.27.jar
+spring-beans-5.3.27.jar
+spring-boot-2.7.11.jar
+spring-boot-autoconfigure-2.7.11.jar
+spring-boot-configuration-processor-2.7.11.jar
+spring-boot-starter-2.7.11.jar
+spring-boot-starter-aop-2.7.11.jar
+spring-boot-starter-jdbc-2.7.11.jar
+spring-boot-starter-json-2.7.11.jar
+spring-boot-starter-quartz-2.7.11.jar
+spring-boot-starter-undertow-2.7.11.jar
+spring-boot-starter-web-2.7.11.jar
+spring-boot-starter-websocket-2.7.11.jar
+spring-context-5.3.27.jar
+spring-context-support-5.3.27.jar
+spring-core-5.3.27.jar
+spring-expression-5.3.27.jar
+spring-jcl-5.3.27.jar
+spring-jdbc-5.3.27.jar
+spring-ldap-core-2.4.1.jar
+spring-messaging-5.3.27.jar
+spring-tx-5.3.27.jar
+spring-web-5.3.27.jar
+spring-webmvc-5.3.27.jar
+spring-websocket-5.3.27.jar
+springdoc-openapi-common-1.6.9.jar
+springdoc-openapi-ui-1.6.9.jar
+springdoc-openapi-webmvc-core-1.6.9.jar
+stax2-api-3.1.4.jar
+stream-2.9.6.jar
+streampark-shaded-jackson-1.0.0.jar
+streampark-shaded-slf4j-1.0.0.jar
+swagger-annotations-1.6.5.jar
+swagger-annotations-2.2.2.jar
+swagger-core-2.2.0.jar
+swagger-models-2.2.2.jar
+swagger-ui-4.11.1.jar
+threeten-extra-1.5.0.jar
+tika-core-1.20.jar
+token-provider-1.0.1.jar
+tomcat-embed-el-9.0.74.jar
+undertow-core-2.2.24.Final.jar
+undertow-servlet-2.2.24.Final.jar
+undertow-websockets-jsr-2.2.24.Final.jar
+univocity-parsers-2.9.1.jar
+unused-1.0.0.jar
+webjars-locator-core-0.50.jar
+wildfly-client-config-1.0.1.Final.jar
+wildfly-common-1.5.4.Final.jar
+woodstox-core-5.0.3.jar
+xbean-asm7-shaded-4.15.jar
+xml-apis-1.4.01.jar
+xnio-api-3.8.7.Final.jar
+xnio-nio-3.8.7.Final.jar
+xz-1.5.jar
+zookeeper-3.4.14.jar
+zstd-jni-1.4.8-1.jar
