This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 988296dcd1e2d5876f41ca2d481b126523f5a457 Author: Grant Henke <[email protected]> AuthorDate: Tue Jun 11 12:05:29 2019 -0500 [build] Fix jar verification via verify_jars.pl This patch fixes jar verification with a few changes. It fixes some dependencies in kudu-backup and kudu-test-utils to be sure we aren’t pulling in unwanted jars. It also updates verify_jars.pl to allow various relocated classes and the Spark DataSourceRegister file. It also ignores the buildSrc, kudu-jepsen and kudu-backup-tools jars. Change-Id: I57272ff8c1db043c7be00f7f65c11d010b33c8b5 Reviewed-on: http://gerrit.cloudera.org:8080/13584 Reviewed-by: Alexey Serbin <[email protected]> Reviewed-by: Mike Percy <[email protected]> Tested-by: Grant Henke <[email protected]> --- build-support/verify_jars.pl | 17 ++++++++++++++--- java/gradle/dependencies.gradle | 3 +-- java/gradle/shadow.gradle | 9 +++++++++ java/kudu-backup/build.gradle | 2 ++ java/kudu-client/build.gradle | 2 -- java/kudu-test-utils/build.gradle | 6 +++++- 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/build-support/verify_jars.pl b/build-support/verify_jars.pl index 7b93ee3..4209c44 100755 --- a/build-support/verify_jars.pl +++ b/build-support/verify_jars.pl @@ -29,13 +29,19 @@ my $pat_kudu_shaded_prefix = qr{^org/apache/kudu/shaded/}; # Allowed filenames of non-Java files in JARs. my $pat_allow_non_java = qr{(?:\.(?:txt|xml|properties|proto|MF)| - LICENSE|NOTICE|DEPENDENCIES)$}x; + LICENSE|NOTICE|DEPENDENCIES| + # The kudu-spark DataSourceRegister file. + DataSourceRegister)$}x; # Allowed filenames of shaded dependencies in JARs. my $pat_allow_kudu_shaded = qr{^org/apache/kudu/shaded/ - (?:com/google/(?:common|gson|protobuf|thirdparty/publicsuffix)| + (?:com/google/(?:common|gson|gradle/osdetector|protobuf|thirdparty/publicsuffix)| com/sangupta/murmur| + kr/motd/maven| + org/apache/(?:commons|http)| + org/checkerframework| + org/HdrHistogram| org/jboss/netty| scopt) }x; @@ -67,7 +73,12 @@ chomp(my @jars = `find . -type f -name \*.jar | grep -v sources\.jar | grep -v javadoc\.jar | grep -v unshaded\.jar | - grep -v gradle-wrapper.jar`); + grep -v buildSrc.jar | + grep -v gradle-wrapper.jar | + # Ignored because it's test only and unpublished. + grep -v kudu-jepsen.*\.jar | + # Ignored because it's a tool jar that shades everything. + grep -v kudu-backup-tools.*\.jar`); my $num_errors = 0; diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle index 6721564..c80763e 100755 --- a/java/gradle/dependencies.gradle +++ b/java/gradle/dependencies.gradle @@ -87,8 +87,7 @@ libs += [ hadoopCommon : "org.apache.hadoop:hadoop-common:$versions.hadoop", hadoopMRClientCommon : "org.apache.hadoop:hadoop-mapreduce-client-common:$versions.hadoop", hadoopMRClientCore : "org.apache.hadoop:hadoop-mapreduce-client-core:$versions.hadoop", - hamcrestCore : "org.hamcrest:hamcrest-core:$versions.hamcrest", - hamcrestLib : "org.hamcrest:hamcrest-library:$versions.hamcrest", + hamcrest : "org.hamcrest:hamcrest:$versions.hamcrest", hdrhistogram : "org.hdrhistogram:HdrHistogram:$versions.hdrhistogram", hiveMetastore : "org.apache.hive:hive-metastore:$versions.hive", hiveMetastoreTest : "org.apache.hive:hive-metastore:$versions.hive:tests", diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle index 499554f..a46e678 100644 --- a/java/gradle/shadow.gradle +++ b/java/gradle/shadow.gradle @@ -45,10 +45,19 @@ configurations.archives.artifacts.removeAll { // location across all modules. shadowJar { relocate "com.google.common", "org.apache.kudu.shaded.com.google.common" + relocate "com.google.gradle.osdetector", "org.apache.kudu.shaded.com.google.gradle.osdetector" relocate "com.google.gson", "org.apache.kudu.shaded.com.google.gson" relocate "com.google.protobuf", "org.apache.kudu.shaded.com.google.protobuf" relocate "com.google.thirdparty", "org.apache.kudu.shaded.com.google.thirdparty" relocate "com.sangupta", "org.apache.kudu.shaded.com.sangupta" + // Pulled in via osdetector. + relocate "kr.motd.maven", "org.apache.kudu.shaded.kr.motd.maven" + relocate "org.apache.http", "org.apache.kudu.shaded.org.apache.http" + relocate "org.apache.commons", "org.apache.kudu.shaded.org.apache.commons" + // Pulled in via Guava. + relocate "org.checkerframework", "org.apache.kudu.shaded.org.checkerframework" + relocate "org.hamcrest", "org.apache.kudu.shaded.org.hamcrest" + relocate "org.HdrHistogram", "org.apache.kudu.shaded.org.HdrHistogram" relocate "org.jboss.netty", "org.apache.kudu.shaded.org.jboss.netty" relocate "scopt", "org.apache.kudu.shaded.scopt" } diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle index 47d414e..7ee49f5 100644 --- a/java/kudu-backup/build.gradle +++ b/java/kudu-backup/build.gradle @@ -23,6 +23,8 @@ dependencies { compile(project(path: ":kudu-backup-common")) { // Ensure we use the hadoop-client provided by Spark to avoid any compatibility issues. exclude group: "org.apache.hadoop", module: "hadoop-common" + // Ensure we use the scala-library provided by Spark to avoid any compatibility issues. + exclude group: "org.scala-lang", module: "scala-library" } compile project(path: ":kudu-client", configuration: "shadow") compile project(path: ":kudu-spark", configuration: "shadow") diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle index e42b4eb..6ac2f07 100644 --- a/java/kudu-client/build.gradle +++ b/java/kudu-client/build.gradle @@ -36,8 +36,6 @@ dependencies { optional libs.yetusAnnotations testCompile project(":kudu-test-utils") - testCompile libs.hamcrestCore - testCompile libs.hamcrestLib testCompile libs.hiveMetastore // The HMS client relies on the MR client-core artifact for JobConf, but only // specifies it as an optional dependency. Gradle doesn't pull in optional diff --git a/java/kudu-test-utils/build.gradle b/java/kudu-test-utils/build.gradle index 3e03012..72db57c 100644 --- a/java/kudu-test-utils/build.gradle +++ b/java/kudu-test-utils/build.gradle @@ -25,7 +25,11 @@ dependencies { compile libs.httpMime compile libs.osdetector - compileUnshaded libs.junit + compileUnshaded libs.hamcrest + compileUnshaded(libs.junit) { + // Use the Kudu specified hamcrest. + exclude group: "org.hamcrest" + } compileUnshaded libs.slf4jApi // Support using any kudu-binary jar for tests via `-PuseBinJar=<version>`.
