This is an automated email from the ASF dual-hosted git repository.
achennaka pushed a commit to branch branch-1.18.x
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/branch-1.18.x by this push:
new bc852ca38 [java] KUDU-3657: fix publishing
bc852ca38 is described below
commit bc852ca38327361e359427e5c5360a29359b1b27
Author: zchovan <[email protected]>
AuthorDate: Wed Jun 11 17:44:14 2025 +0200
[java] KUDU-3657: fix publishing
The changes introduced in KUDU-3610 have fixed the POM generation for
the kudu-hive java module, however the publishing for other modules were
broken as a side-effect. This patch aims to fix these issues.
The change in publishing.gradle file checks if the implementation and shadow
configurations are present to calculate the correct pom to be generated.
If they are not present for the given module, they are simply filled out
as empty arrays.
Additionally, the skipping of publishing wasn't working, this needed to
be fixed for the following modules (to keep in alignment with pre-gradle
upgrade behaviour):
* kudu-backup-common
* kudu-jepsen
* kudu-proto
* kudu-subprocess
Fixed the generated jar names of the following modules:
* kudu-backup -> kudu-backup3_2.12
* kudu-spark -> kudu-spark3_2.12
* kudu-spark-tools -> kudu-spark3-tools_2.12
Changed the compileUnshaded configuration to be extending the
compileOnly configuration. This was necessary to correct the scope of
these dependencies. This change also made it necessary to add the
compileUnshaded dependencies explicitly to the test compile path as
well[0].
[0] https://blog.gradle.org/introducing-compile-only-dependencies
Change-Id: Ie045805a68bfcd16325e1e4dbb38653cefe3a4b0
Reviewed-on: http://gerrit.cloudera.org:8080/22768
Reviewed-by: Abhishek Chennaka <[email protected]>
Tested-by: Alexey Serbin <[email protected]>
Reviewed-by: Alexey Serbin <[email protected]>
(cherry picked from commit 6d72d4a242076468501f3430b9a2cd050c634be2)
Reviewed-on: http://gerrit.cloudera.org:8080/23079
Reviewed-by: Marton Greber <[email protected]>
Tested-by: Marton Greber <[email protected]>
---
java/gradle/publishing.gradle | 22 +++++++++++++++++++---
java/gradle/shadow.gradle | 2 +-
java/kudu-backup-tools/build.gradle | 7 ++++---
java/kudu-backup/build.gradle | 25 +++++++++++++++++++------
java/kudu-client/build.gradle | 13 ++++++++++---
java/kudu-hive/build.gradle | 12 +++++++++---
java/kudu-spark-tools/build.gradle | 20 +++++++++++++++-----
java/kudu-spark/build.gradle | 18 ++++++++++++++----
java/kudu-test-utils/build.gradle | 27 +++++++++++++++++++++------
9 files changed, 112 insertions(+), 34 deletions(-)
diff --git a/java/gradle/publishing.gradle b/java/gradle/publishing.gradle
index e78b8f5af..027c7ac8c 100644
--- a/java/gradle/publishing.gradle
+++ b/java/gradle/publishing.gradle
@@ -48,6 +48,15 @@ publishing {
groupId "org.apache.kudu"
version "${project.version}"
+ // Adjust the artifact names to include the spark and scala base
versions.
+ if (["kudu-backup", "kudu-spark"].contains(project.name)) {
+ artifactId =
"${project.archivesBaseName}${versions.sparkBase}_${versions.scalaBase}"
+ }
+
+ if (project.name.equals("kudu-spark-tools")) {
+ artifactId =
"kudu-spark${versions.sparkBase}-tools_${versions.scalaBase}"
+ }
+
artifact(sourcesJar) {
classifier = "sources"
}
@@ -71,9 +80,16 @@ publishing {
// Remove the shaded dependencies from the generated pom.
pom.withXml {
- def apiDeps = project.configurations.api.allDependencies
- def shadedDeps = project.configurations.shadow.allDependencies
- def finalDeps = apiDeps - shadedDeps
+ def implementationDeps = []
+ def shadedDeps = []
+ if (project.configurations.hasProperty("implementation")) {
+ implementationDeps =
project.configurations.implementation.allDependencies
+ }
+ if (project.configurations.hasProperty("shadow")) {
+ shadedDeps = project.configurations.shadow.allDependencies
+ }
+
+ def finalDeps = implementationDeps - shadedDeps
asNode().dependencies.dependency.findAll{
finalDeps.findAll{ dep ->
dep.name == it.artifactId*.value()[0][0]
diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle
index 6fcae5510..8846e30dc 100644
--- a/java/gradle/shadow.gradle
+++ b/java/gradle/shadow.gradle
@@ -92,7 +92,7 @@ shadowJar {
// By default shadow assumes all dependencies are shaded.
configurations.create("compileUnshaded")
configurations.shadow.extendsFrom(configurations.compileUnshaded)
-configurations.implementation.extendsFrom(configurations.compileUnshaded)
+configurations.compileOnly.extendsFrom(configurations.compileUnshaded)
// We use afterEvaluate to add additional configuration once all the
definitions
// in the projects build script have been applied
diff --git a/java/kudu-backup-tools/build.gradle
b/java/kudu-backup-tools/build.gradle
index eb7963b95..4947af6e9 100644
--- a/java/kudu-backup-tools/build.gradle
+++ b/java/kudu-backup-tools/build.gradle
@@ -16,6 +16,7 @@
// under the License.
apply plugin: "scala"
+apply plugin: "java-library"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
@@ -27,11 +28,11 @@ dependencies {
exclude group: "org.scala-lang", module: "scala-library"
}
implementation libs.scalaLibrary
- implementation libs.slf4jApi
+ compileOnlyApi libs.slf4jApi
- compileOnly libs.hadoopClient
+ compileOnlyApi libs.hadoopClient
- implementation libs.yetusAnnotations
+ compileOnlyApi libs.yetusAnnotations
// MiniDFSCluster needs mockito package to be present in the classpath.
// See: https://issues.apache.org/jira/browse/MAPREDUCE-7471 and
// https://issues.apache.org/jira/browse/HADOOP-19164 for more details.
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index f0c5b710e..12fe82495 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -16,6 +16,7 @@
// under the License.
apply plugin: "scala"
+apply plugin: "java-library"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
@@ -27,7 +28,7 @@ dependencies {
exclude group: "org.scala-lang", module: "scala-library"
}
implementation project(path: ":kudu-client", configuration: "shadow")
- implementation project(path: ":kudu-spark", configuration: "shadow")
+ compileOnlyApi project(path: ":kudu-spark", configuration: "shadow")
implementation libs.protobufJava
implementation (libs.scopt) {
// Make sure wrong Scala version is not pulled in.
@@ -36,20 +37,32 @@ dependencies {
// TODO(KUDU-2500): Spark uses reflection which requires the annotations at
runtime.
implementation libs.yetusAnnotations
- implementation libs.scalaLibrary
- implementation libs.sparkAvro
- implementation libs.sparkCore
- implementation libs.sparkSql
- implementation libs.slf4jApi
+ compileOnlyApi libs.scalaLibrary
+ compileOnlyApi libs.sparkAvro
+ compileOnlyApi libs.sparkCore
+ compileOnlyApi libs.sparkSql
+ compileOnlyApi libs.slf4jApi
testImplementation project(path: ":kudu-test-utils", configuration: "shadow")
testImplementation project(path: ":kudu-spark", configuration: "test")
+ // Note: both configurations are needed to ensure that the classpath is
correct
+ testImplementation project(path: ":kudu-spark")
testImplementation libs.junit
testImplementation libs.log4jApi
testImplementation libs.log4jCore
testImplementation libs.log4jSlf4jImpl
testImplementation libs.scalatest
testImplementation libs.logCaptor
+
+ // Have to re-define some dependencies here, as compile-only dependencies
+ // are not inherited by the test classpath.
+ // See: https://blog.gradle.org/introducing-compile-only-dependencies
+ // Note: compileUnshaded is extended from the compileOnly dependency scope.
+ testImplementation libs.scalaLibrary
+ testImplementation libs.sparkAvro
+ testImplementation libs.sparkCore
+ testImplementation libs.sparkSql
+ testImplementation libs.slf4jApi
}
shadowJar {
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index abcc5c713..bc0d6d96a 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -24,7 +24,7 @@ apply from: "$rootDir/gradle/benchmarks.gradle"
dependencies {
implementation project(path: ":kudu-proto")
- api libs.protobufJava
+ compileOnly libs.protobufJava
// Not shaded in the client JAR because it's part of the public API.
compileUnshaded (libs.async) {
@@ -41,8 +41,8 @@ dependencies {
implementation libs.murmur
implementation libs.netty
- compileOnly libs.jsr305
- compileOnly libs.yetusAnnotations
+ compileOnlyApi libs.jsr305
+ compileOnlyApi libs.yetusAnnotations
testImplementation project(":kudu-test-utils")
testImplementation libs.junit
@@ -51,6 +51,13 @@ dependencies {
testImplementation libs.log4jSlf4jImpl
testImplementation libs.mockitoCore
+ // Have to re-define some dependencies here, as compile-only dependencies
+ // are not inherited by the test classpath.
+ // See: https://blog.gradle.org/introducing-compile-only-dependencies
+ // Note: compileUnshaded is extended from the compileOnly dependency scope.
+ testImplementation libs.protobufJava
+ testImplementation libs.async
+
shadowJar {
dependencies {
exclude(dependency(libs.jsr305))
diff --git a/java/kudu-hive/build.gradle b/java/kudu-hive/build.gradle
index bf27fc002..d4cc09a8a 100644
--- a/java/kudu-hive/build.gradle
+++ b/java/kudu-hive/build.gradle
@@ -14,16 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+apply plugin: "java-library"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
implementation project(path: ":kudu-client", configuration: "shadow")
- implementation libs.hiveMetastore
- compileOnly libs.slf4jApi
+ compileOnlyApi libs.hiveMetastore
+ compileOnlyApi libs.slf4jApi
// Transitively required through hive-metastore,
// which has marked it as an optional dependency.
- implementation libs.hadoopCommon
+ compileOnlyApi libs.hadoopCommon
+ compileOnlyApi libs.hadoopMRClientCommon
testImplementation libs.hadoopMRClientCommon
testImplementation project(path: ":kudu-test-utils", configuration: "shadow")
@@ -32,6 +34,10 @@ dependencies {
testImplementation libs.log4jApi
testImplementation libs.log4jCore
testImplementation libs.log4jSlf4jImpl
+
+ testImplementation libs.hiveMetastore
+ testImplementation libs.hadoopCommon
+ testImplementation libs.hadoopMRClientCommon
}
shadowJar {
diff --git a/java/kudu-spark-tools/build.gradle
b/java/kudu-spark-tools/build.gradle
index 95daad590..58bb21155 100644
--- a/java/kudu-spark-tools/build.gradle
+++ b/java/kudu-spark-tools/build.gradle
@@ -16,11 +16,12 @@
// under the License.
apply plugin: "scala"
+apply plugin: "java-library"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
implementation project(path: ":kudu-client", configuration: "shadow")
- implementation project(':kudu-spark')
+ compileOnlyApi project(':kudu-spark')
// TODO(KUDU-2500): Spark uses reflection which requires the annotations at
runtime.
implementation libs.yetusAnnotations
implementation (libs.scopt) {
@@ -28,11 +29,12 @@ dependencies {
exclude group: "org.scala-lang", module: "scala-library"
}
- implementation libs.sparkCore
- implementation libs.sparkSql
+ compileOnlyApi libs.sparkAvro
+ compileOnlyApi libs.sparkCore
+ compileOnlyApi libs.sparkSql
- compileOnly libs.scalaLibrary
- compileOnly libs.slf4jApi
+ compileOnlyApi libs.scalaLibrary
+ compileOnlyApi libs.slf4jApi
testImplementation project(path: ":kudu-test-utils", configuration: "shadow")
testImplementation project(path: ":kudu-spark", configuration: "test")
@@ -42,6 +44,14 @@ dependencies {
testImplementation libs.log4jSlf4jImpl
testImplementation libs.scalatest
testImplementation libs.sparkAvro
+
+ // Have to re-define some dependencies here, as compile-only dependencies
+ // are not inherited by the test classpath.
+ // See: https://blog.gradle.org/introducing-compile-only-dependencies
+ // Note: compileUnshaded is extended from the compileOnly dependency scope.
+ testImplementation project(':kudu-spark')
+ testImplementation libs.sparkCore
+ testImplementation libs.sparkSql
}
shadowJar {
diff --git a/java/kudu-spark/build.gradle b/java/kudu-spark/build.gradle
index 28d7adce2..e0f3d99d4 100644
--- a/java/kudu-spark/build.gradle
+++ b/java/kudu-spark/build.gradle
@@ -16,6 +16,7 @@
// under the License.
apply plugin: "scala"
+apply plugin: "java-library"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
@@ -24,11 +25,11 @@ dependencies {
// TODO(KUDU-2500): Spark uses reflection which requires the annotations at
runtime.
implementation libs.yetusAnnotations
- implementation libs.sparkCore
- implementation libs.sparkSql
+ compileOnlyApi libs.sparkCore
+ compileOnlyApi libs.sparkSql
- compileOnly libs.scalaLibrary
- compileOnly libs.slf4jApi
+ compileOnlyApi libs.scalaLibrary
+ compileOnlyApi libs.slf4jApi
testImplementation project(path: ":kudu-test-utils", configuration: "shadow")
testImplementation libs.junit
@@ -36,6 +37,15 @@ dependencies {
testImplementation libs.log4jApi
testImplementation libs.log4jCore
testImplementation libs.log4jSlf4jImpl
+
+ // Have to re-define some dependencies here, as compile-only dependencies
+ // are not inherited by the test classpath.
+ // See: https://blog.gradle.org/introducing-compile-only-dependencies
+ // Note: compileUnshaded is extended from the compileOnly dependency scope.
+ testImplementation libs.sparkCore
+ testImplementation libs.sparkSql
+ testImplementation libs.scalaLibrary
+ testImplementation libs.slf4jApi
}
shadowJar {
diff --git a/java/kudu-test-utils/build.gradle
b/java/kudu-test-utils/build.gradle
index f4203789a..e04a63ce0 100644
--- a/java/kudu-test-utils/build.gradle
+++ b/java/kudu-test-utils/build.gradle
@@ -15,6 +15,10 @@
// specific language governing permissions and limitations
// under the License.
+plugins {
+ id("java-library")
+}
+
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
@@ -36,7 +40,7 @@ dependencies {
exclude group: "org.slf4j", module: "slf4j-api"
}
- implementation libs.hamcrest
+ compileOnlyApi libs.hamcrest
compileUnshaded(libs.junit) {
// Use the Kudu specified hamcrest.
exclude group: "org.hamcrest"
@@ -51,15 +55,26 @@ dependencies {
}
// Needed for CapturingLogAppender. Optional otherwise.
- implementation libs.log4jApi
- implementation libs.log4jCore
- implementation libs.log4jSlf4jImpl
+ compileOnlyApi libs.log4jApi
+ compileOnlyApi libs.log4jCore
+ compileOnlyApi libs.log4jSlf4jImpl
- compileOnly libs.jsr305
- compileOnly libs.yetusAnnotations
+ compileOnlyApi libs.jsr305
+ compileOnlyApi libs.yetusAnnotations
testImplementation libs.jetty
testImplementation libs.jettyServlet
+
+ // Have to re-define some dependencies here, as compile-only dependencies
+ // are not inherited by the test classpath.
+ // See: https://blog.gradle.org/introducing-compile-only-dependencies
+ // Note: compileUnshaded is extended from the compileOnly dependency scope.
+ testImplementation libs.junit
+ testImplementation libs.async
+ testImplementation libs.hamcrest
+ testImplementation libs.log4jApi
+ testImplementation libs.log4jCore
+ testImplementation libs.log4jSlf4jImpl
}
shadowJar {
