This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2e72638 [BEAM-13430] Remove propdeps and replace with compileOnly
(#16308)
2e72638 is described below
commit 2e72638c6b08631bbb9a29e6c24f97043ffd0a31
Author: dpcollins-google <[email protected]>
AuthorDate: Tue Dec 21 17:02:12 2021 -0500
[BEAM-13430] Remove propdeps and replace with compileOnly (#16308)
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
* [BEAM-13430] Remove propdeps and replace with compileOnly
Still generate <scope>provided</scope> in the output POM
---
buildSrc/build.gradle.kts | 1 -
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 16 ++---------
examples/java/build.gradle | 4 +--
examples/java/twitter/build.gradle | 4 +--
examples/kotlin/build.gradle | 2 +-
ownership/JAVA_DEPENDENCY_OWNERS.yaml | 5 ----
runners/direct-java/build.gradle | 4 +--
runners/flink/flink_runner.gradle | 3 +-
.../worker/legacy-worker/build.gradle | 16 ++++++++---
.../dataflow/worker/status/DebugCapture.java | 4 +--
runners/java-fn-execution/build.gradle | 5 ++--
runners/spark/job-server/spark_job_server.gradle | 4 +--
runners/spark/spark_runner.gradle | 29 +++++++++++++-------
sdks/java/container/build.gradle | 2 +-
sdks/java/core/build.gradle | 12 ++++----
.../google-cloud-platform-core/build.gradle | 4 +--
sdks/java/extensions/ml/build.gradle | 2 +-
sdks/java/extensions/sorter/build.gradle | 6 ++--
sdks/java/extensions/sql/build.gradle | 32 ++++++++++++++++------
sdks/java/extensions/sql/datacatalog/build.gradle | 14 ++++++++--
sdks/java/extensions/sql/hcatalog/build.gradle | 7 +++--
.../provider/hcatalog/BeamSqlHiveSchemaTest.java | 5 ++++
sdks/java/extensions/sql/jdbc/build.gradle | 7 +----
sdks/java/extensions/sql/zetasql/build.gradle | 2 +-
sdks/java/fn-execution/build.gradle | 2 +-
sdks/java/io/debezium/build.gradle | 2 +-
sdks/java/io/hadoop-common/build.gradle | 9 ++++--
sdks/java/io/hadoop-file-system/build.gradle | 6 ++--
sdks/java/io/hadoop-format/build.gradle | 9 ++++--
sdks/java/io/hbase/build.gradle | 2 +-
sdks/java/io/hcatalog/build.gradle | 13 +++++----
sdks/java/io/kafka/build.gradle | 11 ++++----
sdks/java/io/parquet/build.gradle | 3 +-
sdks/java/io/rabbitmq/build.gradle | 2 +-
sdks/java/testing/tpcds/build.gradle | 2 +-
35 files changed, 147 insertions(+), 104 deletions(-)
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
index 47df89e..3339eba 100644
--- a/buildSrc/build.gradle.kts
+++ b/buildSrc/build.gradle.kts
@@ -43,7 +43,6 @@ dependencies {
runtimeOnly("net.ltgt.gradle:gradle-apt-plugin:0.21")
// Enable a Java annotation processor
runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13")
// Enable proto code generation
- runtimeOnly("io.spring.gradle:propdeps-plugin:0.0.9.RELEASE")
// Enable provided and optional configurations
runtimeOnly("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0")
// Enable Avro code generation
runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1")
// Enable a code formatting plugin
runtimeOnly("gradle.plugin.com.github.blindpirate:gogradle:0.11.4")
// Enable Go code compilation
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 469afc2..a91d62e 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -748,9 +748,6 @@ class BeamModulePlugin implements Plugin<Project> {
// * java
// * maven
// * net.ltgt.apt (plugin to configure annotation processing tool)
- // * propdeps (provide optional and provided dependency configurations)
- // * propdeps-maven
- // * propdeps-idea
// * checkstyle
// * spotbugs
// * shadow (conditional on shadowClosure being specified)
@@ -1032,13 +1029,6 @@ class BeamModulePlugin implements Plugin<Project> {
implementation
"org.checkerframework:checker-qual:$checkerframework_version"
}
- // Add the optional and provided configurations for dependencies
- // TODO: Either remove these plugins and find another way to generate
the Maven poms
- // with the correct dependency scopes configured.
- project.apply plugin: 'propdeps'
- project.apply plugin: 'propdeps-maven'
- project.apply plugin: 'propdeps-idea'
-
// Defines Targets for sonarqube analysis reporting.
project.apply plugin: "org.sonarqube"
@@ -1254,7 +1244,7 @@ class BeamModulePlugin implements Plugin<Project> {
project.dependencies {
shadowTestRuntimeClasspath it.project(path: project.path,
configuration: "shadowTest")
- shadowTestRuntimeClasspath it.project(path: project.path,
configuration: "provided")
+ shadowTestRuntimeClasspath it.project(path: project.path,
configuration: "compileOnly")
}
project.test { classpath =
project.configurations.shadowTestRuntimeClasspath }
@@ -1580,8 +1570,8 @@ class BeamModulePlugin implements Plugin<Project> {
// TODO: Should we use the runtime scope instead of the
compile scope
// which forces all our consumers to declare what they consume?
generateDependenciesFromConfiguration(
- configuration: (configuration.shadowClosure ? 'shadow' :
'compile'), scope: 'compile')
- generateDependenciesFromConfiguration(configuration:
'provided', scope: 'provided')
+ configuration: (configuration.shadowClosure ? 'shadow' :
'implementation'), scope: 'compile')
+ generateDependenciesFromConfiguration(configuration:
'compileOnly', scope: 'provided')
if (!boms.isEmpty()) {
def dependencyManagementNode =
root.appendNode('dependencyManagement')
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
index 677317d..dba5320 100644
--- a/examples/java/build.gradle
+++ b/examples/java/build.gradle
@@ -77,8 +77,8 @@ dependencies {
implementation library.java.proto_google_cloud_bigtable_v2
implementation library.java.proto_google_cloud_datastore_v1
implementation library.java.slf4j_api
- provided library.java.commons_io
- provided library.java.commons_csv
+ implementation library.java.commons_io
+ implementation library.java.commons_csv
runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
implementation library.java.vendored_grpc_1_36_0
implementation library.java.vendored_guava_26_0_jre
diff --git a/examples/java/twitter/build.gradle
b/examples/java/twitter/build.gradle
index f707ae20..693e6ef 100644
--- a/examples/java/twitter/build.gradle
+++ b/examples/java/twitter/build.gradle
@@ -50,8 +50,8 @@ dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation library.java.joda_time
implementation library.java.slf4j_api
- provided library.java.commons_io
- provided library.java.commons_csv
+ implementation library.java.commons_io
+ implementation library.java.commons_csv
runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
implementation ("org.twitter4j:twitter4j-stream:4.0.7")
implementation ("org.twitter4j:twitter4j-core:4.0.7")
diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle
index 1c56e3f..3e78155 100644
--- a/examples/kotlin/build.gradle
+++ b/examples/kotlin/build.gradle
@@ -52,7 +52,7 @@ dependencies {
implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation library.java.vendored_guava_26_0_jre
implementation project(path: ":sdks:java:core", configuration: "shadow")
- // Add the dependency that sdks:java:core that is marked as provided
+ // Add the dependency that sdks:java:core that is marked as compileOnly
implementation library.java.hamcrest_core
implementation project(":sdks:java:extensions:google-cloud-platform-core")
implementation project(":sdks:java:io:google-cloud-platform")
diff --git a/ownership/JAVA_DEPENDENCY_OWNERS.yaml
b/ownership/JAVA_DEPENDENCY_OWNERS.yaml
index 904f440..ad117f2 100644
--- a/ownership/JAVA_DEPENDENCY_OWNERS.yaml
+++ b/ownership/JAVA_DEPENDENCY_OWNERS.yaml
@@ -514,11 +514,6 @@ deps:
artifact: opencensus-contrib-grpc-metrics
owners:
- io.spring.gradle:propdeps-plugin:
- group: io.spring.gradle
- artifact: propdeps-plugin
- owners:
-
javax.xml.bind:jaxb-api:
group: javax.xml.bind
artifact: jaxb-api
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index 0fab169..6219c21 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -74,8 +74,8 @@ dependencies {
shadow library.java.joda_time
shadow library.java.slf4j_api
shadow library.java.jackson_databind
- provided library.java.hamcrest_core
- provided library.java.junit
+ compileOnly library.java.hamcrest_core
+ compileOnly library.java.junit
shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
shadowTest project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
shadowTest library.java.mockito_core
diff --git a/runners/flink/flink_runner.gradle
b/runners/flink/flink_runner.gradle
index 49d22f8..19204a5 100644
--- a/runners/flink/flink_runner.gradle
+++ b/runners/flink/flink_runner.gradle
@@ -148,7 +148,8 @@ dependencies {
implementation "org.apache.flink:flink-runtime_2.11:$flink_version"
implementation "org.apache.flink:flink-streaming-java_2.11:$flink_version"
// RocksDB state backend (included in the Flink distribution)
- provided "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
+ compileOnly "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
+ testImplementation
"org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
// FlinkStateInternalsTest extends abstract StateInternalsTest
testImplementation project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
diff --git
a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
index a0886dd..316a8ef 100644
--- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
@@ -185,17 +185,22 @@ compileTestJava {
}
dependencies {
+ implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+
// Note that any dependency that is modified here should also be modified
within
// runners/google-cloud-dataflow-java/worker/build.gradle using the rules
provided
// there.
sdk_provided_dependencies.each {
- provided(it)
+ compileOnly(it)
+ testImplementation(it)
}
sdk_provided_shaded_project_dependencies.each {
- provided project(path: it, configuration: "shadow")
+ compileOnly project(path: it, configuration: "shadow")
+ testImplementation project(path: it, configuration: "shadow")
}
sdk_provided_project_dependencies.each {
- provided project(it)
+ compileOnly project(path: it)
+ testImplementation project(path: it)
}
implementation project(path: ":model:fn-execution", configuration:
"shadow")
@@ -208,6 +213,9 @@ dependencies {
implementation project(path:
":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")
shadow library.java.vendored_guava_26_0_jre
+ implementation library.java.google_auth_library_credentials
+ implementation library.java.proto_google_common_protos
+
// Conscrypt shouldn't be included here because Conscrypt won't work when
being shaded.
// (Context: https://github.com/apache/beam/pull/13846)
// Conscrypt will be added to runtime dependencies by GrpcVendoring so
compileOnly works for now.
@@ -223,7 +231,7 @@ dependencies {
implementation library.java.joda_time
implementation library.java.slf4j_api
implementation library.java.vendored_grpc_1_36_0
- provided library.java.error_prone_annotations
+ implementation library.java.error_prone_annotations
runtimeOnly library.java.slf4j_jdk14
// Any test dependency which intersects with our relocation rules above
needs to be relocated
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java
index 21691b1..b65410b 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.dataflow.worker.status;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.json.gson.GsonFactory;
import com.google.api.client.util.Key;
import com.google.api.services.dataflow.Dataflow;
import com.google.api.services.dataflow.model.GetDebugConfigRequest;
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
})
public class DebugCapture {
private static final Logger LOG =
LoggerFactory.getLogger(DebugCapture.class);
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private static final JsonFactory JSON_FACTORY = new GsonFactory();
private static final String COMPONENT = "JavaHarness";
// How often to refresh debug capture config.
private static final long UPDATE_CONFIG_PERIOD_SEC = 60;
diff --git a/runners/java-fn-execution/build.gradle
b/runners/java-fn-execution/build.gradle
index 4b42fec..bbdba0b 100644
--- a/runners/java-fn-execution/build.gradle
+++ b/runners/java-fn-execution/build.gradle
@@ -25,7 +25,8 @@ description = "Apache Beam :: Runners :: Java Fn Execution"
dependencies {
implementation library.java.vendored_guava_26_0_jre
implementation project(":runners:core-construction-java")
- provided project(":sdks:java:harness")
+ implementation project(":runners:core-java")
+ compileOnly project(":sdks:java:harness")
implementation project(path: ":model:pipeline", configuration: "shadow")
implementation project(path: ":model:fn-execution", configuration: "shadow")
implementation project(path: ":sdks:java:core", configuration: "shadow")
@@ -37,8 +38,8 @@ dependencies {
implementation library.java.joda_time
testImplementation project(":sdks:java:harness")
testImplementation project(":runners:core-construction-java")
- testImplementation project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
+ testImplementation project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
testImplementation library.java.junit
testImplementation library.java.mockito_core
testRuntimeOnly library.java.slf4j_simple
diff --git a/runners/spark/job-server/spark_job_server.gradle
b/runners/spark/job-server/spark_job_server.gradle
index 7d0acf6..0959094 100644
--- a/runners/spark/job-server/spark_job_server.gradle
+++ b/runners/spark/job-server/spark_job_server.gradle
@@ -52,9 +52,9 @@ configurations.all {
dependencies {
implementation project(sparkRunnerProject)
permitUnusedDeclared project(sparkRunnerProject)
- implementation project(path: sparkRunnerProject, configuration: "provided")
+ implementation project(path: sparkRunnerProject, configuration:
"compileOnly")
validatesPortableRunner project(path: sparkRunnerProject, configuration:
"testRuntimeMigration")
- validatesPortableRunner project(path: sparkRunnerProject, configuration:
"provided")
+ validatesPortableRunner project(path: sparkRunnerProject, configuration:
"compileOnly")
validatesPortableRunner project(path: ":sdks:java:core", configuration:
"shadowTest")
validatesPortableRunner project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
validatesPortableRunner project(path: ":runners:portability:java",
configuration: "testRuntimeMigration")
diff --git a/runners/spark/spark_runner.gradle
b/runners/spark/spark_runner.gradle
index 13307cb..1c70a59 100644
--- a/runners/spark/spark_runner.gradle
+++ b/runners/spark/spark_runner.gradle
@@ -162,10 +162,10 @@ dependencies {
implementation project(":sdks:java:fn-execution")
implementation library.java.vendored_grpc_1_36_0
implementation library.java.vendored_guava_26_0_jre
- provided "org.apache.spark:spark-core_$spark_scala_version:$spark_version"
- provided
"org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
- provided "org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
- provided
"org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
+ compileOnly "org.apache.spark:spark-core_$spark_scala_version:$spark_version"
+ compileOnly
"org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
+ compileOnly "org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
+ compileOnly
"org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
if(project.property("spark_scala_version").equals("2.11")){
runtimeOnly library.java.jackson_module_scala_2_11
} else {
@@ -173,11 +173,20 @@ dependencies {
}
// Force paranamer 2.8 to avoid issues when using Scala 2.12
runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8"
- provided library.java.hadoop_common
- provided library.java.commons_io
- provided library.java.hamcrest_core
- provided library.java.hamcrest_library
- provided "com.esotericsoftware.kryo:kryo:2.21"
+ compileOnly library.java.hadoop_common
+ compileOnly library.java.commons_io
+ compileOnly library.java.hamcrest_core
+ compileOnly library.java.hamcrest_library
+ compileOnly "com.esotericsoftware.kryo:kryo:2.21"
+ testImplementation
"org.apache.spark:spark-core_$spark_scala_version:$spark_version"
+ testImplementation
"org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
+ testImplementation
"org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
+ testImplementation
"org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
+ testImplementation library.java.hadoop_common
+ testImplementation library.java.commons_io
+ testImplementation library.java.hamcrest_core
+ testImplementation library.java.hamcrest_library
+ testImplementation "com.esotericsoftware.kryo:kryo:2.21"
testImplementation project(":sdks:java:io:kafka")
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
// SparkStateInternalsTest extends abstract StateInternalsTest
@@ -196,7 +205,7 @@ dependencies {
validatesRunner project(path: ":examples:java", configuration:
"testRuntimeMigration")
validatesRunner project(path: project.path, configuration:
"testRuntimeMigration")
validatesRunner project(project.path)
- validatesRunner project(path: project.path, configuration: "provided")
+ validatesRunner project(path: project.path, configuration: "compileOnly")
hadoopVersions.each {kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
}
diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
index eae896c..142e4aa 100644
--- a/sdks/java/container/build.gradle
+++ b/sdks/java/container/build.gradle
@@ -40,7 +40,7 @@ dependencies {
dockerDependency project(path: ":sdks:java:harness", configuration: "shadow")
// For executing KafkaIO, e.g. as an external transform
dockerDependency project(":sdks:java:io:kafka")
- // This dependency is set to 'provided' scope in :sdks:java:io:kafka
+ // This dependency is set to 'compileOnly' scope in :sdks:java:io:kafka
dockerDependency library.java.kafka_clients
}
diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle
index 6f1df2b..fa51794 100644
--- a/sdks/java/core/build.gradle
+++ b/sdks/java/core/build.gradle
@@ -90,11 +90,11 @@ dependencies {
shadow library.java.avro
shadow library.java.snappy_java
shadow library.java.joda_time
- provided library.java.junit
- provided library.java.hamcrest_core
- provided library.java.hamcrest_library
- provided 'io.airlift:aircompressor:0.18'
- provided 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
+ compileOnly library.java.junit
+ compileOnly library.java.hamcrest_core
+ compileOnly library.java.hamcrest_library
+ compileOnly 'io.airlift:aircompressor:0.18'
+ compileOnly 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
shadowTest library.java.jackson_dataformat_yaml
shadowTest library.java.guava_testlib
shadowTest library.java.mockito_core
@@ -105,4 +105,6 @@ dependencies {
shadowTest library.java.avro_tests
shadowTest library.java.zstd_jni
testRuntimeOnly library.java.slf4j_jdk14
+ testImplementation 'io.airlift:aircompressor:0.18'
+ testImplementation 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/build.gradle
b/sdks/java/extensions/google-cloud-platform-core/build.gradle
index 90c84d9..4aa1c98 100644
--- a/sdks/java/extensions/google-cloud-platform-core/build.gradle
+++ b/sdks/java/extensions/google-cloud-platform-core/build.gradle
@@ -55,8 +55,8 @@ dependencies {
implementation library.java.jackson_annotations
implementation library.java.jackson_databind
permitUnusedDeclared library.java.jackson_databind // BEAM-11761
- provided library.java.hamcrest_core
- provided library.java.junit
+ compileOnly library.java.hamcrest_core
+ compileOnly library.java.junit
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.mockito_core
testRuntimeOnly library.java.slf4j_jdk14
diff --git a/sdks/java/extensions/ml/build.gradle
b/sdks/java/extensions/ml/build.gradle
index 6f00142..4d84986 100644
--- a/sdks/java/extensions/ml/build.gradle
+++ b/sdks/java/extensions/ml/build.gradle
@@ -48,7 +48,7 @@ dependencies {
implementation library.java.gax
implementation library.java.protobuf_java
implementation library.java.slf4j_api
- provided library.java.junit
+ compileOnly library.java.junit
testImplementation project(path: ':sdks:java:core', configuration:
'shadowTest')
implementation 'com.google.cloud:google-cloud-vision:1.99.3'
testImplementation library.java.mockito_core
diff --git a/sdks/java/extensions/sorter/build.gradle
b/sdks/java/extensions/sorter/build.gradle
index 2374ac9..50093ae 100644
--- a/sdks/java/extensions/sorter/build.gradle
+++ b/sdks/java/extensions/sorter/build.gradle
@@ -36,10 +36,12 @@ dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation library.java.vendored_guava_26_0_jre
implementation library.java.slf4j_api
- provided library.java.hadoop_mapreduce_client_core
- provided library.java.hadoop_common
+ compileOnly library.java.hadoop_mapreduce_client_core
+ compileOnly library.java.hadoop_common
testImplementation library.java.mockito_core
testImplementation library.java.junit
+ testImplementation library.java.hadoop_mapreduce_client_core
+ testImplementation library.java.hadoop_common
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
hadoopVersions.each{kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
diff --git a/sdks/java/extensions/sql/build.gradle
b/sdks/java/extensions/sql/build.gradle
index c722f3c..5cce997 100644
--- a/sdks/java/extensions/sql/build.gradle
+++ b/sdks/java/extensions/sql/build.gradle
@@ -53,8 +53,10 @@ def hadoopVersions = [
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
dependencies {
+ implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+
// TODO(BEAM-13179): Determine how to build without this dependency
- provided "org.immutables:value:2.8.8"
+ compileOnly "org.immutables:value:2.8.8"
javacc "net.java.dev.javacc:javacc:4.0"
fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
fmppTask "org.freemarker:freemarker:2.3.31"
@@ -77,23 +79,37 @@ dependencies {
implementation library.java.slf4j_api
implementation library.java.joda_time
implementation library.java.vendored_guava_26_0_jre
- provided project(":sdks:java:io:kafka")
- provided project(":sdks:java:io:google-cloud-platform")
+ compileOnly project(":sdks:java:io:kafka")
+ implementation project(":sdks:java:extensions:google-cloud-platform-core")
+ implementation project(":sdks:java:io:google-cloud-platform")
+ implementation library.java.proto_google_cloud_bigtable_v2
+ implementation library.java.google_api_services_bigquery
+ implementation library.java.proto_google_cloud_pubsublite_v1
+ implementation library.java.google_cloud_pubsublite
implementation project(":sdks:java:io:mongodb")
implementation library.java.avro
- provided project(":sdks:java:io:parquet")
- provided library.java.jackson_dataformat_xml
- provided library.java.hadoop_client
- provided library.java.kafka_clients
+ compileOnly project(":sdks:java:io:parquet")
+ compileOnly library.java.jackson_dataformat_xml
+ compileOnly library.java.hadoop_client
+ compileOnly library.java.kafka_clients
testImplementation library.java.vendored_calcite_1_28_0
testImplementation library.java.vendored_guava_26_0_jre
testImplementation library.java.junit
testImplementation library.java.quickcheck_core
testImplementation library.java.testcontainers_kafka
+ testImplementation library.java.google_cloud_bigtable_client_core
testImplementation library.java.google_cloud_bigtable_emulator
+ testImplementation library.java.proto_google_cloud_bigtable_admin_v2
+ testImplementation library.java.proto_google_cloud_datastore_v1
+ testImplementation library.java.google_cloud_datastore_v1_proto_client
+ testImplementation library.java.hadoop_client
+ testImplementation library.java.kafka_clients
+ testImplementation project(":sdks:java:io:kafka")
+ testImplementation project(":sdks:java:io:parquet")
testImplementation project(path: ":sdks:java:io:mongodb", configuration:
"testRuntimeMigration")
- testImplementation project(":sdks:java:io:thrift")
+ testImplementation project(path: ":sdks:java:io:thrift")
testImplementation project(path: ":sdks:java:io:thrift", configuration:
"testRuntimeMigration")
+ testImplementation project(":sdks:java:extensions:protobuf")
testImplementation project(path: ":sdks:java:extensions:protobuf",
configuration: "testRuntimeMigration")
testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider")
testRuntimeOnly library.java.slf4j_jdk14
diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle
b/sdks/java/extensions/sql/datacatalog/build.gradle
index 55f42b8..78dd98e 100644
--- a/sdks/java/extensions/sql/datacatalog/build.gradle
+++ b/sdks/java/extensions/sql/datacatalog/build.gradle
@@ -34,15 +34,23 @@ dependencies {
implementation library.java.protobuf_java
implementation library.java.slf4j_api
implementation library.java.vendored_guava_26_0_jre
+ implementation library.java.vendored_calcite_1_28_0
+ implementation "com.alibaba:fastjson:1.2.69"
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation "org.threeten:threetenbp:1.4.5"
- provided project(":sdks:java:extensions:sql")
+ compileOnly project(":sdks:java:extensions:sql")
// Dependencies for the example
- provided project(":sdks:java:io:google-cloud-platform")
- provided library.java.slf4j_api
+ implementation project(":sdks:java:extensions:google-cloud-platform-core")
+ compileOnly project(":sdks:java:io:google-cloud-platform")
+ compileOnly library.java.slf4j_api
+ testImplementation project(":sdks:java:extensions:sql")
testImplementation project(":sdks:java:extensions:sql:zetasql")
+ testImplementation project(":runners:direct-java")
+ testImplementation project(":sdks:java:io:google-cloud-platform")
+ testImplementation library.java.google_api_services_bigquery
+ testImplementation library.java.junit
testRuntimeOnly library.java.slf4j_simple
}
diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle
b/sdks/java/extensions/sql/hcatalog/build.gradle
index dd052ee..cb6380f 100644
--- a/sdks/java/extensions/sql/hcatalog/build.gradle
+++ b/sdks/java/extensions/sql/hcatalog/build.gradle
@@ -43,8 +43,11 @@ configurations.all {
}
dependencies {
- provided project(":sdks:java:extensions:sql")
- provided project(":sdks:java:io:hcatalog")
+ implementation project(":sdks:java:extensions:sql")
+ implementation project(":sdks:java:io:hcatalog")
+ implementation project(":sdks:java:core")
+ implementation "com.alibaba:fastjson:1.2.69"
+ implementation library.java.vendored_guava_26_0_jre
// Needed for HCatalogTableProvider tests,
// they use HCat* types
diff --git
a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java
b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java
index 706cf69..391016f8 100644
---
a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java
+++
b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java
@@ -22,6 +22,7 @@ import static
org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.TEST_RECO
import static
org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.TEST_TABLE;
import static
org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.getExpectedRecordsAsKV;
import static
org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.insertTestData;
+import static org.junit.Assume.assumeFalse;
import java.io.IOException;
import java.io.Serializable;
@@ -43,6 +44,7 @@ import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.Row;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang.SystemUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -68,6 +70,9 @@ public class BeamSqlHiveSchemaTest implements Serializable {
@BeforeClass
public static void setupEmbeddedMetastoreService() throws IOException {
+ // TODO(BEAM-13506): Remove this when hive version 4 is released and
includes
+ //
https://github.com/apache/hive/commit/a234475faa2cab2606f2a74eb9ca071f006998e2
+ assumeFalse(SystemUtils.isJavaVersionAtLeast(1.9f));
service = new
EmbeddedMetastoreService(TMP_FOLDER.getRoot().getAbsolutePath());
}
diff --git a/sdks/java/extensions/sql/jdbc/build.gradle
b/sdks/java/extensions/sql/jdbc/build.gradle
index 1e376b7..7d3158c 100644
--- a/sdks/java/extensions/sql/jdbc/build.gradle
+++ b/sdks/java/extensions/sql/jdbc/build.gradle
@@ -60,12 +60,7 @@ shadowJar {
}
}
-test {
- doFirst {
- // Assert everything is in the output or test jar
- assert classpath.size() == 2
- }
-}
+test {}
task shadowJarTest(type: Test, dependsOn:
":sdks:java:extensions:sql:jdbc:shadowJar") {
group = "Verification"
diff --git a/sdks/java/extensions/sql/zetasql/build.gradle
b/sdks/java/extensions/sql/zetasql/build.gradle
index 739fc8b..d6a84be 100644
--- a/sdks/java/extensions/sql/zetasql/build.gradle
+++ b/sdks/java/extensions/sql/zetasql/build.gradle
@@ -31,7 +31,7 @@ def zetasql_version = "2020.10.1"
dependencies {
// TODO(BEAM-13179): Determine how to build without this dependency
- provided "org.immutables:value:2.8.8"
+ compileOnly "org.immutables:value:2.8.8"
implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
permitUnusedDeclared
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation project(path: ":sdks:java:core", configuration: "shadow")
diff --git a/sdks/java/fn-execution/build.gradle
b/sdks/java/fn-execution/build.gradle
index 5e3faf6..e7ad5ec 100644
--- a/sdks/java/fn-execution/build.gradle
+++ b/sdks/java/fn-execution/build.gradle
@@ -33,7 +33,7 @@ dependencies {
implementation library.java.vendored_guava_26_0_jre
implementation library.java.slf4j_api
implementation library.java.joda_time
- provided library.java.junit
+ compileOnly library.java.junit
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.junit
testImplementation library.java.mockito_core
diff --git a/sdks/java/io/debezium/build.gradle
b/sdks/java/io/debezium/build.gradle
index 5655417..7f08371 100644
--- a/sdks/java/io/debezium/build.gradle
+++ b/sdks/java/io/debezium/build.gradle
@@ -36,7 +36,7 @@ dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation library.java.slf4j_api
implementation library.java.joda_time
- provided library.java.jackson_dataformat_csv
+ compileOnly library.java.jackson_dataformat_csv
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation project(path: ":sdks:java:io:common", configuration:
"testRuntimeMigration")
diff --git a/sdks/java/io/hadoop-common/build.gradle
b/sdks/java/io/hadoop-common/build.gradle
index 4cf57dc..a742bc9 100644
--- a/sdks/java/io/hadoop-common/build.gradle
+++ b/sdks/java/io/hadoop-common/build.gradle
@@ -35,10 +35,13 @@ hadoopVersions.each {kv ->
configurations.create("hadoopVersion$kv.key")}
dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
- provided library.java.hadoop_client
- provided library.java.hadoop_common
- provided library.java.hadoop_mapreduce_client_core
+ compileOnly library.java.hadoop_client
+ compileOnly library.java.hadoop_common
+ compileOnly library.java.hadoop_mapreduce_client_core
testImplementation library.java.junit
+ testImplementation library.java.hadoop_client
+ testImplementation library.java.hadoop_common
+ testImplementation library.java.hadoop_mapreduce_client_core
hadoopVersions.each {kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
diff --git a/sdks/java/io/hadoop-file-system/build.gradle
b/sdks/java/io/hadoop-file-system/build.gradle
index cbc8ea3..db1679d 100644
--- a/sdks/java/io/hadoop-file-system/build.gradle
+++ b/sdks/java/io/hadoop-file-system/build.gradle
@@ -40,9 +40,9 @@ dependencies {
implementation library.java.jackson_core
implementation library.java.jackson_databind
implementation library.java.slf4j_api
- provided library.java.hadoop_client
- provided library.java.hadoop_common
- provided library.java.hadoop_mapreduce_client_core
+ compileOnly library.java.hadoop_client
+ compileOnly library.java.hadoop_common
+ compileOnly library.java.hadoop_mapreduce_client_core
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.mockito_core
testImplementation library.java.junit
diff --git a/sdks/java/io/hadoop-format/build.gradle
b/sdks/java/io/hadoop-format/build.gradle
index 054362d..191e915 100644
--- a/sdks/java/io/hadoop-format/build.gradle
+++ b/sdks/java/io/hadoop-format/build.gradle
@@ -60,9 +60,9 @@ dependencies {
implementation library.java.slf4j_api
implementation project(":sdks:java:io:hadoop-common")
implementation library.java.joda_time
- provided library.java.hadoop_common
- provided library.java.hadoop_hdfs
- provided library.java.hadoop_mapreduce_client_core
+ compileOnly library.java.hadoop_common
+ compileOnly library.java.hadoop_hdfs
+ compileOnly library.java.hadoop_mapreduce_client_core
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation project(path: ":sdks:java:io:common", configuration:
"testRuntimeMigration")
testImplementation project(":sdks:java:testing:test-utils")
@@ -93,6 +93,9 @@ dependencies {
testImplementation library.java.cassandra_driver_core
testImplementation library.java.cassandra_driver_mapping
testImplementation "org.apache.cassandra:cassandra-all:3.11.8"
+ testImplementation library.java.hadoop_common
+ testImplementation library.java.hadoop_hdfs
+ testImplementation library.java.hadoop_mapreduce_client_core
testImplementation library.java.postgres
testImplementation library.java.junit
testImplementation library.java.hamcrest_core
diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle
index d40d250..92b9a40 100644
--- a/sdks/java/io/hbase/build.gradle
+++ b/sdks/java/io/hbase/build.gradle
@@ -41,7 +41,7 @@ dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation project(":sdks:java:io:hadoop-common")
implementation library.java.slf4j_api
- provided "org.apache.hbase:hbase-shaded-client:$hbase_version"
+ compileOnly "org.apache.hbase:hbase-shaded-client:$hbase_version"
testImplementation project(path: ":sdks:java:io:common", configuration:
"testRuntimeMigration")
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.junit
diff --git a/sdks/java/io/hcatalog/build.gradle
b/sdks/java/io/hcatalog/build.gradle
index d1e8eab..53711c6 100644
--- a/sdks/java/io/hcatalog/build.gradle
+++ b/sdks/java/io/hcatalog/build.gradle
@@ -72,14 +72,14 @@ dependencies {
implementation library.java.joda_time
// Hive bundles without repackaging Jackson which is why we redeclare it
here so that it appears
// on the compile/test/runtime classpath before Hive.
- provided library.java.jackson_annotations
- provided library.java.jackson_core
- provided library.java.jackson_databind
+ compileOnly library.java.jackson_annotations
+ compileOnly library.java.jackson_core
+ compileOnly library.java.jackson_databind
// Calcite (a dependency of Hive) bundles without repackaging Guava which is
why we redeclare it
// here so that it appears on the compile/test/runtime classpath before
Calcite.
- provided library.java.hadoop_common
- provided "org.apache.hive:hive-exec:$hive_version"
- provided(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core",
version: hive_version) {
+ compileOnly library.java.hadoop_common
+ compileOnly "org.apache.hive:hive-exec:$hive_version"
+ compileOnly(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core",
version: hive_version) {
exclude group: "org.apache.hive", module: "hive-exec"
exclude group: "com.google.protobuf", module: "protobuf-java"
}
@@ -92,6 +92,7 @@ dependencies {
testImplementation "org.apache.hive:hive-exec:$hive_version"
testImplementation "org.apache.hive:hive-common:$hive_version"
testImplementation "org.apache.hive:hive-cli:$hive_version"
+ testImplementation
"org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version"
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
hadoopVersions.each {kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle
index b361980..07ee6c1 100644
--- a/sdks/java/io/kafka/build.gradle
+++ b/sdks/java/io/kafka/build.gradle
@@ -52,8 +52,9 @@ dependencies {
implementation project(":sdks:java:expansion-service")
permitUnusedDeclared project(":sdks:java:expansion-service") // BEAM-11761
implementation library.java.avro
- // Get back to "provided" since 2.14
- provided library.java.kafka_clients
+ // Get back to "compileOnly" since 2.14
+ compileOnly library.java.kafka_clients
+ testImplementation library.java.kafka_clients
implementation library.java.slf4j_api
implementation library.java.joda_time
implementation library.java.jackson_annotations
@@ -63,17 +64,17 @@ dependencies {
// zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with
current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
- // "kafka-clients" has to be provided since user can use its own version.
+ // "kafka-clients" has to be compileOnly since user can use its own
version.
exclude group: "org.apache.kafka", module: "kafka-clients"
}
implementation ("io.confluent:kafka-schema-registry-client:5.3.2") {
// It depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
- // "kafka-clients" has to be provided since user can use its own version.
+ // "kafka-clients" has to be compileOnly since user can use its own
version.
exclude group: "org.apache.kafka", module: "kafka-clients"
}
- provided library.java.jackson_dataformat_csv
+ compileOnly library.java.jackson_dataformat_csv
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation project(":sdks:java:io:synthetic")
testImplementation project(path: ":sdks:java:io:common", configuration:
"testRuntimeMigration")
diff --git a/sdks/java/io/parquet/build.gradle
b/sdks/java/io/parquet/build.gradle
index b9fc223..7f10f92 100644
--- a/sdks/java/io/parquet/build.gradle
+++ b/sdks/java/io/parquet/build.gradle
@@ -47,7 +47,8 @@ dependencies {
implementation "org.apache.parquet:parquet-common:$parquet_version"
implementation "org.apache.parquet:parquet-hadoop:$parquet_version"
implementation library.java.avro
- provided library.java.hadoop_client
+ compileOnly library.java.hadoop_client
+ testImplementation library.java.hadoop_client
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.junit
testRuntimeOnly library.java.slf4j_jdk14
diff --git a/sdks/java/io/rabbitmq/build.gradle
b/sdks/java/io/rabbitmq/build.gradle
index e9392c3..49d516e 100644
--- a/sdks/java/io/rabbitmq/build.gradle
+++ b/sdks/java/io/rabbitmq/build.gradle
@@ -29,7 +29,7 @@ dependencies {
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation library.java.joda_time
implementation "com.rabbitmq:amqp-client:5.7.3"
- provided library.java.jackson_dataformat_xml
+ compileOnly library.java.jackson_dataformat_xml
testImplementation project(path: ":sdks:java:io:common", configuration:
"testRuntimeMigration")
testImplementation "org.apache.qpid:qpid-broker-core:$qpid_version"
testImplementation
"org.apache.qpid:qpid-broker-plugins-memory-store:$qpid_version"
diff --git a/sdks/java/testing/tpcds/build.gradle
b/sdks/java/testing/tpcds/build.gradle
index 6e4f27a..d0eeb9d 100644
--- a/sdks/java/testing/tpcds/build.gradle
+++ b/sdks/java/testing/tpcds/build.gradle
@@ -57,7 +57,7 @@ dependencies {
implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
// TODO(BEAM-13179): Determine how to build without this dependency
- provided "org.immutables:value:2.8.8"
+ compileOnly "org.immutables:value:2.8.8"
implementation library.java.avro
implementation library.java.vendored_guava_26_0_jre
implementation library.java.vendored_calcite_1_28_0
