This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to
refs/heads/spark-runner_structured-streaming by this push:
new 20fad2c [TO REVERT] Change de wordcount build to test on new spark
runner
20fad2c is described below
commit 20fad2c96b31fdd3dcdbc1402dd199d8cb1b1975
Author: Etienne Chauchot <[email protected]>
AuthorDate: Thu Feb 28 14:35:05 2019 +0100
[TO REVERT] Change de wordcount build to test on new spark runner
---
examples/java/build.gradle | 35 ++++++++++++++++++++---------------
1 file changed, 20 insertions(+), 15 deletions(-)
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
index 7b7fe3e..69b4a9d 100644
--- a/examples/java/build.gradle
+++ b/examples/java/build.gradle
@@ -37,18 +37,23 @@ def preCommitRunners = ["directRunner", "flinkRunner",
"sparkRunner"]
for (String runner : preCommitRunners) {
configurations.create(runner + "PreCommit")
}
-configurations.sparkRunnerPreCommit {
- // Ban certain dependencies to prevent a StackOverflow within Spark
- // because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
- exclude group: "org.slf4j", module: "jul-to-slf4j"
- exclude group: "org.slf4j", module: "slf4j-jdk14"
-}
-
dependencies {
compile library.java.guava
shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
shadow project(path:
":beam-sdks-java-extensions-google-cloud-platform-core", configuration:
"shadow")
shadow project(path: ":beam-sdks-java-io-google-cloud-platform",
configuration: "shadow")
+ shadow project(path: ":beam-runners-spark-structured-streaming",
configuration: "shadow")
+ shadow library.java.guava
+ shadow library.java.slf4j_api
+ shadow library.java.joda_time
+ provided library.java.spark_sql
+ provided library.java.commons_compress
+ provided library.java.commons_lang3
+ provided library.java.commons_io_2x
+ provided library.java.hamcrest_core
+ provided library.java.hamcrest_library
+ shadow "com.fasterxml.jackson.module:jackson-module-scala_2.11:2.9.8"
+
shadow library.java.google_api_client
shadow library.java.google_api_services_bigquery
shadow library.java.google_http_client
@@ -79,13 +84,7 @@ dependencies {
// apexRunnerPreCommit project(path: ":beam-runners-apex", configuration:
"shadow")
directRunnerPreCommit project(path: ":beam-runners-direct-java",
configuration: "shadow")
flinkRunnerPreCommit project(path: ":beam-runners-flink_2.11",
configuration: "shadow")
- // TODO: Make the netty version used configurable, we add netty-all
4.1.17.Final so it appears on the classpath
- // before 4.1.8.Final defined by Apache Beam
- sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final"
- sparkRunnerPreCommit project(path: ":beam-runners-spark", configuration:
"shadow")
- sparkRunnerPreCommit project(path: ":beam-sdks-java-io-hadoop-file-system",
configuration: "shadow")
- sparkRunnerPreCommit library.java.spark_streaming
- sparkRunnerPreCommit library.java.spark_core
+ sparkRunnerPreCommit project(path:
":beam-runners-spark-structured-streaming", configuration: "shadow")
}
/*
@@ -96,7 +95,7 @@ def preCommitRunnerClass = [
apexRunner: "org.apache.beam.runners.apex.TestApexRunner",
directRunner: "org.apache.beam.runners.direct.DirectRunner",
flinkRunner: "org.apache.beam.runners.flink.TestFlinkRunner",
- sparkRunner: "org.apache.beam.runners.spark.TestSparkRunner",
+ sparkRunner: "org.apache.beam.runners.spark.structuredstreaming.SparkRunner",
]
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests/'
@@ -124,5 +123,11 @@ task preCommit() {
for (String runner : preCommitRunners) {
dependsOn runner + "PreCommit"
}
+
+ configurations.all {
+ // Ban certain dependencies to prevent a StackOverflow within Spark
+ // because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
+ exclude group: "org.slf4j", module: "slf4j-jdk14"
+ }
}
