This is an automated email from the ASF dual-hosted git repository. goenka pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new d0f1968 [BEAM-7053] prevent errors in Spark options new 8334eb5 Merge pull request #8283 from ibzib/options-exception d0f1968 is described below commit d0f19682e62d090fe96175805198f198ed313391 Author: Kyle Weaver <kcwea...@google.com> AuthorDate: Wed Apr 10 14:22:25 2019 -0700 [BEAM-7053] prevent errors in Spark options --- .../main/java/org/apache/beam/runners/spark/SparkJobInvoker.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java index cea4b07..e47c851 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java @@ -56,6 +56,14 @@ public class SparkJobInvoker extends JobInvoker { String.format("%s_%s", sparkOptions.getJobName(), UUID.randomUUID().toString()); LOG.info("Invoking job {}", invocationId); + // Options can't be translated to proto if runner class is unresolvable, so set it to null. + sparkOptions.setRunner(null); + + if (sparkOptions.getAppName() == null) { + LOG.debug("App name was null. Using invocationId {}", invocationId); + sparkOptions.setAppName(invocationId); + } + return createJobInvocation( invocationId, retrievalToken, executorService, pipeline, sparkOptions); }