Repository: beam Updated Branches: refs/heads/master 060889701 -> e9190834b
Fix default temp location for DataflowRunner Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5fdb4055 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5fdb4055 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5fdb4055 Branch: refs/heads/master Commit: 5fdb40554c7625a6e1ecfc5a8516885d112ae1d0 Parents: 0608897 Author: Vikas Kedigehalli <[email protected]> Authored: Thu May 4 18:30:35 2017 -0700 Committer: Davor Bonaci <[email protected]> Committed: Fri May 5 16:20:17 2017 -0700 ---------------------------------------------------------------------- .../beam/runners/dataflow/options/DataflowPipelineOptions.java | 4 ++++ .../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java | 4 ++-- .../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index 11618af..4af420b 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -33,6 +33,8 @@ import org.apache.beam.sdk.options.Hidden; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.options.Validation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Options that can be used to configure the {@link DataflowRunner}. @@ -117,10 +119,12 @@ public interface DataflowPipelineOptions * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}. */ class StagingLocationFactory implements DefaultValueFactory<String> { + private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class); @Override public String create(PipelineOptions options) { GcsOptions gcsOptions = options.as(GcsOptions.class); + LOG.info("No stagingLocation provided, falling back to gcpTempLocation"); String gcpTempLocation; try { gcpTempLocation = gcsOptions.getGcpTempLocation(); http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java index 126b795..a4128e8 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java @@ -278,7 +278,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { } final String bucketName = "dataflow-staging-" + region + "-" + projectNumber; - LOG.info("No staging location provided, attempting to use default bucket: {}", + LOG.info("No tempLocation specified, attempting to use default bucket: {}", bucketName); Bucket bucket = new Bucket() .setName(bucketName) @@ -306,7 +306,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { throw new RuntimeException( "Unable to determine the owner of the default bucket at gs://" + bucketName, e); } - return "gs://" + bucketName; + return "gs://" + bucketName + "/temp/"; } /** http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java index 68b3818..3cf6980 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java @@ -220,7 +220,7 @@ public class GcpOptionsTest { when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L); String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient); - assertEquals("gs://dataflow-staging-us-north1-1", bucket); + assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket); } @Test
