Repository: beam Updated Branches: refs/heads/release-2.0.0 b9c8cfe22 -> 96aeb97cc
Cherry-pick pull request #2907 into release-2.0.0 branch Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/67ea7ae4 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/67ea7ae4 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/67ea7ae4 Branch: refs/heads/release-2.0.0 Commit: 67ea7ae4d2144525582c7de03b17d06daa9f35bb Parents: d2fa51b Author: Davor Bonaci <[email protected]> Authored: Fri May 5 16:20:28 2017 -0700 Committer: Dan Halperin <[email protected]> Committed: Fri May 5 17:26:24 2017 -0700 ---------------------------------------------------------------------- .../beam/runners/dataflow/options/DataflowPipelineOptions.java | 4 ++++ .../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java | 4 ++-- .../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index 11618af..4af420b 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -33,6 +33,8 @@ import org.apache.beam.sdk.options.Hidden; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.options.Validation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Options that can be used to configure the {@link DataflowRunner}. @@ -117,10 +119,12 @@ public interface DataflowPipelineOptions * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}. */ class StagingLocationFactory implements DefaultValueFactory<String> { + private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class); @Override public String create(PipelineOptions options) { GcsOptions gcsOptions = options.as(GcsOptions.class); + LOG.info("No stagingLocation provided, falling back to gcpTempLocation"); String gcpTempLocation; try { gcpTempLocation = gcsOptions.getGcpTempLocation(); http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java index 126b795..a4128e8 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java @@ -278,7 +278,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { } final String bucketName = "dataflow-staging-" + region + "-" + projectNumber; - LOG.info("No staging location provided, attempting to use default bucket: {}", + LOG.info("No tempLocation specified, attempting to use default bucket: {}", bucketName); Bucket bucket = new Bucket() .setName(bucketName) @@ -306,7 +306,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { throw new RuntimeException( "Unable to determine the owner of the default bucket at gs://" + bucketName, e); } - return "gs://" + bucketName; + return "gs://" + bucketName + "/temp/"; } /** http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java index 68b3818..3cf6980 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java @@ -220,7 +220,7 @@ public class GcpOptionsTest { when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L); String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient); - assertEquals("gs://dataflow-staging-us-north1-1", bucket); + assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket); } @Test
