Repository: beam Updated Branches: refs/heads/master c2a555783 -> ba539b6ce
[BEAM-1871] Hide CloudResourceManager client in sdks/java/extensions/gcp-core Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0f010009 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0f010009 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0f010009 Branch: refs/heads/master Commit: 0f0100098f7a86fa0b34daaba03495a8110b6811 Parents: c2a5557 Author: Luke Cwik <lc...@google.com> Authored: Mon May 1 13:30:36 2017 -0700 Committer: Luke Cwik <lc...@google.com> Committed: Mon May 1 15:04:35 2017 -0700 ---------------------------------------------------------------------- .../sdk/extensions/gcp/options/GcpOptions.java | 42 +++++++++++++++++++- .../org/apache/beam/sdk/util/Transport.java | 21 ---------- .../extensions/gcp/GcpCoreApiSurfaceTest.java | 1 - .../gcp/options/GoogleApiDebugOptionsTest.java | 6 +-- 4 files changed, 43 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java index b2a83e9..445f00f 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java @@ -21,15 +21,19 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.util.BackOff; import com.google.api.client.util.Sleeper; import com.google.api.services.cloudresourcemanager.CloudResourceManager; import com.google.api.services.cloudresourcemanager.model.Project; import com.google.api.services.storage.model.Bucket; import com.google.auth.Credentials; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer; import com.google.cloud.hadoop.util.ResilientOperation; import com.google.cloud.hadoop.util.RetryDeterminer; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import java.io.File; import java.io.IOException; @@ -43,6 +47,7 @@ import java.util.regex.Pattern; import javax.annotation.Nullable; import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory; import org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory; +import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.Description; @@ -50,6 +55,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.InstanceBuilder; import org.apache.beam.sdk.util.PathValidator; +import org.apache.beam.sdk.util.RetryHttpRequestInitializer; import org.apache.beam.sdk.util.Transport; import org.apache.beam.sdk.util.gcsfs.GcsPath; import org.joda.time.Duration; @@ -229,8 +235,8 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { public String create(PipelineOptions options) { String tempLocation = options.getTempLocation(); if (isNullOrEmpty(tempLocation)) { - tempLocation = tryCreateDefaultBucket(options, Transport.newCloudResourceManagerClient( - options.as(CloudResourceManagerOptions.class)).build()); + tempLocation = tryCreateDefaultBucket(options, + newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)).build()); options.setTempLocation(tempLocation); } else { try { @@ -347,5 +353,37 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { checkArgument(zoneParts.length >= 2, "Invalid zone provided: %s", zone); return zoneParts[0] + "-" + zoneParts[1]; } + + /** + * Returns a CloudResourceManager client builder using the specified + * {@link CloudResourceManagerOptions}. + */ + @VisibleForTesting + static CloudResourceManager.Builder newCloudResourceManagerClient( + CloudResourceManagerOptions options) { + Credentials credentials = options.getGcpCredential(); + if (credentials == null) { + NullCredentialInitializer.throwNullCredentialException(); + } + return new CloudResourceManager.Builder(Transport.getTransport(), Transport.getJsonFactory(), + chainHttpRequestInitializer( + credentials, + // Do not log 404. It clutters the output and is possibly even required by the caller. + new RetryHttpRequestInitializer(ImmutableList.of(404)))) + .setApplicationName(options.getAppName()) + .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); + } + + private static HttpRequestInitializer chainHttpRequestInitializer( + Credentials credential, HttpRequestInitializer httpRequestInitializer) { + if (credential == null) { + return new ChainingHttpRequestInitializer( + new NullCredentialInitializer(), httpRequestInitializer); + } else { + return new ChainingHttpRequestInitializer( + new HttpCredentialsAdapter(credential), + httpRequestInitializer); + } + } } } http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java index 7903011..b8474bb 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java @@ -22,7 +22,6 @@ import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.cloudresourcemanager.CloudResourceManager; import com.google.api.services.storage.Storage; import com.google.auth.Credentials; import com.google.auth.http.HttpCredentialsAdapter; @@ -33,7 +32,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.security.GeneralSecurityException; import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; -import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; /** @@ -88,25 +86,6 @@ public class Transport { } /** - * Returns a CloudResourceManager client builder using the specified - * {@link CloudResourceManagerOptions}. - */ - public static CloudResourceManager.Builder - newCloudResourceManagerClient(CloudResourceManagerOptions options) { - Credentials credentials = options.getGcpCredential(); - if (credentials == null) { - NullCredentialInitializer.throwNullCredentialException(); - } - return new CloudResourceManager.Builder(getTransport(), getJsonFactory(), - chainHttpRequestInitializer( - credentials, - // Do not log 404. It clutters the output and is possibly even required by the caller. - new RetryHttpRequestInitializer(ImmutableList.of(404)))) - .setApplicationName(options.getAppName()) - .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); - } - - /** * Returns a Cloud Storage client builder using the specified {@link GcsOptions}. */ public static Storage.Builder http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java index 215b0f4..a8772c3 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java @@ -39,7 +39,6 @@ public class GcpCoreApiSurfaceTest { ImmutableSet.of( "org.apache.beam", "com.google.api.client", - "com.google.api.services.cloudresourcemanager", "com.google.api.services.storage", "com.google.auth", "com.fasterxml.jackson.annotation", http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java index 79d24f6..67d5880 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java @@ -113,9 +113,9 @@ public class GoogleApiDebugOptionsTest { Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); assertEquals("TraceDestination", getRequest.get("$trace")); - Delete deleteRequest = - Transport.newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)) - .build().projects().delete("testProjectId"); + Delete deleteRequest = GcpOptions.GcpTempLocationFactory.newCloudResourceManagerClient( + options.as(CloudResourceManagerOptions.class)) + .build().projects().delete("testProjectId"); assertNull(deleteRequest.get("$trace")); }
