http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GcsOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GcsOptions.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GcsOptions.java index e86f087..7cf695e 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GcsOptions.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GcsOptions.java @@ -19,21 +19,17 @@ package org.apache.beam.sdk.options; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.util.GcsPathValidator; import org.apache.beam.sdk.util.GcsUtil; -import org.apache.beam.sdk.util.InstanceBuilder; import org.apache.beam.sdk.util.PathValidator; /** * Options used to configure Google Cloud Storage. */ +@Deprecated public interface GcsOptions extends ApplicationNameOptions, GcpOptions, PipelineOptions { /** @@ -57,7 +53,8 @@ public interface GcsOptions extends + "to specify an ExecutorService that is compatible with the users environment. If unset, " + "the default is to create an ExecutorService with an unbounded number of threads; this " + "is compatible with Google AppEngine.") - @Default.InstanceFactory(ExecutorServiceFactory.class) + @Default.InstanceFactory( + org.apache.beam.sdk.extensions.gcp.options.GcsOptions.ExecutorServiceFactory.class) @Hidden ExecutorService getExecutorService(); void setExecutorService(ExecutorService value); @@ -106,49 +103,8 @@ public interface GcsOptions extends @Description("The path validator instance that should be used to validate paths. " + "If no path validator has been set explicitly, the default is to use the instance factory " + "that constructs a path validator based upon the currently set pathValidatorClass.") - @Default.InstanceFactory(PathValidatorFactory.class) + @Default.InstanceFactory( + org.apache.beam.sdk.extensions.gcp.options.GcsOptions.PathValidatorFactory.class) PathValidator getPathValidator(); void setPathValidator(PathValidator validator); - - /** - * Returns the default {@link ExecutorService} to use within the Apache Beam SDK. The - * {@link ExecutorService} is compatible with AppEngine. - */ - class ExecutorServiceFactory implements DefaultValueFactory<ExecutorService> { - @SuppressWarnings("deprecation") // IS_APP_ENGINE is deprecated for internal use only. - @Override - public ExecutorService create(PipelineOptions options) { - ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); - threadFactoryBuilder.setThreadFactory(MoreExecutors.platformThreadFactory()); - threadFactoryBuilder.setDaemon(true); - /* The SDK requires an unbounded thread pool because a step may create X writers - * each requiring their own thread to perform the writes otherwise a writer may - * block causing deadlock for the step because the writers buffer is full. - * Also, the MapTaskExecutor launches the steps in reverse order and completes - * them in forward order thus requiring enough threads so that each step's writers - * can be active. - */ - return new ThreadPoolExecutor( - 0, Integer.MAX_VALUE, // Allow an unlimited number of re-usable threads. - Long.MAX_VALUE, TimeUnit.NANOSECONDS, // Keep non-core threads alive forever. - new SynchronousQueue<Runnable>(), - threadFactoryBuilder.build()); - } - } - - /** - * Creates a {@link PathValidator} object using the class specified in - * {@link #getPathValidatorClass()}. - */ - class PathValidatorFactory implements DefaultValueFactory<PathValidator> { - @Override - public PathValidator create(PipelineOptions options) { - GcsOptions gcsOptions = options.as(GcsOptions.class); - return InstanceBuilder.ofType(PathValidator.class) - .fromClass(gcsOptions.getPathValidatorClass()) - .fromFactoryMethod("fromOptions") - .withArg(PipelineOptions.class, options) - .build(); - } - } }
http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GoogleApiDebugOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GoogleApiDebugOptions.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GoogleApiDebugOptions.java deleted file mode 100644 index f9cb575..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/options/GoogleApiDebugOptions.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.options; - -import com.google.api.client.googleapis.services.AbstractGoogleClient; -import com.google.api.client.googleapis.services.AbstractGoogleClientRequest; -import com.google.api.client.googleapis.services.GoogleClientRequestInitializer; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * These options configure debug settings for Google API clients created within the Apache Beam SDK. - */ -public interface GoogleApiDebugOptions extends PipelineOptions { - /** - * This option enables tracing of API calls to Google services used within the Apache - * Beam SDK. Values are expected in JSON format <code>{"ApiName":"TraceDestination",...} - * </code> where the {@code ApiName} represents the request classes canonical name. The - * {@code TraceDestination} is a logical trace consumer to whom the trace will be reported. - * Typically, "producer" is the right destination to use: this makes API traces available to the - * team offering the API. Note that by enabling this option, the contents of the requests to and - * from Google Cloud services will be made available to Google. For example, by specifying - * <code>{"Dataflow":"producer"}</code>, all calls to the Dataflow service will be made available - * to Google, specifically to the Google Cloud Dataflow team. - */ - @Description("This option enables tracing of API calls to Google services used within the Apache " - + "Beam SDK. Values are expected in JSON format {\"ApiName\":\"TraceDestination\",...} " - + "where the ApiName represents the request classes canonical name. The TraceDestination is " - + "a logical trace consumer to whom the trace will be reported. Typically, \"producer\" is " - + "the right destination to use: this makes API traces available to the team offering the " - + "API. Note that by enabling this option, the contents of the requests to and from " - + "Google Cloud services will be made available to Google. For example, by specifying " - + "{\"Dataflow\":\"producer\"}, all calls to the Dataflow service will be made available to " - + "Google, specifically to the Google Cloud Dataflow team.") - GoogleApiTracer getGoogleApiTrace(); - void setGoogleApiTrace(GoogleApiTracer commands); - - /** - * A {@link GoogleClientRequestInitializer} that adds the trace destination to Google API calls. - */ - class GoogleApiTracer extends HashMap<String, String> - implements GoogleClientRequestInitializer { - /** - * Creates a {@link GoogleApiTracer} that sets the trace destination on all - * calls that match the given client type. - */ - public GoogleApiTracer addTraceFor(AbstractGoogleClient client, String traceDestination) { - put(client.getClass().getCanonicalName(), traceDestination); - return this; - } - - /** - * Creates a {@link GoogleApiTracer} that sets the trace {@code traceDestination} on all - * calls that match for the given request type. - */ - public GoogleApiTracer addTraceFor( - AbstractGoogleClientRequest<?> request, String traceDestination) { - put(request.getClass().getCanonicalName(), traceDestination); - return this; - } - - @Override - public void initialize(AbstractGoogleClientRequest<?> request) throws IOException { - for (Map.Entry<String, String> entry : this.entrySet()) { - if (request.getClass().getCanonicalName().contains(entry.getKey())) { - request.set("$trace", entry.getValue()); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java deleted file mode 100644 index 6229650..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.auth.Credentials; -import java.io.IOException; -import java.security.GeneralSecurityException; - -/** - * Construct an oauth credential to be used by the SDK and the SDK workers. - */ -public interface CredentialFactory { - Credentials getCredential() throws IOException, GeneralSecurityException; -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/DefaultBucket.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/DefaultBucket.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/DefaultBucket.java index 75954c0..6e7298c 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/DefaultBucket.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/DefaultBucket.java @@ -24,8 +24,8 @@ import com.google.api.services.storage.model.Bucket; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; -import org.apache.beam.sdk.options.CloudResourceManagerOptions; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.gcsfs.GcsPath; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java deleted file mode 100644 index e1fa18f..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import org.apache.beam.sdk.options.PipelineOptions; - -/** - * Construct an oauth credential to be used by the SDK and the SDK workers. - * Returns a GCP credential. - */ -public class GcpCredentialFactory implements CredentialFactory { - /** - * The scope cloud-platform provides access to all Cloud Platform resources. - * cloud-platform isn't sufficient yet for talking to datastore so we request - * those resources separately. - * - * <p>Note that trusted scope relationships don't apply to OAuth tokens, so for - * services we access directly (GCS) as opposed to through the backend - * (BigQuery, GCE), we need to explicitly request that scope. - */ - private static final List<String> SCOPES = Arrays.asList( - "https://www.googleapis.com/auth/cloud-platform";, - "https://www.googleapis.com/auth/devstorage.full_control";, - "https://www.googleapis.com/auth/userinfo.email";, - "https://www.googleapis.com/auth/datastore";, - "https://www.googleapis.com/auth/pubsub";); - - private static final GcpCredentialFactory INSTANCE = new GcpCredentialFactory(); - - public static GcpCredentialFactory fromOptions(PipelineOptions options) { - return INSTANCE; - } - - /** - * Returns a default GCP {@link Credentials} or null when it fails. - */ - @Override - public Credentials getCredential() { - try { - return GoogleCredentials.getApplicationDefault().createScoped(SCOPES); - } catch (IOException e) { - // Ignore the exception - // Pipelines that only access to public data should be able to run without credentials. - return null; - } - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpProjectUtil.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpProjectUtil.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpProjectUtil.java index f73afe0..02b402a 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpProjectUtil.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcpProjectUtil.java @@ -25,7 +25,7 @@ import com.google.cloud.hadoop.util.ResilientOperation; import com.google.cloud.hadoop.util.RetryDeterminer; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; -import org.apache.beam.sdk.options.CloudResourceManagerOptions; +import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.PipelineOptions; import org.joda.time.Duration; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java index 745dcb9..3a12620 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java @@ -24,7 +24,7 @@ import java.nio.file.Path; import java.util.Collection; import java.util.LinkedList; import java.util.List; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.gcsfs.GcsPath; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java index f2e7a83..4d58424 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java @@ -20,7 +20,7 @@ package org.apache.beam.sdk.util; import static com.google.common.base.Preconditions.checkArgument; import java.io.IOException; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.gcsfs.GcsPath; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java index d0dfd3e..c8e6839 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java @@ -68,9 +68,8 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; - +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.DefaultValueFactory; -import org.apache.beam.sdk.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.gcsfs.GcsPath; import org.joda.time.Duration; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java deleted file mode 100644 index f703e4c..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.auth.Credentials; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.options.PipelineOptions; - -/** - * Construct an oauth credential to be used by the SDK and the SDK workers. - * Always returns a null Credential object. - */ -public class NoopCredentialFactory implements CredentialFactory { - private static final NoopCredentialFactory INSTANCE = new NoopCredentialFactory(); - private static final NoopCredentials NOOP_CREDENTIALS = new NoopCredentials(); - - public static NoopCredentialFactory fromOptions(PipelineOptions options) { - return INSTANCE; - } - - @Override - public Credentials getCredential() throws IOException { - return NOOP_CREDENTIALS; - } - - private static class NoopCredentials extends Credentials { - @Override - public String getAuthenticationType() { - return null; - } - - @Override - public Map<String, List<String>> getRequestMetadata(URI uri) throws IOException { - return null; - } - - @Override - public boolean hasRequestMetadata() { - return false; - } - - @Override - public boolean hasRequestMetadataOnly() { - return false; - } - - @Override - public void refresh() throws IOException {} - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NullCredentialInitializer.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NullCredentialInitializer.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NullCredentialInitializer.java deleted file mode 100644 index 4ed35c6..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/NullCredentialInitializer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.beam.sdk.util; - -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpUnsuccessfulResponseHandler; -import java.io.IOException; - -/** - * A {@link HttpRequestInitializer} for requests that don't have credentials. - * - * <p>When the access is denied, it throws {@link IOException} with a detailed error message. - */ -public class NullCredentialInitializer implements HttpRequestInitializer { - private static final int ACCESS_DENIED = 401; - private static final String NULL_CREDENTIAL_REASON = - "Unable to get application default credentials. Please see " - + "https://developers.google.com/accounts/docs/application-default-credentials " - + "for details on how to specify credentials. This version of the SDK is " - + "dependent on the gcloud core component version 2015.02.05 or newer to " - + "be able to get credentials from the currently authorized user via gcloud auth."; - - @Override - public void initialize(HttpRequest httpRequest) throws IOException { - httpRequest.setUnsuccessfulResponseHandler(new NullCredentialHttpUnsuccessfulResponseHandler()); - } - - private static class NullCredentialHttpUnsuccessfulResponseHandler - implements HttpUnsuccessfulResponseHandler { - - @Override - public boolean handleResponse( - HttpRequest httpRequest, - HttpResponse httpResponse, boolean supportsRetry) throws IOException { - if (!httpResponse.isSuccessStatusCode() && httpResponse.getStatusCode() == ACCESS_DENIED) { - throwNullCredentialException(); - } - return supportsRetry; - } - } - - public static void throwNullCredentialException() { - throw new RuntimeException(NULL_CREDENTIAL_REASON); - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/TestCredential.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/TestCredential.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/TestCredential.java deleted file mode 100644 index f34527e..0000000 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/TestCredential.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.auth.Credentials; -import java.io.IOException; -import java.net.URI; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Fake credential, for use in testing. - */ -public class TestCredential extends Credentials { - @Override - public String getAuthenticationType() { - return "Test"; - } - - @Override - public Map<String, List<String>> getRequestMetadata() throws IOException { - return Collections.emptyMap(); - } - - @Override - public Map<String, List<String>> getRequestMetadata(URI uri) throws IOException { - return Collections.emptyMap(); - } - - @Override - public boolean hasRequestMetadata() { - return false; - } - - @Override - public boolean hasRequestMetadataOnly() { - return true; - } - - @Override - public void refresh() throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java index 93d067a..7903011 100644 --- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java +++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java @@ -32,8 +32,9 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.security.GeneralSecurityException; -import org.apache.beam.sdk.options.CloudResourceManagerOptions; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; +import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; /** * Helpers for cloud communication. http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/GcpCoreApiSurfaceTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/GcpCoreApiSurfaceTest.java deleted file mode 100644 index 50edd83..0000000 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/GcpCoreApiSurfaceTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam; - -import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.common.collect.ImmutableSet; -import java.util.Set; -import org.apache.beam.sdk.util.ApiSurface; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** API surface verification for Google Cloud Platform core components. */ -@RunWith(JUnit4.class) -public class GcpCoreApiSurfaceTest { - - @Test - public void testApiSurface() throws Exception { - - @SuppressWarnings("unchecked") - final Set<String> allowed = - ImmutableSet.of( - "org.apache.beam", - "com.google.api.client", - "com.google.api.services.cloudresourcemanager", - "com.google.api.services.storage", - "com.google.auth", - "com.google.protobuf", - "com.fasterxml.jackson.annotation", - "com.fasterxml.jackson.core", - "com.fasterxml.jackson.databind", - "org.apache.avro", - "org.hamcrest", - // via DataflowMatchers - "org.codehaus.jackson", - // via Avro - "org.joda.time", - "org.junit"); - - assertThat( - ApiSurface.getSdkApiSurface(getClass().getClassLoader()), containsOnlyPackages(allowed)); - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java new file mode 100644 index 0000000..50edd83 --- /dev/null +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam; + +import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.apache.beam.sdk.util.ApiSurface; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** API surface verification for Google Cloud Platform core components. */ +@RunWith(JUnit4.class) +public class GcpCoreApiSurfaceTest { + + @Test + public void testApiSurface() throws Exception { + + @SuppressWarnings("unchecked") + final Set<String> allowed = + ImmutableSet.of( + "org.apache.beam", + "com.google.api.client", + "com.google.api.services.cloudresourcemanager", + "com.google.api.services.storage", + "com.google.auth", + "com.google.protobuf", + "com.fasterxml.jackson.annotation", + "com.fasterxml.jackson.core", + "com.fasterxml.jackson.databind", + "org.apache.avro", + "org.hamcrest", + // via DataflowMatchers + "org.codehaus.jackson", + // via Avro + "org.joda.time", + "org.junit"); + + assertThat( + ApiSurface.getSdkApiSurface(getClass().getClassLoader()), containsOnlyPackages(allowed)); + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/auth/TestCredential.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/auth/TestCredential.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/auth/TestCredential.java new file mode 100644 index 0000000..6f0846e --- /dev/null +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/auth/TestCredential.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.gcp.auth; + +import com.google.auth.Credentials; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Fake credential, for use in testing. + */ +public class TestCredential extends Credentials { + @Override + public String getAuthenticationType() { + return "Test"; + } + + @Override + public Map<String, List<String>> getRequestMetadata() throws IOException { + return Collections.emptyMap(); + } + + @Override + public Map<String, List<String>> getRequestMetadata(URI uri) throws IOException { + return Collections.emptyMap(); + } + + @Override + public boolean hasRequestMetadata() { + return false; + } + + @Override + public boolean hasRequestMetadataOnly() { + return true; + } + + @Override + public void refresh() throws IOException { + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java new file mode 100644 index 0000000..d334359 --- /dev/null +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.gcp.options; + +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions.DefaultProjectFactory; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.RestoreSystemProperties; +import org.apache.beam.sdk.util.NoopPathValidator; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link GcpOptions}. */ +@RunWith(JUnit4.class) +public class GcpOptionsTest { + @Rule public TestRule restoreSystemProperties = new RestoreSystemProperties(); + @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); + @Rule public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testGetProjectFromCloudSdkConfigEnv() throws Exception { + Map<String, String> environment = + ImmutableMap.of("CLOUDSDK_CONFIG", tmpFolder.getRoot().getAbsolutePath()); + assertEquals("test-project", + runGetProjectTest(tmpFolder.newFile("properties"), environment)); + } + + @Test + public void testGetProjectFromAppDataEnv() throws Exception { + Map<String, String> environment = + ImmutableMap.of("APPDATA", tmpFolder.getRoot().getAbsolutePath()); + System.setProperty("os.name", "windows"); + assertEquals("test-project", + runGetProjectTest(new File(tmpFolder.newFolder("gcloud"), "properties"), + environment)); + } + + @Test + public void testGetProjectFromUserHomeEnvOld() throws Exception { + Map<String, String> environment = ImmutableMap.of(); + System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); + assertEquals("test-project", + runGetProjectTest( + new File(tmpFolder.newFolder(".config", "gcloud"), "properties"), + environment)); + } + + @Test + public void testGetProjectFromUserHomeEnv() throws Exception { + Map<String, String> environment = ImmutableMap.of(); + System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); + assertEquals("test-project", + runGetProjectTest( + new File(tmpFolder.newFolder(".config", "gcloud", "configurations"), "config_default"), + environment)); + } + + @Test + public void testGetProjectFromUserHomeOldAndNewPrefersNew() throws Exception { + Map<String, String> environment = ImmutableMap.of(); + System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); + makePropertiesFileWithProject(new File(tmpFolder.newFolder(".config", "gcloud"), "properties"), + "old-project"); + assertEquals("test-project", + runGetProjectTest( + new File(tmpFolder.newFolder(".config", "gcloud", "configurations"), "config_default"), + environment)); + } + + @Test + public void testUnableToGetDefaultProject() throws Exception { + System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); + DefaultProjectFactory projectFactory = spy(new DefaultProjectFactory()); + when(projectFactory.getEnvironment()).thenReturn(ImmutableMap.<String, String>of()); + assertNull(projectFactory.create(PipelineOptionsFactory.create())); + } + + @Test + public void testEmptyGcpTempLocation() throws Exception { + GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); + options.setProject(""); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("--project is a required option"); + options.getGcpTempLocation(); + } + + @Test + public void testDefaultGcpTempLocation() throws Exception { + GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); + String tempLocation = "gs://bucket"; + options.setTempLocation(tempLocation); + options.as(GcsOptions.class).setPathValidatorClass(NoopPathValidator.class); + assertEquals(tempLocation, options.getGcpTempLocation()); + } + + @Test + public void testDefaultGcpTempLocationInvalid() throws Exception { + GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); + options.setTempLocation("file://"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage( + "Error constructing default value for gcpTempLocation: tempLocation is not" + + " a valid GCS path"); + options.getGcpTempLocation(); + } + + @Test + public void testDefaultGcpTempLocationDoesNotExist() { + GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); + String tempLocation = "gs://does/not/exist"; + options.setTempLocation(tempLocation); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage( + "Error constructing default value for gcpTempLocation: tempLocation is not" + + " a valid GCS path"); + thrown.expectCause( + hasMessage(containsString("Output path does not exist or is not writeable"))); + + options.getGcpTempLocation(); + } + + private static void makePropertiesFileWithProject(File path, String projectId) + throws IOException { + String properties = String.format("[core]%n" + + "account = [email protected]%n" + + "project = %s%n" + + "%n" + + "[dataflow]%n" + + "magic = true%n", projectId); + Files.write(properties, path, StandardCharsets.UTF_8); + } + + private static String runGetProjectTest(File path, Map<String, String> environment) + throws Exception { + makePropertiesFileWithProject(path, "test-project"); + DefaultProjectFactory projectFactory = spy(new DefaultProjectFactory()); + when(projectFactory.getEnvironment()).thenReturn(environment); + return projectFactory.create(PipelineOptionsFactory.create()); + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java new file mode 100644 index 0000000..79d24f6 --- /dev/null +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.gcp.options; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.services.cloudresourcemanager.CloudResourceManager.Projects.Delete; +import com.google.api.services.storage.Storage; +import org.apache.beam.sdk.extensions.gcp.auth.TestCredential; +import org.apache.beam.sdk.extensions.gcp.options.GoogleApiDebugOptions.GoogleApiTracer; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.util.Transport; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link GoogleApiDebugOptions}. */ +@RunWith(JUnit4.class) +public class GoogleApiDebugOptionsTest { + private static final String STORAGE_GET_TRACE = + "--googleApiTrace={\"Objects.Get\":\"GetTraceDestination\"}"; + private static final String STORAGE_GET_AND_LIST_TRACE = + "--googleApiTrace={\"Objects.Get\":\"GetTraceDestination\"," + + "\"Objects.List\":\"ListTraceDestination\"}"; + private static final String STORAGE_TRACE = "--googleApiTrace={\"Storage\":\"TraceDestination\"}"; + + @Test + public void testWhenTracingMatches() throws Exception { + String[] args = new String[] {STORAGE_GET_TRACE}; + GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + assertNotNull(options.getGoogleApiTrace()); + + Storage.Objects.Get request = + Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); + assertEquals("GetTraceDestination", request.get("$trace")); + } + + @Test + public void testWhenTracingDoesNotMatch() throws Exception { + String[] args = new String[] {STORAGE_GET_TRACE}; + GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + + assertNotNull(options.getGoogleApiTrace()); + + Storage.Objects.List request = + Transport.newStorageClient(options).build().objects().list("testProjectId"); + assertNull(request.get("$trace")); + } + + @Test + public void testWithMultipleTraces() throws Exception { + String[] args = new String[] {STORAGE_GET_AND_LIST_TRACE}; + GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + + assertNotNull(options.getGoogleApiTrace()); + + Storage.Objects.Get getRequest = + Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); + assertEquals("GetTraceDestination", getRequest.get("$trace")); + + Storage.Objects.List listRequest = + Transport.newStorageClient(options).build().objects().list("testProjectId"); + assertEquals("ListTraceDestination", listRequest.get("$trace")); + } + + @Test + public void testMatchingAllCalls() throws Exception { + String[] args = new String[] {STORAGE_TRACE}; + GcsOptions options = + PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + + assertNotNull(options.getGoogleApiTrace()); + + Storage.Objects.Get getRequest = + Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); + assertEquals("TraceDestination", getRequest.get("$trace")); + + Storage.Objects.List listRequest = + Transport.newStorageClient(options).build().objects().list("testProjectId"); + assertEquals("TraceDestination", listRequest.get("$trace")); + } + + @Test + public void testMatchingAgainstClient() throws Exception { + GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + options.setGoogleApiTrace(new GoogleApiTracer().addTraceFor( + Transport.newStorageClient(options).build(), "TraceDestination")); + + Storage.Objects.Get getRequest = + Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); + assertEquals("TraceDestination", getRequest.get("$trace")); + + Delete deleteRequest = + Transport.newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)) + .build().projects().delete("testProjectId"); + assertNull(deleteRequest.get("$trace")); + } + + @Test + public void testMatchingAgainstRequestType() throws Exception { + GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class); + options.setGcpCredential(new TestCredential()); + options.setGoogleApiTrace(new GoogleApiTracer().addTraceFor( + Transport.newStorageClient(options).build().objects() + .get("aProjectId", "aObjectId"), "TraceDestination")); + + Storage.Objects.Get getRequest = + Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); + assertEquals("TraceDestination", getRequest.get("$trace")); + + Storage.Objects.List listRequest = + Transport.newStorageClient(options).build().objects().list("testProjectId"); + assertNull(listRequest.get("$trace")); + } + + @Test + public void testDeserializationAndSerializationOfGoogleApiTracer() throws Exception { + String serializedValue = "{\"Api\":\"Token\"}"; + ObjectMapper objectMapper = new ObjectMapper(); + assertEquals(serializedValue, + objectMapper.writeValueAsString( + objectMapper.readValue(serializedValue, GoogleApiTracer.class))); + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java deleted file mode 100644 index 288383e..0000000 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.options; - -import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import com.google.common.io.Files; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import org.apache.beam.sdk.options.GcpOptions.DefaultProjectFactory; -import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.sdk.util.NoopPathValidator; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link GcpOptions}. */ -@RunWith(JUnit4.class) -public class GcpOptionsTest { - @Rule public TestRule restoreSystemProperties = new RestoreSystemProperties(); - @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Rule public ExpectedException thrown = ExpectedException.none(); - - @Test - public void testGetProjectFromCloudSdkConfigEnv() throws Exception { - Map<String, String> environment = - ImmutableMap.of("CLOUDSDK_CONFIG", tmpFolder.getRoot().getAbsolutePath()); - assertEquals("test-project", - runGetProjectTest(tmpFolder.newFile("properties"), environment)); - } - - @Test - public void testGetProjectFromAppDataEnv() throws Exception { - Map<String, String> environment = - ImmutableMap.of("APPDATA", tmpFolder.getRoot().getAbsolutePath()); - System.setProperty("os.name", "windows"); - assertEquals("test-project", - runGetProjectTest(new File(tmpFolder.newFolder("gcloud"), "properties"), - environment)); - } - - @Test - public void testGetProjectFromUserHomeEnvOld() throws Exception { - Map<String, String> environment = ImmutableMap.of(); - System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); - assertEquals("test-project", - runGetProjectTest( - new File(tmpFolder.newFolder(".config", "gcloud"), "properties"), - environment)); - } - - @Test - public void testGetProjectFromUserHomeEnv() throws Exception { - Map<String, String> environment = ImmutableMap.of(); - System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); - assertEquals("test-project", - runGetProjectTest( - new File(tmpFolder.newFolder(".config", "gcloud", "configurations"), "config_default"), - environment)); - } - - @Test - public void testGetProjectFromUserHomeOldAndNewPrefersNew() throws Exception { - Map<String, String> environment = ImmutableMap.of(); - System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); - makePropertiesFileWithProject(new File(tmpFolder.newFolder(".config", "gcloud"), "properties"), - "old-project"); - assertEquals("test-project", - runGetProjectTest( - new File(tmpFolder.newFolder(".config", "gcloud", "configurations"), "config_default"), - environment)); - } - - @Test - public void testUnableToGetDefaultProject() throws Exception { - System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); - DefaultProjectFactory projectFactory = spy(new DefaultProjectFactory()); - when(projectFactory.getEnvironment()).thenReturn(ImmutableMap.<String, String>of()); - assertNull(projectFactory.create(PipelineOptionsFactory.create())); - } - - @Test - public void testEmptyGcpTempLocation() throws Exception { - GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); - options.setProject(""); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("--project is a required option"); - options.getGcpTempLocation(); - } - - @Test - public void testDefaultGcpTempLocation() throws Exception { - GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); - String tempLocation = "gs://bucket"; - options.setTempLocation(tempLocation); - options.as(GcsOptions.class).setPathValidatorClass(NoopPathValidator.class); - assertEquals(tempLocation, options.getGcpTempLocation()); - } - - @Test - public void testDefaultGcpTempLocationInvalid() throws Exception { - GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); - options.setTempLocation("file://"); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage( - "Error constructing default value for gcpTempLocation: tempLocation is not" - + " a valid GCS path"); - options.getGcpTempLocation(); - } - - @Test - public void testDefaultGcpTempLocationDoesNotExist() { - GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class); - String tempLocation = "gs://does/not/exist"; - options.setTempLocation(tempLocation); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage( - "Error constructing default value for gcpTempLocation: tempLocation is not" - + " a valid GCS path"); - thrown.expectCause( - hasMessage(containsString("Output path does not exist or is not writeable"))); - - options.getGcpTempLocation(); - } - - private static void makePropertiesFileWithProject(File path, String projectId) - throws IOException { - String properties = String.format("[core]%n" - + "account = [email protected]%n" - + "project = %s%n" - + "%n" - + "[dataflow]%n" - + "magic = true%n", projectId); - Files.write(properties, path, StandardCharsets.UTF_8); - } - - private static String runGetProjectTest(File path, Map<String, String> environment) - throws Exception { - makePropertiesFileWithProject(path, "test-project"); - DefaultProjectFactory projectFactory = spy(new DefaultProjectFactory()); - when(projectFactory.getEnvironment()).thenReturn(environment); - return projectFactory.create(PipelineOptionsFactory.create()); - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GoogleApiDebugOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GoogleApiDebugOptionsTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GoogleApiDebugOptionsTest.java deleted file mode 100644 index 376972e..0000000 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/options/GoogleApiDebugOptionsTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.options; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.services.cloudresourcemanager.CloudResourceManager.Projects.Delete; -import com.google.api.services.storage.Storage; -import org.apache.beam.sdk.options.GoogleApiDebugOptions.GoogleApiTracer; -import org.apache.beam.sdk.util.TestCredential; -import org.apache.beam.sdk.util.Transport; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link GoogleApiDebugOptions}. */ -@RunWith(JUnit4.class) -public class GoogleApiDebugOptionsTest { - private static final String STORAGE_GET_TRACE = - "--googleApiTrace={\"Objects.Get\":\"GetTraceDestination\"}"; - private static final String STORAGE_GET_AND_LIST_TRACE = - "--googleApiTrace={\"Objects.Get\":\"GetTraceDestination\"," - + "\"Objects.List\":\"ListTraceDestination\"}"; - private static final String STORAGE_TRACE = "--googleApiTrace={\"Storage\":\"TraceDestination\"}"; - - @Test - public void testWhenTracingMatches() throws Exception { - String[] args = new String[] {STORAGE_GET_TRACE}; - GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - assertNotNull(options.getGoogleApiTrace()); - - Storage.Objects.Get request = - Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); - assertEquals("GetTraceDestination", request.get("$trace")); - } - - @Test - public void testWhenTracingDoesNotMatch() throws Exception { - String[] args = new String[] {STORAGE_GET_TRACE}; - GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - - assertNotNull(options.getGoogleApiTrace()); - - Storage.Objects.List request = - Transport.newStorageClient(options).build().objects().list("testProjectId"); - assertNull(request.get("$trace")); - } - - @Test - public void testWithMultipleTraces() throws Exception { - String[] args = new String[] {STORAGE_GET_AND_LIST_TRACE}; - GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - - assertNotNull(options.getGoogleApiTrace()); - - Storage.Objects.Get getRequest = - Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); - assertEquals("GetTraceDestination", getRequest.get("$trace")); - - Storage.Objects.List listRequest = - Transport.newStorageClient(options).build().objects().list("testProjectId"); - assertEquals("ListTraceDestination", listRequest.get("$trace")); - } - - @Test - public void testMatchingAllCalls() throws Exception { - String[] args = new String[] {STORAGE_TRACE}; - GcsOptions options = - PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - - assertNotNull(options.getGoogleApiTrace()); - - Storage.Objects.Get getRequest = - Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); - assertEquals("TraceDestination", getRequest.get("$trace")); - - Storage.Objects.List listRequest = - Transport.newStorageClient(options).build().objects().list("testProjectId"); - assertEquals("TraceDestination", listRequest.get("$trace")); - } - - @Test - public void testMatchingAgainstClient() throws Exception { - GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - options.setGoogleApiTrace(new GoogleApiTracer().addTraceFor( - Transport.newStorageClient(options).build(), "TraceDestination")); - - Storage.Objects.Get getRequest = - Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); - assertEquals("TraceDestination", getRequest.get("$trace")); - - Delete deleteRequest = - Transport.newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)) - .build().projects().delete("testProjectId"); - assertNull(deleteRequest.get("$trace")); - } - - @Test - public void testMatchingAgainstRequestType() throws Exception { - GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class); - options.setGcpCredential(new TestCredential()); - options.setGoogleApiTrace(new GoogleApiTracer().addTraceFor( - Transport.newStorageClient(options).build().objects() - .get("aProjectId", "aObjectId"), "TraceDestination")); - - Storage.Objects.Get getRequest = - Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); - assertEquals("TraceDestination", getRequest.get("$trace")); - - Storage.Objects.List listRequest = - Transport.newStorageClient(options).build().objects().list("testProjectId"); - assertNull(listRequest.get("$trace")); - } - - @Test - public void testDeserializationAndSerializationOfGoogleApiTracer() throws Exception { - String serializedValue = "{\"Api\":\"Token\"}"; - ObjectMapper objectMapper = new ObjectMapper(); - assertEquals(serializedValue, - objectMapper.writeValueAsString( - objectMapper.readValue(serializedValue, GoogleApiTracer.class))); - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/DefaultBucketTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/DefaultBucketTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/DefaultBucketTest.java index 395e1f3..65cb90b 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/DefaultBucketTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/DefaultBucketTest.java @@ -24,9 +24,9 @@ import static org.mockito.Mockito.when; import com.google.api.services.storage.model.Bucket; import java.io.IOException; -import org.apache.beam.sdk.options.CloudResourceManagerOptions; -import org.apache.beam.sdk.options.GcpOptions; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.gcsfs.GcsPath; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcpProjectUtilTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcpProjectUtilTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcpProjectUtilTest.java index 23f0418..253787d 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcpProjectUtilTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcpProjectUtilTest.java @@ -25,7 +25,8 @@ import com.google.api.client.util.BackOff; import com.google.api.services.cloudresourcemanager.CloudResourceManager; import com.google.api.services.cloudresourcemanager.model.Project; import java.net.SocketTimeoutException; -import org.apache.beam.sdk.options.CloudResourceManagerOptions; +import org.apache.beam.sdk.extensions.gcp.auth.TestCredential; +import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.FastNanoClockAndSleeper; import org.junit.Rule; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java index 7248b38..f53490a 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java @@ -19,7 +19,7 @@ package org.apache.beam.sdk.util; import static org.junit.Assert.assertEquals; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.junit.Before; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java index d4c804a..65fb228 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java @@ -20,7 +20,8 @@ package org.apache.beam.sdk.util; import static org.mockito.Matchers.any; import static org.mockito.Mockito.when; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.auth.TestCredential; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.gcsfs.GcsPath; import org.junit.Before; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java index 03668ce..6ffcaeb 100644 --- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java +++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java @@ -73,7 +73,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.auth.TestCredential; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.FastNanoClockAndSleeper; import org.apache.beam.sdk.util.GcsUtil.StorageObjectOrIOException; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java ---------------------------------------------------------------------- diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java index 738d6eb..d587986 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java @@ -31,7 +31,7 @@ import org.apache.beam.fn.harness.fn.ThrowingFunction; import org.apache.beam.fn.harness.logging.BeamFnLoggingClient; import org.apache.beam.fn.harness.stream.StreamObserverFactory; import org.apache.beam.fn.v1.BeamFnApi; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.IOChannelUtils; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java ---------------------------------------------------------------------- diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java index e1ec03d..c8d11ed 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java @@ -50,7 +50,7 @@ import java.util.logging.SimpleFormatter; import org.apache.beam.fn.v1.BeamFnApi; import org.apache.beam.fn.v1.BeamFnLoggingGrpc; import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; /** http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java ---------------------------------------------------------------------- diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java index 9326e11..063d5af 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.function.Function; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; /** http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java index 6a45647..d92ba72 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java @@ -36,7 +36,7 @@ import org.apache.beam.fn.v1.BeamFnApi.InstructionResponse; import org.apache.beam.fn.v1.BeamFnApi.LogControl; import org.apache.beam.fn.v1.BeamFnControlGrpc; import org.apache.beam.fn.v1.BeamFnLoggingGrpc; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml index 261d427..6023489 100644 --- a/sdks/java/io/google-cloud-platform/pom.xml +++ b/sdks/java/io/google-cloud-platform/pom.xml @@ -257,6 +257,13 @@ <dependency> <groupId>org.apache.beam</groupId> + <artifactId>beam-sdks-java-extensions-gcp-core</artifactId> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> <artifactId>beam-runners-direct-java</artifactId> <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java index 7ab0d73..7ade33f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java @@ -42,6 +42,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.VoidCoder; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.BeamJobUuidToBigQueryJobUuid; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreateJsonTableRefFromUuid; @@ -52,7 +53,6 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSche import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService; -import org.apache.beam.sdk.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java index 06bab00..d3116eb 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java @@ -17,10 +17,10 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.ApplicationNameOptions; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.StreamingOptions; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java index 0e0c365..8e395f0 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java @@ -61,11 +61,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; -import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.sdk.util.NullCredentialInitializer; import org.apache.beam.sdk.util.RetryHttpRequestInitializer; import org.apache.beam.sdk.util.Transport; import org.joda.time.Duration; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java index 503be18..1ca7460 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java @@ -44,13 +44,13 @@ import java.util.concurrent.ConcurrentLinkedQueue; import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.io.BoundedSource.BoundedReader; import org.apache.beam.sdk.io.range.ByteKey; import org.apache.beam.sdk.io.range.ByteKeyRange; import org.apache.beam.sdk.io.range.ByteKeyRangeTracker; -import org.apache.beam.sdk.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.runners.PipelineRunner; import org.apache.beam.sdk.transforms.DoFn; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java index 98787fa..aa0019c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java @@ -72,7 +72,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.annotations.Experimental.Kind; import org.apache.beam.sdk.coders.SerializableCoder; -import org.apache.beam.sdk.options.GcpOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java index 1d02a1e..16de648 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java @@ -61,7 +61,7 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; -import org.apache.beam.sdk.options.GcpOptions; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; /** * A helper class for talking to Pubsub via grpc. http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubOptions.java index 6158584..66c9fb4 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubOptions.java @@ -17,10 +17,10 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.ApplicationNameOptions; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.GcpOptions; import org.apache.beam.sdk.options.Hidden; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.StreamingOptions; http://git-wip-us.apache.org/repos/asf/beam/blob/0aed801a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java index fac1db3..2ae6b7e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java @@ -37,12 +37,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.regex.Pattern; +import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.io.FileSystem; import org.apache.beam.sdk.io.fs.CreateOptions; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.MatchResult.Status; -import org.apache.beam.sdk.options.GcsOptions; import org.apache.beam.sdk.util.GcsUtil; import org.apache.beam.sdk.util.GcsUtil.StorageObjectOrIOException; import org.apache.beam.sdk.util.gcsfs.GcsPath;
