johnjcasey commented on code in PR #27681: URL: https://github.com/apache/beam/pull/27681#discussion_r1287549221
########## sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14.java: ########## @@ -0,0 +1,647 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.ads.googleads.v14.errors.GoogleAdsError; +import com.google.ads.googleads.v14.errors.GoogleAdsException; +import com.google.ads.googleads.v14.errors.GoogleAdsFailure; +import com.google.ads.googleads.v14.errors.InternalErrorEnum; +import com.google.ads.googleads.v14.errors.QuotaErrorEnum; +import com.google.ads.googleads.v14.services.GoogleAdsRow; +import com.google.ads.googleads.v14.services.GoogleAdsServiceClient; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamRequest; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamResponse; +import com.google.auto.value.AutoValue; +import com.google.protobuf.Message; +import com.google.protobuf.util.Durations; +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.Optional; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.DoFn.ProcessContext; +import org.apache.beam.sdk.transforms.DoFn.ProcessElement; +import org.apache.beam.sdk.transforms.DoFn.Setup; +import org.apache.beam.sdk.transforms.DoFn.Teardown; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.display.DisplayData; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.sdk.util.BackOffUtils; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.sdk.util.Sleeper; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.RateLimiter; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; + +/** + * {@link GoogleAdsV14} provides an API to read Google Ads API v14 reports. + * + * <p>The Google Ads API does not use service account credentials in the same way as Google Cloud + * Platform APIs do. Service account credentials are typically only used to delegate (using + * domain-wide delegation) access through end user accounts. Providing credentials using the OAuth2 + * desktop flow may be preferable over domain wide delegation. Defaults for OAuth 2.0 credentials, + * refresh token and developer token can be provided using the following flags: + * + * <pre> + * --googleAdsClientId=your-client-id + * --googleAdsClientSecret=your-client-secret + * --googleAdsRefreshToken=your-refresh-token + * --googleAdsDeveloperToken=your-developer-token + * </pre> + * + * <p>Use {@link GoogleAdsV14#read()} to read a bounded {@link PCollection} of {@link GoogleAdsRow} + * from a query using {@link Read#withQuery(String)} and one or a few customer IDs using either + * {@link Read#withCustomerId(Long)} or {@link Read#withCustomerIds(List)}. Alternatively, use + * {@link GoogleAdsV14#readAll()} to read either a bounded or unbounded {@link PCollection} of + * {@link GoogleAdsRow} from a {@link PCollection} of {@link SearchGoogleAdsStreamRequest}. + * + * <p>For example, using {@link GoogleAdsV14#read()}: + * + * <pre>{@code + * Pipeline p = Pipeline.create(); + * PCollection<GoogleAdsRow> rows = + * p.apply( + * GoogleAdsIO.v14() + * .read() + * .withCustomerId(1234567890l) + * .withQuery( + * "SELECT" + * + "campaign.id," + * + "campaign.name," + * + "campaign.status" + * + "FROM campaign")); + * p.run(); + * }</pre> + * + * <p>Alternatively, using {@link GoogleAdsV14#readAll()} to execute requests from a {@link + * PCollection} of {@link SearchGoogleAdsStreamRequest}: + * + * <pre>{@code + * Pipeline p = Pipeline.create(); + * PCollection<SearchGoogleAdsStreamRequest> requests = + * p.apply( + * Create.of( + * ImmutableList.of( + * SearchGoogleAdsStreamRequest.newBuilder() + * .setCustomerId(Long.toString(1234567890l)) + * .setQuery( + * "SELECT" + * + "campaign.id," + * + "campaign.name," + * + "campaign.status" + * + "FROM campaign") + * .build()))); + * PCollection<GoogleAdsRow> rows = requests.apply(GoogleAdsIO.v14().readAll()); + * p.run(); + * }</pre> + * + * <h2>Client-side rate limiting</h2> + * + * On construction of a {@link GoogleAdsV14#read()} or {@link GoogleAdsV14#readAll()} transform a + * default rate limiting policy is provided to stay well under the rate limit for the Google Ads + * API, but this limit is only local to a single worker and operates without any knowledge of other + * applications using the same developer token for any customer ID. The Google Ads API enforces + * global limits from the developer token down to the customer ID and it is recommended to host a + * shared rate limiting service to coordinate traffic to the Google Ads API across all applications + * using the same developer token. Users of these transforms are strongly advised to implement their + * own {@link RateLimitPolicy} and {@link RateLimitPolicyFactory} to interact with a shared rate + * limiting service for any production workloads. + * + * @see GoogleAdsIO#v14() + * @see GoogleAdsOptions + * @see <a href="https://developers.google.com/google-ads/api/docs/best-practices/overview";>Best + * Practices in the Google Ads documentation</a> + */ +public class GoogleAdsV14 { Review Comment: That makes sense to me. It isn't ideal within the beam model, but it makes sense for this case -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
