Repository: incubator-beam Updated Branches: refs/heads/master e2868d687 -> a5d129361
Converts Datastore to use AutoValue Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/251e3e1e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/251e3e1e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/251e3e1e Branch: refs/heads/master Commit: 251e3e1ef2868b9901be3dbd27793b613855704a Parents: e2868d6 Author: Eugene Kirpichov <kirpic...@google.com> Authored: Wed Sep 28 17:29:11 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Wed Oct 12 17:12:32 2016 -0700 ---------------------------------------------------------------------- sdks/java/io/google-cloud-platform/pom.xml | 6 ++ .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 95 +++++++------------- 2 files changed, 38 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/251e3e1e/sdks/java/io/google-cloud-platform/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml index 15c0447..1a598bb 100644 --- a/sdks/java/io/google-cloud-platform/pom.xml +++ b/sdks/java/io/google-cloud-platform/pom.xml @@ -213,6 +213,12 @@ <artifactId>avro</artifactId> </dependency> + <dependency> + <groupId>com.google.auto.value</groupId> + <artifactId>auto-value</artifactId> + <scope>provided</scope> + </dependency> + <!-- test --> <dependency> <groupId>org.apache.beam</groupId> http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/251e3e1e/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java index 45b2d6f..bfbff32 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java @@ -35,6 +35,7 @@ import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.util.BackOff; import com.google.api.client.util.BackOffUtils; import com.google.api.client.util.Sleeper; +import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; @@ -194,7 +195,7 @@ public class DatastoreV1 { * {@link DatastoreV1.Read#withNamespace}, {@link DatastoreV1.Read#withNumQuerySplits}. */ public DatastoreV1.Read read() { - return new DatastoreV1.Read(null, null, null, 0); + return new AutoValue_DatastoreV1_Read.Builder().setNumQuerySplits(0).build(); } /** @@ -203,7 +204,8 @@ public class DatastoreV1 { * * @see DatastoreIO */ - public static class Read extends PTransform<PBegin, PCollection<Entity>> { + @AutoValue + public abstract static class Read extends PTransform<PBegin, PCollection<Entity>> { private static final Logger LOG = LoggerFactory.getLogger(Read.class); /** An upper bound on the number of splits for a query. */ @@ -222,16 +224,23 @@ public class DatastoreV1 { */ static final int QUERY_BATCH_LIMIT = 500; - @Nullable - private final String projectId; + @Nullable public abstract String getProjectId(); + @Nullable public abstract Query getQuery(); + @Nullable public abstract String getNamespace(); + public abstract int getNumQuerySplits(); - @Nullable - private final Query query; + public abstract String toString(); - @Nullable - private final String namespace; + abstract Builder toBuilder(); - private final int numQuerySplits; + @AutoValue.Builder + abstract static class Builder { + abstract Builder setProjectId(String projectId); + abstract Builder setQuery(Query query); + abstract Builder setNamespace(String namespace); + abstract Builder setNumQuerySplits(int numQuerySplits); + abstract Read build(); + } /** * Computes the number of splits to be performed on the given query by querying the estimated @@ -345,25 +354,12 @@ public class DatastoreV1 { } /** - * Note that only {@code namespace} is really {@code @Nullable}. The other parameters may be - * {@code null} as a matter of build order, but if they are {@code null} at instantiation time, - * an error will be thrown. - */ - private Read(@Nullable String projectId, @Nullable Query query, @Nullable String namespace, - int numQuerySplits) { - this.projectId = projectId; - this.query = query; - this.namespace = namespace; - this.numQuerySplits = numQuerySplits; - } - - /** * Returns a new {@link DatastoreV1.Read} that reads from the Cloud Datastore for the specified * project. */ public DatastoreV1.Read withProjectId(String projectId) { checkNotNull(projectId, "projectId"); - return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits); + return toBuilder().setProjectId(projectId).build(); } /** @@ -378,14 +374,14 @@ public class DatastoreV1 { checkNotNull(query, "query"); checkArgument(!query.hasLimit() || query.getLimit().getValue() > 0, "Invalid query limit %s: must be positive", query.getLimit().getValue()); - return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits); + return toBuilder().setQuery(query).build(); } /** * Returns a new {@link DatastoreV1.Read} that reads from the given namespace. */ public DatastoreV1.Read withNamespace(String namespace) { - return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits); + return toBuilder().setNamespace(namespace).build(); } /** @@ -405,29 +401,11 @@ public class DatastoreV1 { * </ul> */ public DatastoreV1.Read withNumQuerySplits(int numQuerySplits) { - return new DatastoreV1.Read(projectId, query, namespace, - Math.min(Math.max(numQuerySplits, 0), NUM_QUERY_SPLITS_MAX)); - } - - @Nullable - public Query getQuery() { - return query; - } - - @Nullable - public String getProjectId() { - return projectId; + return toBuilder() + .setNumQuerySplits(Math.min(Math.max(numQuerySplits, 0), NUM_QUERY_SPLITS_MAX)) + .build(); } - @Nullable - public String getNamespace() { - return namespace; - } - - - /** - * {@inheritDoc} - */ @Override public PCollection<Entity> apply(PBegin input) { V1Options v1Options = V1Options.from(getProjectId(), getQuery(), @@ -451,8 +429,8 @@ public class DatastoreV1 { * a {@code PCollection<Entity>}. */ PCollection<KV<Integer, Query>> queries = input - .apply(Create.of(query)) - .apply(ParDo.of(new SplitQueryFn(v1Options, numQuerySplits))); + .apply(Create.of(getQuery())) + .apply(ParDo.of(new SplitQueryFn(v1Options, getNumQuerySplits()))); PCollection<Query> shardedQueries = queries .apply(GroupByKey.<Integer, Query>create()) @@ -467,31 +445,22 @@ public class DatastoreV1 { @Override public void validate(PBegin input) { - checkNotNull(projectId, "projectId"); - checkNotNull(query, "query"); + checkNotNull(getProjectId(), "projectId"); + checkNotNull(getQuery(), "query"); } @Override public void populateDisplayData(DisplayData.Builder builder) { super.populateDisplayData(builder); builder - .addIfNotNull(DisplayData.item("projectId", projectId) + .addIfNotNull(DisplayData.item("projectId", getProjectId()) .withLabel("ProjectId")) - .addIfNotNull(DisplayData.item("namespace", namespace) + .addIfNotNull(DisplayData.item("namespace", getNamespace()) .withLabel("Namespace")) - .addIfNotNull(DisplayData.item("query", query.toString()) + .addIfNotNull(DisplayData.item("query", getQuery().toString()) .withLabel("Query")); } - @Override - public String toString() { - return MoreObjects.toStringHelper(getClass()) - .add("projectId", projectId) - .add("query", query) - .add("namespace", namespace) - .toString(); - } - /** * A class for v1 Cloud Datastore related options. */ @@ -596,7 +565,7 @@ public class DatastoreV1 { } @Override - public void populateDisplayData(Builder builder) { + public void populateDisplayData(DisplayData.Builder builder) { super.populateDisplayData(builder); builder .addIfNotNull(DisplayData.item("projectId", options.getProjectId())
