Repository: beam Updated Branches: refs/heads/master f5ace5c6a -> ff589c711
Remove protobuf and http-client dependency from runners/google-cloud-dataflow Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aee91877 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aee91877 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aee91877 Branch: refs/heads/master Commit: aee9187746a1dc0f10865e803a9eb57bdf296686 Parents: f5ace5c Author: Vikas Kedigehalli <[email protected]> Authored: Wed May 10 17:39:01 2017 -0700 Committer: Luke Cwik <[email protected]> Committed: Thu May 11 10:21:47 2017 -0700 ---------------------------------------------------------------------- runners/google-cloud-dataflow-java/pom.xml | 11 ----------- .../beam/runners/dataflow/internal/CustomSources.java | 12 ------------ 2 files changed, 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/aee91877/runners/google-cloud-dataflow-java/pom.xml ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml index f679976..073cee8 100644 --- a/runners/google-cloud-dataflow-java/pom.xml +++ b/runners/google-cloud-dataflow-java/pom.xml @@ -294,12 +294,6 @@ </dependency> <dependency> - <groupId>com.google.http-client</groupId> - <artifactId>google-http-client-protobuf</artifactId> - <scope>runtime</scope> - </dependency> - - <dependency> <groupId>com.google.apis</groupId> <artifactId>google-api-services-dataflow</artifactId> </dependency> @@ -353,11 +347,6 @@ </dependency> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - - <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/aee91877/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java index 0d93566..ad2a666 100755 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java @@ -25,7 +25,6 @@ import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; import com.google.api.services.dataflow.model.SourceMetadata; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; import java.util.ArrayList; import java.util.List; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; @@ -47,20 +46,9 @@ import org.slf4j.LoggerFactory; public class CustomSources { private static final String SERIALIZED_SOURCE = "serialized_source"; @VisibleForTesting static final String SERIALIZED_SOURCE_SPLITS = "serialized_source_splits"; - /** - * The current limit on the size of a ReportWorkItemStatus RPC to Google Cloud Dataflow, which - * includes the initial splits, is 20 MB. - */ - public static final long DATAFLOW_SPLIT_RESPONSE_API_SIZE_BYTES = 20 * (1 << 20); private static final Logger LOG = LoggerFactory.getLogger(CustomSources.class); - private static final ByteString firstSplitKey = ByteString.copyFromUtf8("0000000000000001"); - - public static boolean isFirstUnboundedSourceSplit(ByteString splitKey) { - return splitKey.equals(firstSplitKey); - } - private static int getDesiredNumUnboundedSourceSplits(DataflowPipelineOptions options) { int cores = 4; //TODO: decide at runtime? if (options.getMaxNumWorkers() > 0) {
