Repository: beam Updated Branches: refs/heads/release-2.0.0 66b3ce075 -> 16d281c05
Remove protobuf and http-client dependency from runners/google-cloud-dataflow Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/998e42f9 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/998e42f9 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/998e42f9 Branch: refs/heads/release-2.0.0 Commit: 998e42f90c58726f31be5621369784532d2a67ff Parents: 66b3ce0 Author: Vikas Kedigehalli <[email protected]> Authored: Wed May 10 17:39:01 2017 -0700 Committer: Vikas Kedigehalli <[email protected]> Committed: Thu May 11 10:45:08 2017 -0700 ---------------------------------------------------------------------- runners/google-cloud-dataflow-java/pom.xml | 11 ----------- .../beam/runners/dataflow/internal/CustomSources.java | 12 ------------ 2 files changed, 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/998e42f9/runners/google-cloud-dataflow-java/pom.xml ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml index 9dece95..a0c5ac7 100644 --- a/runners/google-cloud-dataflow-java/pom.xml +++ b/runners/google-cloud-dataflow-java/pom.xml @@ -295,12 +295,6 @@ </dependency> <dependency> - <groupId>com.google.http-client</groupId> - <artifactId>google-http-client-protobuf</artifactId> - <scope>runtime</scope> - </dependency> - - <dependency> <groupId>com.google.apis</groupId> <artifactId>google-api-services-dataflow</artifactId> </dependency> @@ -354,11 +348,6 @@ </dependency> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - - <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/998e42f9/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java index 0d93566..ad2a666 100755 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java @@ -25,7 +25,6 @@ import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; import com.google.api.services.dataflow.model.SourceMetadata; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; import java.util.ArrayList; import java.util.List; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; @@ -47,20 +46,9 @@ import org.slf4j.LoggerFactory; public class CustomSources { private static final String SERIALIZED_SOURCE = "serialized_source"; @VisibleForTesting static final String SERIALIZED_SOURCE_SPLITS = "serialized_source_splits"; - /** - * The current limit on the size of a ReportWorkItemStatus RPC to Google Cloud Dataflow, which - * includes the initial splits, is 20 MB. - */ - public static final long DATAFLOW_SPLIT_RESPONSE_API_SIZE_BYTES = 20 * (1 << 20); private static final Logger LOG = LoggerFactory.getLogger(CustomSources.class); - private static final ByteString firstSplitKey = ByteString.copyFromUtf8("0000000000000001"); - - public static boolean isFirstUnboundedSourceSplit(ByteString splitKey) { - return splitKey.equals(firstSplitKey); - } - private static int getDesiredNumUnboundedSourceSplits(DataflowPipelineOptions options) { int cores = 4; //TODO: decide at runtime? if (options.getMaxNumWorkers() > 0) {
