Repository: beam Updated Branches: refs/heads/master 74f799509 -> 697b19fe5
[BEAM-1269] Update bigtable library dependency Enable cached data pools for efficiency and remove pegging data channel count to 1. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/badac8dc Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/badac8dc Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/badac8dc Branch: refs/heads/master Commit: badac8dc156d8c85d23802dd2092e9a8a4d1f6c1 Parents: 74f7995 Author: mdshalda <[email protected]> Authored: Thu Mar 30 14:06:03 2017 -0700 Committer: Dan Halperin <[email protected]> Committed: Wed Apr 5 16:42:43 2017 -0700 ---------------------------------------------------------------------- pom.xml | 6 ++--- sdks/java/io/google-cloud-platform/pom.xml | 24 ++------------------ .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 ++--- 3 files changed, 8 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/badac8dc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4ecfdd7..3f53b1f 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ <google-cloud-bigdataoss.version>1.4.5</google-cloud-bigdataoss.version> <google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version> <guava.version>20.0</guava.version> - <grpc.version>1.0.1</grpc.version> + <grpc.version>1.2.0</grpc.version> <grpc-google-common-protos.version>0.1.0</grpc-google-common-protos.version> <hamcrest.version>1.3</hamcrest.version> <jackson.version>2.7.2</jackson.version> @@ -127,9 +127,9 @@ <joda.version>2.4</joda.version> <junit.version>4.12</junit.version> <mockito.version>1.9.5</mockito.version> - <netty.version>4.1.6.Final</netty.version> + <netty.version>4.1.8.Final</netty.version> <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version> - <protobuf.version>3.1.0</protobuf.version> + <protobuf.version>3.2.0</protobuf.version> <pubsub.version>v1-rev10-1.22.0</pubsub.version> <slf4j.version>1.7.14</slf4j.version> <spark.version>1.6.2</spark.version> http://git-wip-us.apache.org/repos/asf/beam/blob/badac8dc/sdks/java/io/google-cloud-platform/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml index f784e27..a5b2e56 100644 --- a/sdks/java/io/google-cloud-platform/pom.xml +++ b/sdks/java/io/google-cloud-platform/pom.xml @@ -32,7 +32,7 @@ <packaging>jar</packaging> <properties> - <bigtable.version>0.9.5.1</bigtable.version> + <bigtable.version>0.9.6</bigtable.version> </properties> <build> @@ -81,26 +81,6 @@ </execution> </executions> </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <goals><goal>analyze-only</goal></goals> - <configuration> - <usedDependencies> - <!-- - BEAM-1632: Conflicting classes are brought in by bigtable-protos resulting in - maven-dependency:analyze `unused dependency` error. Marking this as a - `usedDependency` until the issue is resolved. - --> - <usedDependency>com.google.api.grpc:grpc-google-common-protos</usedDependency> - </usedDependencies> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> @@ -223,7 +203,7 @@ <dependency> <groupId>io.netty</groupId> <artifactId>netty-tcnative-boringssl-static</artifactId> - <version>1.1.33.Fork18</version> + <version>1.1.33.Fork26</version> <scope>runtime</scope> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/badac8dc/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java index 2cdd11d..89c67a4 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java @@ -212,9 +212,8 @@ public class BigtableIO { // TODO: is there a better way to clone a Builder? Want it to be immune from user changes. BigtableOptions options = optionsBuilder.build(); - // Set data channel count to one because there is only 1 scanner in this session BigtableOptions.Builder clonedBuilder = options.toBuilder() - .setDataChannelCount(1); + .setUseCachedDataPool(true); BigtableOptions optionsWithAgent = clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build(); @@ -449,7 +448,8 @@ public class BigtableIO { .setBulkOptions( options.getBulkOptions().toBuilder() .setUseBulkApi(true) - .build()); + .build()) + .setUseCachedDataPool(true); BigtableOptions optionsWithAgent = clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build(); return new Write(optionsWithAgent, tableId, bigtableService);
