BigQueryIO.Write: raise size limit to 11 TiB BigQuery has changed their total size quota to 12 TiB. https://cloud.google.com/bigquery/quota-policy#import
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/481a40f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/481a40f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/481a40f3 Branch: refs/heads/master Commit: 481a40f38ec32b7999c2fa29299542655f2d1cca Parents: 0f07241 Author: Dan Halperin <dhalp...@google.com> Authored: Wed Aug 24 09:49:46 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Wed Aug 24 15:48:41 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/481a40f3/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java index 04fb041..01a8a1c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java @@ -1420,8 +1420,8 @@ public class BigQueryIO { // Maximum number of files in a single partition. static final int MAX_NUM_FILES = 10000; - // Maximum number of bytes in a single partition. - static final long MAX_SIZE_BYTES = 3 * (1L << 40); + // Maximum number of bytes in a single partition -- 11 TiB just under BQ's 12 TiB limit. + static final long MAX_SIZE_BYTES = 11 * (1L << 40); // The maximum number of retry jobs. static final int MAX_RETRY_JOBS = 3;