Strip table decorators before creating tables.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0f50eb75 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0f50eb75 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0f50eb75 Branch: refs/heads/master Commit: 0f50eb759dd3c810f0ef70d66a8077df227cb372 Parents: 0a073af Author: Reuven Lax <re...@relax-macbookpro2.roam.corp.google.com> Authored: Tue Sep 19 11:38:13 2017 -0700 Committer: Reuven Lax <re...@relax-macbookpro2.roam.corp.google.com> Committed: Thu Sep 21 20:16:22 2017 -0700 ---------------------------------------------------------------------- .../apache/beam/sdk/io/gcp/bigquery/CreateTables.java | 2 +- .../beam/sdk/io/gcp/bigquery/TableDestination.java | 10 ++++++++++ .../apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 11 +++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java index 7f83b83..aff5ff1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java @@ -113,7 +113,7 @@ public class CreateTables<DestinationT> private void possibleCreateTable( BigQueryOptions options, TableDestination tableDestination, TableSchema tableSchema) throws InterruptedException, IOException { - String tableSpec = tableDestination.getTableSpec(); + String tableSpec = tableDestination.getStrippedTableSpec(); TableReference tableReference = tableDestination.getTableReference(); String tableDescription = tableDestination.getTableDescription(); if (createDisposition != createDisposition.CREATE_NEVER && !createdTables.contains(tableSpec)) { http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java index 79f1b22..4a4f66b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java @@ -30,6 +30,7 @@ import javax.annotation.Nullable; public class TableDestination implements Serializable { private static final long serialVersionUID = 1L; private final String tableSpec; + @Nullable private String strippedTableSpec; @Nullable private final String tableDescription; @Nullable @@ -59,6 +60,7 @@ public class TableDestination implements Serializable { public TableDestination(String tableSpec, @Nullable String tableDescription, @Nullable String jsonTimePartitioning) { this.tableSpec = tableSpec; + this.strippedTableSpec = null; this.tableDescription = tableDescription; this.jsonTimePartitioning = jsonTimePartitioning; } @@ -68,6 +70,14 @@ public class TableDestination implements Serializable { return tableSpec; } + public String getStrippedTableSpec() { + if (strippedTableSpec == null) { + int index = tableSpec.lastIndexOf('$'); + strippedTableSpec = (index == -1) ? tableSpec : tableSpec.substring(0, index); + } + return strippedTableSpec; + } + public TableReference getTableReference() { return BigQueryHelpers.parseTableSpec(tableSpec); } http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java index 9120507..7927282 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java @@ -2343,4 +2343,15 @@ public class BigQueryIOTest implements Serializable { } return converted; } + + @Test + public void testTableDecoratorStripping() { + TableDestination tableDestination = tableDestination = new TableDestination( + "project:dataset.table$decorator", ""); + assertEquals("project:dataset.table", tableDestination.getStrippedTableSpec()); + + tableDestination = new TableDestination("project:dataset.table", ""); + assertEquals("project:dataset.table", tableDestination.getStrippedTableSpec()); + + } }
