[
https://issues.apache.org/jira/browse/BEAM-4229?focusedWorklogId=98679&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98679
]
ASF GitHub Bot logged work on BEAM-4229:
----------------------------------------
Author: ASF GitHub Bot
Created on: 05/May/18 07:15
Start Date: 05/May/18 07:15
Worklog Time Spent: 10m
Work Description: chamikaramj closed pull request #5286: [BEAM-4229]
Revert BigQuery SQL support so Spanner works
URL: https://github.com/apache/beam/pull/5286
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/build_rules.gradle b/build_rules.gradle
index 37570f044c3..1ab17ddb954 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -234,7 +234,6 @@ ext.library = [
google_api_services_storage:
"com.google.apis:google-api-services-storage:v1-rev124-$google_clients_version",
google_auth_library_credentials:
"com.google.auth:google-auth-library-credentials:$google_auth_version",
google_auth_library_oauth2_http:
"com.google.auth:google-auth-library-oauth2-http:$google_auth_version",
- google_cloud_bigquery:
"com.google.cloud:google-cloud-bigquery:$google_clients_version",
google_cloud_core: "com.google.cloud:google-cloud-core:1.0.2",
google_cloud_core_grpc:
"com.google.cloud:google-cloud-core-grpc:$grpc_version",
google_cloud_dataflow_java_proto_library_all:
"com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304",
diff --git a/pom.xml b/pom.xml
index fa925131441..987291c261a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1120,12 +1120,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-bigquery</artifactId>
- <version>${google-clients.version}</version>
- </dependency>
-
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-core-grpc</artifactId>
diff --git a/sdks/java/extensions/sql/build.gradle
b/sdks/java/extensions/sql/build.gradle
index dcf98586a14..02955a0d2ff 100644
--- a/sdks/java/extensions/sql/build.gradle
+++ b/sdks/java/extensions/sql/build.gradle
@@ -66,7 +66,6 @@ dependencies {
shadow library.java.joda_time
shadow project(path: ":beam-runners-direct-java", configuration: "shadow")
provided project(path: ":beam-sdks-java-io-kafka", configuration: "shadow")
- provided project(path: ":beam-sdks-java-io-google-cloud-platform",
configuration: "shadow")
provided library.java.kafka_clients
testCompile library.java.slf4j_jdk14
testCompile library.java.junit
diff --git a/sdks/java/extensions/sql/pom.xml b/sdks/java/extensions/sql/pom.xml
index 586847eaf4e..61fcb319f68 100644
--- a/sdks/java/extensions/sql/pom.xml
+++ b/sdks/java/extensions/sql/pom.xml
@@ -396,12 +396,6 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
- <scope>provided</scope>
- </dependency>
-
<!-- for tests -->
<dependency>
<groupId>junit</groupId>
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQueryTable.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQueryTable.java
deleted file mode 100644
index 6bfd839c018..00000000000
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQueryTable.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery;
-
-import java.io.Serializable;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
-import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils;
-import org.apache.beam.sdk.io.gcp.bigquery.WriteResult;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.POutput;
-import org.apache.beam.sdk.values.Row;
-
-/**
- * {@code BeamBigQueryTable} represent a BigQuery table as a target.
- * This provider does not currently support being a source.
- *
- */
-@Experimental
-public class BeamBigQueryTable extends BaseBeamTable implements Serializable {
- private String tableSpec;
-
- public BeamBigQueryTable(Schema beamSchema, String tableSpec) {
- super(beamSchema);
- this.tableSpec = tableSpec;
- }
-
- @Override
- public BeamIOType getSourceType() {
- return BeamIOType.BOUNDED;
- }
-
- @Override
- public PCollection<Row> buildIOReader(Pipeline pipeline) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public PTransform<? super PCollection<Row>, POutput> buildIOWriter() {
- return new PTransform<PCollection<Row>, POutput>() {
- @Override
- public WriteResult expand(PCollection<Row> input) {
- return input.apply(BigQueryIO.<Row>write()
- .withSchema(BigQueryUtils.toTableSchema(getSchema()))
- .withFormatFunction(BigQueryUtils.toTableRow())
- .to(tableSpec));
- }
- };
- }
-
- public String getTableSpec() {
- return tableSpec;
- }
-}
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
deleted file mode 100644
index dd728d0faeb..00000000000
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery;
-
-import static
org.apache.beam.sdk.extensions.sql.meta.provider.MetaUtils.getRowTypeFromTable;
-
-import java.util.Collections;
-import java.util.List;
-import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
-import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
-import org.apache.beam.sdk.schemas.Schema;
-
-/**
- * BigQuery table provider.
- *
- * <p>A sample of text table is:
- * <pre>{@code
- * CREATE TABLE ORDERS(
- * ID INT COMMENT 'this is the primary key',
- * NAME VARCHAR(127) COMMENT 'this is the name'
- * )
- * TYPE 'bigquery'
- * COMMENT 'this is the table orders'
- * LOCATION '[PROJECT_ID]:[DATASET].[TABLE]'
- * }</pre>
- */
-public class BigQueryTableProvider implements TableProvider {
-
- @Override public String getTableType() {
- return "bigquery";
- }
-
- @Override public BeamSqlTable buildBeamSqlTable(Table table) {
- Schema schema = getRowTypeFromTable(table);
- String filePattern = table.getLocation();
-
- return new BeamBigQueryTable(schema, filePattern);
- }
-
- @Override public void createTable(Table table) {
- // empty
- }
-
- @Override public void dropTable(String tableName) {
- // empty
- }
-
- @Override public List<Table> listTables() {
- return Collections.emptyList();
- }
-
- @Override public void init() {
- // empty
- }
-
- @Override public void close() {
- // empty
- }
-}
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/package-info.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/package-info.java
deleted file mode 100644
index 129714104b2..00000000000
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Table schema for BigQuery.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery;
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
index 19e88601545..4101da775d5 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
@@ -17,6 +17,6 @@
*/
/**
- * Table schema for KafkaIO.
+ * table schema for KafkaIO.
*/
package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
diff --git
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java
deleted file mode 100644
index 51b3452247d..00000000000
---
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery;
-
-import static org.apache.beam.sdk.extensions.sql.RowSqlTypes.INTEGER;
-import static org.apache.beam.sdk.extensions.sql.RowSqlTypes.VARCHAR;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
-import org.apache.beam.sdk.extensions.sql.meta.Column;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
-import org.junit.Test;
-
-/**
- * UnitTest for {@link BigQueryTableProvider}.
- */
-public class BigQueryTableProviderTest {
- private BigQueryTableProvider provider = new BigQueryTableProvider();
-
- @Test
- public void testGetTableType() throws Exception {
- assertEquals("bigquery", provider.getTableType());
- }
-
- @Test
- public void testBuildBeamSqlTable() throws Exception {
- Table table = fakeTable("hello");
- BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
-
- assertNotNull(sqlTable);
- assertTrue(sqlTable instanceof BeamBigQueryTable);
-
- BeamBigQueryTable bqTable = (BeamBigQueryTable) sqlTable;
- assertEquals("project:dataset.table", bqTable.getTableSpec());
- }
-
- private static Table fakeTable(String name) {
- return Table.builder()
- .name(name)
- .comment(name + " table")
- .location("project:dataset.table")
- .columns(ImmutableList.of(
- Column.builder()
- .name("id")
- .fieldType(INTEGER)
- .nullable(true)
- .build(),
- Column.builder()
- .name("name")
- .fieldType(VARCHAR)
- .nullable(true)
- .build()
- ))
- .type("bigquery")
- .build();
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/build.gradle
b/sdks/java/io/google-cloud-platform/build.gradle
index d41a8ca2750..66f44f75215 100644
--- a/sdks/java/io/google-cloud-platform/build.gradle
+++ b/sdks/java/io/google-cloud-platform/build.gradle
@@ -40,7 +40,6 @@ dependencies {
shadow library.java.jackson_databind
shadow library.java.grpc_core
shadow library.java.google_api_services_bigquery
- shadow library.java.google_cloud_bigquery
shadow library.java.gax_grpc
shadow library.java.google_cloud_core_grpc
shadow library.java.google_api_services_pubsub
diff --git a/sdks/java/io/google-cloud-platform/pom.xml
b/sdks/java/io/google-cloud-platform/pom.xml
index 0cf3180b429..c315cff0823 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -187,11 +187,6 @@
<artifactId>google-api-services-bigquery</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-bigquery</artifactId>
- </dependency>
-
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
deleted file mode 100644
index 97795154e88..00000000000
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.cloud.bigquery.Field.Mode;
-import com.google.cloud.bigquery.StandardSQLTypeName;
-import com.google.common.collect.ImmutableMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.coders.RowCoder;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.Schema.Field;
-import org.apache.beam.sdk.schemas.Schema.FieldType;
-import org.apache.beam.sdk.schemas.Schema.TypeName;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.Row;
-
-/**
- * Utility methods for BigQuery related operations.
- *
- * <p><b>Example: Writing to BigQuery</b>
- *
- * <pre>{@code
- * PCollection<Row> rows = ...;
- *
- * rows.apply(BigQueryIO.<Row>write()
- * .withSchema(BigQueryUtils.toTableSchema(rows))
- * .withFormatFunction(BigQueryUtils.toTableRow())
- * .to("my-project:my_dataset.my_table"));
- * }</pre>
- */
-public class BigQueryUtils {
- private static final Map<TypeName, StandardSQLTypeName>
BEAM_TO_BIGQUERY_TYPE_MAPPING =
- ImmutableMap.<TypeName, StandardSQLTypeName>builder()
- .put(TypeName.BYTE, StandardSQLTypeName.INT64)
- .put(TypeName.INT16, StandardSQLTypeName.INT64)
- .put(TypeName.INT32, StandardSQLTypeName.INT64)
- .put(TypeName.INT64, StandardSQLTypeName.INT64)
-
- .put(TypeName.FLOAT, StandardSQLTypeName.FLOAT64)
- .put(TypeName.DOUBLE, StandardSQLTypeName.FLOAT64)
-
- .put(TypeName.DECIMAL, StandardSQLTypeName.FLOAT64)
-
- .put(TypeName.BOOLEAN, StandardSQLTypeName.BOOL)
-
- .put(TypeName.ARRAY, StandardSQLTypeName.ARRAY)
- .put(TypeName.ROW, StandardSQLTypeName.STRUCT)
-
- .put(TypeName.DATETIME, StandardSQLTypeName.TIMESTAMP)
- .put(TypeName.STRING, StandardSQLTypeName.STRING)
-
- .build();
-
- private static final Map<byte[], StandardSQLTypeName>
BEAM_TO_BIGQUERY_METADATA_MAPPING =
- ImmutableMap.<byte[], StandardSQLTypeName>builder()
- .put("DATE".getBytes(), StandardSQLTypeName.DATE)
- .put("TIME".getBytes(), StandardSQLTypeName.TIME)
- .put("TIME_WITH_LOCAL_TZ".getBytes(), StandardSQLTypeName.TIME)
- .put("TS".getBytes(), StandardSQLTypeName.TIMESTAMP)
- .put("TS_WITH_LOCAL_TZ".getBytes(), StandardSQLTypeName.TIMESTAMP)
- .build();
-
- /**
- * Get the corresponding BigQuery {@link StandardSQLTypeName}
- * for supported Beam {@link FieldType}.
- */
- private static StandardSQLTypeName toStandardSQLTypeName(FieldType
fieldType) {
- StandardSQLTypeName sqlType =
BEAM_TO_BIGQUERY_TYPE_MAPPING.get(fieldType.getTypeName());
-
- if (sqlType == StandardSQLTypeName.TIMESTAMP && fieldType.getMetadata() !=
null) {
- sqlType = BEAM_TO_BIGQUERY_METADATA_MAPPING.get(fieldType.getMetadata());
- }
-
- return sqlType;
- }
-
- private static List<TableFieldSchema> toTableFieldSchema(Schema schema) {
- List<TableFieldSchema> fields = new
ArrayList<TableFieldSchema>(schema.getFieldCount());
- for (Field schemaField : schema.getFields()) {
- FieldType type = schemaField.getType();
-
- TableFieldSchema field = new TableFieldSchema()
- .setName(schemaField.getName());
- if (schemaField.getDescription() != null &&
!"".equals(schemaField.getDescription())) {
- field.setDescription(schemaField.getDescription());
- }
-
- if (!schemaField.getNullable()) {
- field.setMode(Mode.REQUIRED.toString());
- }
- if (TypeName.ARRAY == type.getTypeName()) {
- type = type.getCollectionElementType();
- field.setMode(Mode.REPEATED.toString());
- }
- if (TypeName.ROW == type.getTypeName()) {
- Schema subType = type.getRowSchema();
- field.setFields(toTableFieldSchema(subType));
- }
- field.setType(toStandardSQLTypeName(type).toString());
-
- fields.add(field);
- }
- return fields;
- }
-
- /**
- * Convert a Beam {@link Schema} to a BigQuery {@link TableSchema}.
- */
- public static TableSchema toTableSchema(Schema schema) {
- return new TableSchema().setFields(toTableFieldSchema(schema));
- }
-
- /**
- * Convert a Beam {@link PCollection} to a BigQuery {@link TableSchema}.
- */
- public static TableSchema toTableSchema(PCollection<Row> rows) {
- RowCoder coder = (RowCoder) rows.getCoder();
- return toTableSchema(coder.getSchema());
- }
-
- private static final SerializableFunction<Row, TableRow> TO_TABLE_ROW = new
ToTableRow();
-
- /**
- * Convert a Beam {@link Row} to a BigQuery {@link TableRow}.
- */
- public static SerializableFunction<Row, TableRow> toTableRow() {
- return TO_TABLE_ROW;
- }
-
- /**
- * Convert a Beam {@link Row} to a BigQuery {@link TableRow}.
- */
- private static class ToTableRow implements SerializableFunction<Row,
TableRow> {
- @Override
- public TableRow apply(Row input) {
- TableRow output = new TableRow();
- for (int i = 0; i < input.getFieldCount(); i++) {
- Object value = input.getValue(i);
-
- Field schemaField = input.getSchema().getField(i);
- TypeName type = schemaField.getType().getTypeName();
- if (TypeName.ARRAY == type) {
- type =
schemaField.getType().getCollectionElementType().getTypeName();
- if (TypeName.ROW == type) {
- List<Row> rows = (List<Row>) value;
- List<TableRow> tableRows = new ArrayList<TableRow>(rows.size());
- for (int j = 0; j < rows.size(); j++) {
- tableRows.add(apply(rows.get(j)));
- }
- value = tableRows;
- }
- } else if (TypeName.ROW == type) {
- value = apply((Row) value);
- }
-
- output = output.set(
- schemaField.getName(),
- value);
- }
- return output;
- }
- }
-}
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java
deleted file mode 100644
index 150187f6267..00000000000
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableRow;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableSchema;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.collection.IsMapContaining.hasEntry;
-import static org.junit.Assert.assertThat;
-
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.cloud.bigquery.Field.Mode;
-import com.google.cloud.bigquery.StandardSQLTypeName;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.values.Row;
-import org.joda.time.DateTime;
-import org.junit.Test;
-
-/**
- * Tests for {@link BigQueryUtils}.
- */
-public class BigQueryUtilsTest {
- private static final Schema FLAT_TYPE = Schema
- .builder()
- .addInt64Field("id", true)
- .addDoubleField("value", true)
- .addStringField("name", true)
- .addDateTimeField("timestamp", true)
- .addBooleanField("valid", true)
- .build();
-
- private static final Schema ARRAY_TYPE = Schema
- .builder()
- .addArrayField("ids", Schema.TypeName.INT64.type())
- .build();
-
- private static final Schema ROW_TYPE = Schema
- .builder()
- .addRowField("row", FLAT_TYPE, true)
- .build();
-
- private static final Schema ARRAY_ROW_TYPE = Schema
- .builder()
- .addArrayField("rows", Schema.FieldType
- .of(Schema.TypeName.ROW)
- .withRowSchema(FLAT_TYPE))
- .build();
-
- private static final TableFieldSchema ID =
- new TableFieldSchema().setName("id")
- .setType(StandardSQLTypeName.INT64.toString());
-
- private static final TableFieldSchema VALUE =
- new TableFieldSchema().setName("value")
- .setType(StandardSQLTypeName.FLOAT64.toString());
-
- private static final TableFieldSchema NAME =
- new TableFieldSchema().setName("name")
- .setType(StandardSQLTypeName.STRING.toString());
-
- private static final TableFieldSchema TIMESTAMP =
- new TableFieldSchema().setName("timestamp")
- .setType(StandardSQLTypeName.TIMESTAMP.toString());
-
- private static final TableFieldSchema VALID =
- new TableFieldSchema().setName("valid")
- .setType(StandardSQLTypeName.BOOL.toString());
-
- private static final TableFieldSchema IDS =
- new TableFieldSchema().setName("ids")
- .setType(StandardSQLTypeName.INT64.toString())
- .setMode(Mode.REPEATED.toString());
-
- private static final Row FLAT_ROW =
- Row
- .withSchema(FLAT_TYPE)
- .addValues(123L, 123.456, "test", new DateTime(123456), false)
- .build();
-
- private static final Row ARRAY_ROW =
- Row
- .withSchema(ARRAY_TYPE)
- .addValues((Object) Arrays.asList(123L, 124L))
- .build();
-
- private static final Row ROW_ROW =
- Row
- .withSchema(ROW_TYPE)
- .addValues(FLAT_ROW)
- .build();
-
- private static final Row ARRAY_ROW_ROW =
- Row
- .withSchema(ARRAY_ROW_TYPE)
- .addValues((Object) Arrays.asList(FLAT_ROW))
- .build();
-
- @Test public void testToTableSchema_flat() {
- TableSchema schema = toTableSchema(FLAT_TYPE);
-
- assertThat(schema.getFields(), containsInAnyOrder(ID, VALUE, NAME,
TIMESTAMP, VALID));
- }
-
- @Test public void testToTableSchema_array() {
- TableSchema schema = toTableSchema(ARRAY_TYPE);
-
- assertThat(schema.getFields(), contains(IDS));
- }
-
- @Test public void testToTableSchema_row() {
- TableSchema schema = toTableSchema(ROW_TYPE);
-
- assertThat(schema.getFields().size(), equalTo(1));
- TableFieldSchema field = schema.getFields().get(0);
- assertThat(field.getName(), equalTo("row"));
- assertThat(field.getType(),
equalTo(StandardSQLTypeName.STRUCT.toString()));
- assertThat(field.getMode(), nullValue());
- assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME,
TIMESTAMP, VALID));
- }
-
- @Test public void testToTableSchema_array_row() {
- TableSchema schema = toTableSchema(ARRAY_ROW_TYPE);
-
- assertThat(schema.getFields().size(), equalTo(1));
- TableFieldSchema field = schema.getFields().get(0);
- assertThat(field.getName(), equalTo("rows"));
- assertThat(field.getType(),
equalTo(StandardSQLTypeName.STRUCT.toString()));
- assertThat(field.getMode(), equalTo(Mode.REPEATED.toString()));
- assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME,
TIMESTAMP, VALID));
- }
-
- @Test public void testToTableRow_flat() {
- TableRow row = toTableRow().apply(FLAT_ROW);
-
- assertThat(row.size(), equalTo(5));
- assertThat(row, hasEntry("id", 123L));
- assertThat(row, hasEntry("value", 123.456));
- assertThat(row, hasEntry("name", "test"));
- assertThat(row, hasEntry("valid", false));
- }
-
- @Test public void testToTableRow_array() {
- TableRow row = toTableRow().apply(ARRAY_ROW);
-
- assertThat(row, hasEntry("ids", Arrays.asList(123L, 124L)));
- assertThat(row.size(), equalTo(1));
- }
-
- @Test public void testToTableRow_row() {
- TableRow row = toTableRow().apply(ROW_ROW);
-
- assertThat(row.size(), equalTo(1));
- row = (TableRow) row.get("row");
- assertThat(row.size(), equalTo(5));
- assertThat(row, hasEntry("id", 123L));
- assertThat(row, hasEntry("value", 123.456));
- assertThat(row, hasEntry("name", "test"));
- assertThat(row, hasEntry("valid", false));
- }
-
- @Test public void testToTableRow_array_row() {
- TableRow row = toTableRow().apply(ARRAY_ROW_ROW);
-
- assertThat(row.size(), equalTo(1));
- row = ((List<TableRow>) row.get("rows")).get(0);
- assertThat(row.size(), equalTo(5));
- assertThat(row, hasEntry("id", 123L));
- assertThat(row, hasEntry("value", 123.456));
- assertThat(row, hasEntry("name", "test"));
- assertThat(row, hasEntry("valid", false));
- }
-}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 98679)
Time Spent: 1.5h (was: 1h 20m)
> PostCommits broken on SpannerReadIT tests
> -----------------------------------------
>
> Key: BEAM-4229
> URL: https://issues.apache.org/jira/browse/BEAM-4229
> Project: Beam
> Issue Type: Bug
> Components: testing
> Affects Versions: 2.5.0
> Reporter: Scott Wegner
> Assignee: Andrew Pilloud
> Priority: Blocker
> Fix For: 2.5.0
>
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> SpannerReadIT tests are currently broken in Java post-commit tests:
> [https://scans.gradle.com/s/ajlgofucflem2/tests/failed]
> {quote}:beam-runners-google-cloud-dataflow-java:googleCloudPlatformIntegrationTestorg.apache.beam.sdk.io.gcp.spanner.SpannerReadIT
> » testQuery (0.134s)
> java.lang.NoClassDefFoundError:
> com/google/api/gax/retrying/ExceptionRetryAlgorithm
> Caused by: java.lang.ClassNotFoundException:
> com.google.api.gax.retrying.ExceptionRetryAlgorithm{quote}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
