This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 6fa94c9 [BEAM-7862] Moving FakeBigQueryServices to published
artifacts (#9206)
6fa94c9 is described below
commit 6fa94c9eb0447537b3bc65b58633930f64d58052
Author: Pablo <[email protected]>
AuthorDate: Tue Aug 6 14:56:40 2019 -0700
[BEAM-7862] Moving FakeBigQueryServices to published artifacts (#9206)
* Moving FakeBigQueryServices to published artifacts
* Fixing ApiSurface tests
---
.../apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 2 +-
.../apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 6 ++----
.../apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 ++--
.../org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 6 ++++++
.../beam/sdk/io/gcp/testing}/FakeBigQueryServices.java | 14 +++++++++-----
.../beam/sdk/io/gcp/testing}/FakeDatasetService.java | 5 ++++-
.../apache/beam/sdk/io/gcp/testing}/FakeJobService.java | 7 +++++--
.../apache/beam/sdk/io/gcp/testing}/TableContainer.java | 2 +-
.../java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 3 +++
.../sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java | 5 ++++-
.../sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java | 4 +++-
.../beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 3 +++
13 files changed, 44 insertions(+), 19 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java
index e9196a8..d425a96 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java
@@ -65,7 +65,7 @@ class BigQueryAvroUtils {
* <p>Some BigQuery types are duplicated here since slightly different Avro
records are produced
* when exporting data in Avro format and when reading data directly using
the read API.
*/
- public static final ImmutableMultimap<String, Type> BIG_QUERY_TO_AVRO_TYPES =
+ static final ImmutableMultimap<String, Type> BIG_QUERY_TO_AVRO_TYPES =
ImmutableMultimap.<String, Type>builder()
.put("STRING", Type.STRING)
.put("GEOGRAPHY", Type.STRING)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 3dbfeb2..6fd3f95 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -471,8 +471,7 @@ public class BigQueryHelpers {
}
}
- @VisibleForTesting
- static String toJsonString(Object item) {
+ public static String toJsonString(Object item) {
if (item == null) {
return null;
}
@@ -485,8 +484,7 @@ public class BigQueryHelpers {
}
}
- @VisibleForTesting
- static <T> T fromJsonString(String json, Class<T> clazz) {
+ public static <T> T fromJsonString(String json, Class<T> clazz) {
if (json == null) {
return null;
}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
index 24d0865..ecd4a85 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
@@ -55,7 +55,7 @@ public interface BigQueryServices extends Serializable {
StorageClient getStorageClient(BigQueryOptions bqOptions) throws IOException;
/** An interface for the Cloud BigQuery load service. */
- interface JobService {
+ public interface JobService {
/** Start a BigQuery load job. */
void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig)
throws InterruptedException, IOException;
@@ -91,7 +91,7 @@ public interface BigQueryServices extends Serializable {
}
/** An interface to get, create and delete Cloud BigQuery datasets and
tables. */
- interface DatasetService {
+ public interface DatasetService {
/**
* Gets the specified {@link Table} resource by table ID.
*
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
index 1316a18..011f0c4 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java
@@ -269,6 +269,12 @@ public class BigQueryUtils {
return fromTableFieldSchema(tableSchema.getFields());
}
+ /** Convert a list of BigQuery {@link TableFieldSchema} to Avro {@link
org.apache.avro.Schema}. */
+ public static org.apache.avro.Schema toGenericAvroSchema(
+ String schemaName, List<TableFieldSchema> fieldSchemas) {
+ return BigQueryAvroUtils.toGenericAvroSchema(schemaName, fieldSchemas);
+ }
+
private static final BigQueryIO.TypedRead.ToBeamRowFunction<TableRow>
TABLE_ROW_TO_BEAM_ROW_FUNCTION = beamSchema -> (TableRow tr) ->
toBeamRow(beamSchema, tr);
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
similarity index 87%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
rename to
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
index bd13c7a..10e8faf 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.sdk.io.gcp.bigquery;
+package org.apache.beam.sdk.io.gcp.testing;
import com.google.api.client.util.Base64;
import com.google.api.services.bigquery.model.Table;
@@ -29,6 +29,10 @@ import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices;
+import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
import org.apache.beam.sdk.values.KV;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
@@ -90,11 +94,11 @@ public class FakeBigQueryServices implements
BigQueryServices {
public void cancel() {}
}
- static String encodeQueryResult(Table table) throws IOException {
+ public static String encodeQueryResult(Table table) throws IOException {
return encodeQueryResult(table, ImmutableList.of());
}
- static String encodeQueryResult(Table table, List<TableRow> rows) throws
IOException {
+ public static String encodeQueryResult(Table table, List<TableRow> rows)
throws IOException {
KvCoder<String, List<TableRow>> coder =
KvCoder.of(StringUtf8Coder.of(), ListCoder.of(TableRowJsonCoder.of()));
KV<String, List<TableRow>> kv = KV.of(BigQueryHelpers.toJsonString(table),
rows);
@@ -103,7 +107,7 @@ public class FakeBigQueryServices implements
BigQueryServices {
return Base64.encodeBase64String(outputStream.toByteArray());
}
- static KV<Table, List<TableRow>> decodeQueryResult(String queryResult)
throws IOException {
+ public static KV<Table, List<TableRow>> decodeQueryResult(String
queryResult) throws IOException {
KvCoder<String, List<TableRow>> coder =
KvCoder.of(StringUtf8Coder.of(), ListCoder.of(TableRowJsonCoder.of()));
ByteArrayInputStream inputStream = new
ByteArrayInputStream(Base64.decodeBase64(queryResult));
@@ -115,7 +119,7 @@ public class FakeBigQueryServices implements
BigQueryServices {
}
// Longs tend to get converted back to Integers due to JSON serialization.
Convert them back.
- static TableRow convertNumbers(TableRow tableRow) {
+ public static TableRow convertNumbers(TableRow tableRow) {
for (TableRow.Entry entry : tableRow.entrySet()) {
if (entry.getValue() instanceof Integer) {
entry.setValue(Long.valueOf((Integer) entry.getValue()));
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
similarity index 98%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
rename to
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
index 2cf450f..7916513 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.sdk.io.gcp.bigquery;
+package org.apache.beam.sdk.io.gcp.testing;
import static org.junit.Assert.assertEquals;
@@ -35,7 +35,10 @@ import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.ErrorContainer;
+import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy;
import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy.Context;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java
similarity index 98%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
rename to
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java
index d9d5ffe..9729f78 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.sdk.io.gcp.bigquery;
+package org.apache.beam.sdk.io.gcp.testing;
import static
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
import static
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull;
@@ -66,9 +66,12 @@ import
org.apache.beam.sdk.extensions.gcp.util.BackOffAdapter;
import org.apache.beam.sdk.extensions.gcp.util.Transport;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils;
+import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
import org.apache.beam.sdk.util.FluentBackoff;
import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.values.KV;
@@ -469,7 +472,7 @@ public class FakeJobService implements JobService,
Serializable {
private long writeRows(
String tableId, List<TableRow> rows, TableSchema schema, String
destinationPattern)
throws IOException {
- Schema avroSchema = BigQueryAvroUtils.toGenericAvroSchema(tableId,
schema.getFields());
+ Schema avroSchema = BigQueryUtils.toGenericAvroSchema(tableId,
schema.getFields());
List<TableRow> rowsToWrite = Lists.newArrayList();
int shard = 0;
for (TableRow row : rows) {
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
similarity index 97%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
rename to
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
index 18dbba3..cc7ad4f 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.sdk.io.gcp.bigquery;
+package org.apache.beam.sdk.io.gcp.testing;
import com.google.api.services.bigquery.model.Table;
import com.google.api.services.bigquery.model.TableRow;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
index fcd9233..50f6548 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
@@ -39,7 +39,7 @@ public class GcpApiSurfaceTest {
@Test
public void testGcpApiSurface() throws Exception {
- final Package thisPackage = getClass().getPackage();
+ final Package thisPackage = this.getClass().getPackage();
final ClassLoader thisClassLoader = getClass().getClassLoader();
final ApiSurface apiSurface =
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java
index fc9a43a..3edd6e3 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java
@@ -48,6 +48,9 @@ import
org.apache.beam.sdk.extensions.protobuf.ByteStringCoder;
import org.apache.beam.sdk.extensions.protobuf.ProtoCoder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.QueryPriority;
+import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices;
+import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService;
+import org.apache.beam.sdk.io.gcp.testing.FakeJobService;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.ValueProvider;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java
index 6166733..ffd88ff 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java
@@ -69,7 +69,10 @@ import
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.QueryPriority;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StorageClient;
-import
org.apache.beam.sdk.io.gcp.bigquery.FakeBigQueryServices.FakeBigQueryServerStream;
+import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices;
+import
org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices.FakeBigQueryServerStream;
+import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService;
+import org.apache.beam.sdk.io.gcp.testing.FakeJobService;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java
index 814a368..46c0538 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java
@@ -77,7 +77,9 @@ import
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TableRowParser;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StorageClient;
-import
org.apache.beam.sdk.io.gcp.bigquery.FakeBigQueryServices.FakeBigQueryServerStream;
+import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices;
+import
org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices.FakeBigQueryServerStream;
+import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.testing.PAssert;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
index 2e9a522..cd0312d 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
@@ -64,6 +64,9 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method;
+import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices;
+import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService;
+import org.apache.beam.sdk.io.gcp.testing.FakeJobService;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.schemas.JavaFieldSchema;
