This is an automated email from the ASF dual-hosted git repository.
ahmedabualsaud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d22a7e783ab Make defaults for optional SchemaTransformProvider methods
(#30560)
d22a7e783ab is described below
commit d22a7e783abf445ae7bef2ce26075f9d93b409a7
Author: Ahmed Abualsaud <[email protected]>
AuthorDate: Mon Mar 11 17:26:54 2024 -0400
Make defaults for optional SchemaTransformProvider methods (#30560)
* simplify schematransformprovider
---
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 1 +
.../transforms/SchemaTransformProvider.java | 9 ++++++--
.../transforms/TypedSchemaTransformProvider.java | 20 +++++++++++++++++-
.../TypedSchemaTransformProviderTest.java | 24 ++++++++++++++++++++--
...ueryStorageWriteApiSchemaTransformProvider.java | 5 -----
.../BigtableReadSchemaTransformProvider.java | 10 ---------
.../BigtableWriteSchemaTransformProvider.java | 10 ---------
...gQueryDirectReadSchemaTransformProviderIT.java} | 2 +-
...yStorageWriteApiSchemaTransformProviderIT.java} | 2 +-
9 files changed, 51 insertions(+), 32 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
index e69de29bb2d..8b137891791 100644
--- a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
+++ b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
@@ -0,0 +1 @@
+
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
index c76d7a25e69..9d0ad61b7a6 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.schemas.transforms;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.beam.sdk.annotations.Internal;
@@ -58,10 +59,14 @@ public interface SchemaTransformProvider {
SchemaTransform from(Row configuration);
/** Returns the input collection names of this transform. */
- List<String> inputCollectionNames();
+ default List<String> inputCollectionNames() {
+ return Collections.emptyList();
+ }
/** Returns the output collection names of this transform. */
- List<String> outputCollectionNames();
+ default List<String> outputCollectionNames() {
+ return Collections.emptyList();
+ }
/**
* List the dependencies needed for this transform. Jars from classpath are
used by default when
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
index 1117f59a748..e75fa27d2d1 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProvider.java
@@ -17,8 +17,13 @@
*/
package org.apache.beam.sdk.schemas.transforms;
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
+import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
+
+import java.lang.reflect.ParameterizedType;
import java.util.List;
import java.util.Optional;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.schemas.NoSuchSchemaException;
@@ -39,7 +44,20 @@ import org.apache.beam.sdk.values.Row;
@Internal
public abstract class TypedSchemaTransformProvider<ConfigT> implements
SchemaTransformProvider {
- protected abstract Class<ConfigT> configurationClass();
+ @SuppressWarnings("unchecked")
+ protected Class<ConfigT> configurationClass() {
+ @Nullable
+ ParameterizedType parameterizedType = (ParameterizedType)
getClass().getGenericSuperclass();
+ checkStateNotNull(
+ parameterizedType, "Could not get the TypedSchemaTransformProvider's
parameterized type.");
+ checkArgument(
+ parameterizedType.getActualTypeArguments().length == 1,
+ String.format(
+ "Expected one parameterized type, but got %s.",
+ parameterizedType.getActualTypeArguments().length));
+
+ return (Class<ConfigT>) parameterizedType.getActualTypeArguments()[0];
+ }
/**
* Produce a SchemaTransform from ConfigT. Can throw a {@link
InvalidConfigurationException} or a
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
index 2b698f4f67b..6b5ccbff4e4 100644
---
a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
+++
b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
@@ -94,6 +94,19 @@ public class TypedSchemaTransformProviderTest {
}
}
+ private static class FakeMinimalTypedProvider
+ extends TypedSchemaTransformProvider<Configuration> {
+ @Override
+ public String identifier() {
+ return "fake:v1";
+ }
+
+ @Override
+ public SchemaTransform from(Configuration config) {
+ return new FakeSchemaTransform(config);
+ }
+ }
+
public static class FakeSchemaTransform extends SchemaTransform {
public Configuration config;
@@ -111,6 +124,8 @@ public class TypedSchemaTransformProviderTest {
@Test
public void testFrom() {
SchemaTransformProvider provider = new FakeTypedSchemaIOProvider();
+ SchemaTransformProvider minimalProvider = new FakeMinimalTypedProvider();
+
Row inputConfig =
Row.withSchema(provider.configurationSchema())
.withFieldValue("field1", "field1")
@@ -118,8 +133,13 @@ public class TypedSchemaTransformProviderTest {
.build();
Configuration outputConfig = ((FakeSchemaTransform)
provider.from(inputConfig)).config;
- assertEquals("field1", outputConfig.getField1());
- assertEquals(13, outputConfig.getField2().intValue());
+ Configuration minimalOutputConfig =
+ ((FakeSchemaTransform) minimalProvider.from(inputConfig)).config;
+
+ for (Configuration config : Arrays.asList(outputConfig,
minimalOutputConfig)) {
+ assertEquals("field1", config.getField1());
+ assertEquals(13, config.getField2().intValue());
+ }
assertEquals("Description of fake provider", provider.description());
}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
index ca0d0a2784b..980d783ec43 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
@@ -88,11 +88,6 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
// magic string that tells us to write to dynamic destinations
protected static final String DYNAMIC_DESTINATIONS = "DYNAMIC_DESTINATIONS";
- @Override
- protected Class<BigQueryStorageWriteApiSchemaTransformConfiguration>
configurationClass() {
- return BigQueryStorageWriteApiSchemaTransformConfiguration.class;
- }
-
@Override
protected SchemaTransform from(
BigQueryStorageWriteApiSchemaTransformConfiguration configuration) {
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java
index a9f113f1ce9..f48a2355914 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java
@@ -70,11 +70,6 @@ public class BigtableReadSchemaTransformProvider
Schema.FieldType.array(Schema.FieldType.row(CELL_SCHEMA))))
.build();
- @Override
- protected Class<BigtableReadSchemaTransformConfiguration>
configurationClass() {
- return BigtableReadSchemaTransformConfiguration.class;
- }
-
@Override
protected SchemaTransform from(BigtableReadSchemaTransformConfiguration
configuration) {
return new BigtableReadSchemaTransform(configuration);
@@ -85,11 +80,6 @@ public class BigtableReadSchemaTransformProvider
return "beam:schematransform:org.apache.beam:bigtable_read:v1";
}
- @Override
- public List<String> inputCollectionNames() {
- return Collections.emptyList();
- }
-
@Override
public List<String> outputCollectionNames() {
return Collections.singletonList(OUTPUT_TAG);
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java
index b99b69621a8..cc480be6aa7 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java
@@ -60,11 +60,6 @@ public class BigtableWriteSchemaTransformProvider
private static final String INPUT_TAG = "input";
- @Override
- protected Class<BigtableWriteSchemaTransformConfiguration>
configurationClass() {
- return BigtableWriteSchemaTransformConfiguration.class;
- }
-
@Override
protected SchemaTransform from(BigtableWriteSchemaTransformConfiguration
configuration) {
return new BigtableWriteSchemaTransform(configuration);
@@ -80,11 +75,6 @@ public class BigtableWriteSchemaTransformProvider
return Collections.singletonList(INPUT_TAG);
}
- @Override
- public List<String> outputCollectionNames() {
- return Collections.emptyList();
- }
-
/** Configuration for writing to Bigtable. */
@DefaultSchema(AutoValueSchema.class)
@AutoValue
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderIT.java
similarity index 99%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java
rename to
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderIT.java
index 2363a870bbd..958409eb5e3 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderIT.java
@@ -74,7 +74,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
-public class BigQueryDirectReadSchemaTransformProviderTest {
+public class BigQueryDirectReadSchemaTransformProviderIT {
private static PipelineOptions testOptions =
TestPipeline.testingPipelineOptions();
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderIT.java
similarity index 99%
rename from
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
rename to
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderIT.java
index 64ea0b11d1b..8b8a3b75949 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderIT.java
@@ -61,7 +61,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
-public class BigQueryStorageWriteApiSchemaTransformProviderTest {
+public class BigQueryStorageWriteApiSchemaTransformProviderIT {
private FakeDatasetService fakeDatasetService = new FakeDatasetService();
private FakeJobService fakeJobService = new FakeJobService();
