[
https://issues.apache.org/jira/browse/BEAM-5364?focusedWorklogId=145035&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-145035
]
ASF GitHub Bot logged work on BEAM-5364:
----------------------------------------
Author: ASF GitHub Bot
Created on: 17/Sep/18 19:44
Start Date: 17/Sep/18 19:44
Worklog Time Spent: 10m
Work Description: kevinsi4508 closed pull request #6379: [BEAM-5364]
Check if validation is disabled when validating BigtableSource
URL: https://github.com/apache/beam/pull/6379
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java
index 93e3f1e6cbb..620af4573f1 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java
@@ -159,13 +159,33 @@ void validate() {
"Could not obtain Bigtable instance id");
}
+ /**
+ * @param <T> parameter The runtime parameter.
+ * @return {@code null} if the input parameter is {@code null}, return the
String value of runtime
+ * parameter if the parameter is accessible, returns "Unavailable during
pipeline
+ * construction" if the runtime parameter is not accessible for
debugging purpose.
+ */
+ @Nullable
+ static <T> String getDisplayDataForParameter(@Nullable ValueProvider<T>
parameter) {
+ if (parameter == null) return null;
+
+ if (parameter.isAccessible()) {
+ return String.valueOf(parameter.get());
+ }
+ return "Unavailable during pipeline construction";
+ }
+
void populateDisplayData(DisplayData.Builder builder) {
builder
.addIfNotNull(
- DisplayData.item("projectId", getProjectId()).withLabel("Bigtable
Project Id"))
+ DisplayData.item("projectId",
getDisplayDataForParameter(getProjectId()))
+ .withLabel("Bigtable Project Id"))
+ .addIfNotNull(
+ DisplayData.item("instanceId",
getDisplayDataForParameter(getInstanceId()))
+ .withLabel("Bigtable Instance Id"))
.addIfNotNull(
- DisplayData.item("instanceId",
getInstanceId()).withLabel("Bigtable Instance Id"))
- .addIfNotNull(DisplayData.item("tableId",
getTableId()).withLabel("Bigtable Table Id"))
+ DisplayData.item("tableId",
getDisplayDataForParameter(getTableId()))
+ .withLabel("Bigtable Table Id"))
.add(DisplayData.item("withValidation",
getValidate()).withLabel("Check is table exists"));
if (getBigtableOptions() != null) {
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index ae8fe7d04d9..7990a50a363 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -1080,6 +1080,11 @@ private long
getEstimatedSizeBytesBasedOnSamples(List<SampleRowKeysResponse> sam
@Override
public void validate() {
+ if (!config.getValidate()) {
+ LOG.debug("Validation is disabled");
+ return;
+ }
+
ValueProvider<String> tableId = config.getTableId();
checkArgument(
tableId != null && tableId.isAccessible() &&
!tableId.get().isEmpty(),
@@ -1090,7 +1095,10 @@ public void validate() {
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- builder.add(DisplayData.item("tableId",
config.getTableId().get()).withLabel("Table ID"));
+ builder.add(
+ DisplayData.item(
+ "tableId",
BigtableConfig.getDisplayDataForParameter(config.getTableId()))
+ .withLabel("Table ID"));
if (filter != null) {
builder.add(DisplayData.item("rowFilter",
filter.toString()).withLabel("Table Row Filter"));
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
index 47727e5b8a1..cea8ead82c2 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
@@ -87,11 +87,13 @@
import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO.BigtableSource;
import org.apache.beam.sdk.io.range.ByteKey;
import org.apache.beam.sdk.io.range.ByteKeyRange;
+import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.testing.ExpectedLogs;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipeline.PipelineRunMissingException;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -115,6 +117,27 @@
@Rule public ExpectedException thrown = ExpectedException.none();
@Rule public ExpectedLogs logged = ExpectedLogs.none(BigtableIO.class);
+ /** Read Options for testing */
+ public interface ReadOptions extends GcpOptions {
+ @Description("The project that contains the table to export.")
+ ValueProvider<String> getBigtableProject();
+
+ @SuppressWarnings("unused")
+ void setBigtableProject(ValueProvider<String> projectId);
+
+ @Description("The Bigtable instance id that contains the table to export.")
+ ValueProvider<String> getBigtableInstanceId();
+
+ @SuppressWarnings("unused")
+ void setBigtableInstanceId(ValueProvider<String> instanceId);
+
+ @Description("The Bigtable table id to export.")
+ ValueProvider<String> getBigtableTableId();
+
+ @SuppressWarnings("unused")
+ void setBigtableTableId(ValueProvider<String> tableId);
+ }
+
static final ValueProvider<String> NOT_ACCESSIBLE_VALUE =
new ValueProvider<String>() {
@Override
@@ -223,6 +246,39 @@ public void
testReadValidationFailsMissingInstanceIdAndProjectId() {
read.expand(null);
}
+ @Test
+ public void testReadWithRuntimeParametersValidationFailed() {
+ ReadOptions options =
PipelineOptionsFactory.fromArgs().withValidation().as(ReadOptions.class);
+
+ BigtableIO.Read read =
+ BigtableIO.read()
+ .withProjectId(options.getBigtableProject())
+ .withInstanceId(options.getBigtableInstanceId())
+ .withTableId(options.getBigtableTableId());
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("tableId was not supplied");
+
+ p.apply(read);
+ }
+
+ @Test
+ public void testReadWithRuntimeParametersValidationDisabled() {
+ ReadOptions options =
PipelineOptionsFactory.fromArgs().withValidation().as(ReadOptions.class);
+
+ BigtableIO.Read read =
+ BigtableIO.read()
+ .withoutValidation()
+ .withProjectId(options.getBigtableProject())
+ .withInstanceId(options.getBigtableInstanceId())
+ .withTableId(options.getBigtableTableId());
+
+ // Not running a pipeline therefore this is expected.
+ thrown.expect(PipelineRunMissingException.class);
+
+ p.apply(read);
+ }
+
@Test
public void testWriteBuildsCorrectly() {
BigtableIO.Write write =
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 145035)
Time Spent: 1h 40m (was: 1.5h)
> BigtableIO source tries to validate table ID even though validation is
> turned off
> ----------------------------------------------------------------------------------
>
> Key: BEAM-5364
> URL: https://issues.apache.org/jira/browse/BEAM-5364
> Project: Beam
> Issue Type: Bug
> Components: io-java-gcp
> Reporter: Kevin Si
> Assignee: Chamikara Jayalath
> Priority: Minor
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> [https://github.com/apache/beam/blob/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java#L1084|https://www.google.com/url?q=https://github.com/apache/beam/blob/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java%23L1084&sa=D&usg=AFQjCNEfHprTOvnwAwFSrXwUuLvc__JBWg]
> The validation can be turned off with following:
> BigtableIO.read()
> .withoutValidation() // skip validation when constructing the
> pipelline.
> A Dataflow template cannot be constructed due to this validation failure.
>
> Error log when trying to construct a template:
> Exception in thread "main" java.lang.IllegalArgumentException: tableId was
> not supplied
> at
> com.google.common.base.Preconditions.checkArgument(Preconditions.java:122)
> at
> org.apache.beam.sdk.io.gcp.bigtable.BigtableIO$BigtableSource.validate(BigtableIO.java:1084)
> at org.apache.beam.sdk.io.Read$Bounded.expand(Read.java:95)
> at org.apache.beam.sdk.io.Read$Bounded.expand(Read.java:85)
> at org.apache.beam.sdk.Pipeline.applyInternal(Pipeline.java:537)
> at org.apache.beam.sdk.Pipeline.applyTransform(Pipeline.java:471)
> at org.apache.beam.sdk.values.PBegin.apply(PBegin.java:44)
> at org.apache.beam.sdk.Pipeline.apply(Pipeline.java:167)
> at
> org.apache.beam.sdk.io.gcp.bigtable.BigtableIO$Read.expand(BigtableIO.java:423)
> at
> org.apache.beam.sdk.io.gcp.bigtable.BigtableIO$Read.expand(BigtableIO.java:179)
> at org.apache.beam.sdk.Pipeline.applyInternal(Pipeline.java:537)
> at org.apache.beam.sdk.Pipeline.applyTransform(Pipeline.java:488)
> at org.apache.beam.sdk.values.PBegin.apply(PBegin.java:56)
> at org.apache.beam.sdk.Pipeline.apply(Pipeline.java:182)
> at
> com.google.cloud.teleport.bigtable.BigtableToAvro.main(BigtableToAvro.java:89)
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
