chamikaramj commented on code in PR #30910:
URL: https://github.com/apache/beam/pull/30910#discussion_r1573067555
##########
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java:
##########
@@ -51,38 +53,47 @@ static class IcebergReadSchemaTransformTranslator
@Override
public String getUrn() {
- return READ_PROVIDER.identifier();
+ return BeamUrns.getUrn(SCHEMA_TRANSFORM);
}
@Override
public @Nullable FunctionSpec translate(
AppliedPTransform<?, ?, IcebergReadSchemaTransform> application,
SdkComponents components)
throws IOException {
- SchemaApi.Schema expansionSchema =
SchemaTranslation.schemaToProto(READ_SCHEMA, true);
+ Schema snakeCaseSchema = READ_SCHEMA.toSnakeCase();
+ SchemaApi.Schema expansionSchema =
SchemaTranslation.schemaToProto(snakeCaseSchema, true);
Row configRow = toConfigRow(application.getTransform());
ByteArrayOutputStream os = new ByteArrayOutputStream();
- RowCoder.of(READ_SCHEMA).encode(configRow, os);
+ RowCoder.of(snakeCaseSchema).encode(configRow, os);
return FunctionSpec.newBuilder()
.setUrn(getUrn())
.setPayload(
SchemaTransformPayload.newBuilder()
+ .setIdentifier(READ_PROVIDER.identifier())
.setConfigurationSchema(expansionSchema)
.setConfigurationRow(ByteString.copyFrom(os.toByteArray()))
- .setIdentifier(getUrn())
.build()
.toByteString())
.build();
}
@Override
public Row toConfigRow(IcebergReadSchemaTransform transform) {
- return transform.getConfigurationRow();
+ // Will retrieve a Row with snake_case naming convention.
+ // Transform expects camelCase convention, so convert back
+ // TODO(https://github.com/apache/beam/issues/31061): Remove conversion
when
+ // TypedSchemaTransformProvider starts generating with snake_case
convention
+ return transform.getConfigurationRow().toSnakeCase();
}
@Override
public IcebergReadSchemaTransform fromConfigRow(Row configRow,
PipelineOptions options) {
Review Comment:
We also want to make sure that the SchemaTransformProvider.configuration()
method for schema-aware transforms we develop returns a schema consistent with
what we provide here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
