lokeshj1703 commented on code in PR #8574:
URL: https://github.com/apache/hudi/pull/8574#discussion_r1204125181
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java:
##########
@@ -191,11 +192,14 @@ public static SchemaPostProcessor
createSchemaPostProcessor(
}
- public static Option<Transformer> createTransformer(Option<List<String>>
classNamesOpt, Boolean isErrorTableWriterEnabled) throws IOException {
+ public static Option<Transformer> createTransformer(Option<List<String>>
classNamesOpt, Option<Schema> sourceSchema,
+ boolean
enableTransformerSchemaValidation, Boolean isErrorTableWriterEnabled) throws
IOException {
+
try {
- return classNamesOpt.map(classNames -> classNames.isEmpty() ? null :
- isErrorTableWriterEnabled ? new
ErrorTableAwareChainedTransformer(classNames) : new
ChainedTransformer(classNames)
- );
+ Function<List<String>, Transformer> chainedTransformerFunction =
classNames ->
+ isErrorTableWriterEnabled ? new
ErrorTableAwareChainedTransformer(classNames, sourceSchema,
enableTransformerSchemaValidation)
Review Comment:
Addressed
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java:
##########
@@ -289,6 +289,11 @@ public static class Config implements Serializable {
)
public List<String> transformerClassNames = null;
+ @Parameter(names = {"--enable-transformer-schema-validation"},
+ description = "If enabled, schema is validated for the transformed
data against expected schema. "
+ + "Expected schema is provided by the transformer.")
+ public Boolean enableTransformerSchemaValidation = false;
Review Comment:
Addressed
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java:
##########
@@ -93,9 +105,19 @@ public List<String> getTransformersNames() {
@Override
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
