rmahindra123 commented on code in PR #8514:
URL: https://github.com/apache/hudi/pull/8514#discussion_r1177388252
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ChainedTransformer.java:
##########
@@ -19,24 +19,73 @@
package org.apache.hudi.utilities.transform;
import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
/**
* A {@link Transformer} to chain other {@link Transformer}s and apply
sequentially.
*/
public class ChainedTransformer implements Transformer {
- private List<Transformer> transformers;
+ // Delimiter used to separate class name and the property key suffix. The
suffix comes first.
+ private static final String TRANSFORMER_CLASS_NAME_KEY_SUFFIX_DELIMITER =
":";
+
+ private final List<Transformer> transformers;
+ private final Map<Transformer, String> transformerToPropKeySuffix;
public ChainedTransformer(List<Transformer> transformers) {
this.transformers = transformers;
+ this.transformerToPropKeySuffix = new HashMap<>(transformers.size());
+ for (Transformer transformer : this.transformers) {
+ transformerToPropKeySuffix.put(transformer, "");
+ }
+ }
+
+ /**
+ * Creates a chained transformer using the input transformer class names.
The name can also include
+ * a suffix. This suffix can be appended with the property keys to identify
properties related to the transformer.
+ * E:g - tr1:org.apache.hudi.utilities.transform.SqlQueryBasedTransformer
can be used along with property key
+ * hoodie.deltastreamer.transformer.sql.tr1. Here tr1 is a suffix used to
identify the keys specific to this transformer.
+ * This suffix is removed from the configuration keys when the transformer
is used. This is useful when there are two or more
+ * transformers using the same config keys and expect different values for
those keys.
+ *
+ * @param configuredTransformers List of configured transformer class names.
+ * @param ignore Added for avoiding two methods with same erasure. Ignored.
+ */
+ public ChainedTransformer(List<String> configuredTransformers, int...
ignore) {
+ this.transformerToPropKeySuffix = new
HashMap<>(configuredTransformers.size());
+ this.transformers = new ArrayList<>(configuredTransformers.size());
+
+ List<Pair<String, String>> transformerClassNamesToSuffixList = new
ArrayList<>(configuredTransformers.size());
+ for (String configuredTransformer : configuredTransformers) {
+ if (!configuredTransformer.contains(":")) {
+ transformerClassNamesToSuffixList.add(Pair.of(configuredTransformer,
""));
+ } else {
+ String[] splits = configuredTransformer.split(":");
+ if (splits.length > 2) {
+ throw new IllegalArgumentException("There should only be one colon
in a configured transformer");
+ }
+ transformerClassNamesToSuffixList.add(Pair.of(splits[1], splits[0]));
Review Comment:
the config doc says we use "trX", but here we are only using the index?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
