nsivabalan commented on a change in pull request #1834:
URL: https://github.com/apache/hudi/pull/1834#discussion_r468975269
##########
File path:
hudi-spark/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
##########
@@ -125,49 +130,58 @@ public TimestampBasedKeyGenerator(TypedProperties config,
String partitionPathFi
@Override
public String getPartitionPath(GenericRecord record) {
- Object partitionVal = HoodieAvroUtils.getNestedFieldVal(record,
partitionPathField, true);
+ Object partitionVal = HoodieAvroUtils.getNestedFieldVal(record,
getPartitionPathFields().get(0), true);
if (partitionVal == null) {
partitionVal = 1L;
}
+ try {
+ return getPartitionPath(partitionVal);
+ } catch (Exception e) {
+ throw new HoodieDeltaStreamerException("Unable to parse input partition
field :" + partitionVal, e);
+ }
+ }
+ /**
+ * Parse and fetch partition path based on data type.
+ *
+ * @param partitionVal partition path object value fetched from record/row
+ * @return the parsed partition path based on data type
+ * @throws ParseException on any parse exception
+ */
+ private String getPartitionPath(Object partitionVal) throws ParseException {
DateTimeFormatter partitionFormatter =
DateTimeFormat.forPattern(outputDateFormat);
if (this.outputDateTimeZone != null) {
partitionFormatter = partitionFormatter.withZone(outputDateTimeZone);
}
-
- try {
- long timeMs;
- if (partitionVal instanceof Double) {
- timeMs = convertLongTimeToMillis(((Double) partitionVal).longValue());
- } else if (partitionVal instanceof Float) {
- timeMs = convertLongTimeToMillis(((Float) partitionVal).longValue());
- } else if (partitionVal instanceof Long) {
- timeMs = convertLongTimeToMillis((Long) partitionVal);
- } else if (partitionVal instanceof CharSequence) {
- DateTime parsedDateTime =
inputFormatter.parseDateTime(partitionVal.toString());
- if (this.outputDateTimeZone == null) {
- // Use the timezone that came off the date that was passed in, if it
had one
- partitionFormatter =
partitionFormatter.withZone(parsedDateTime.getZone());
- }
-
- timeMs =
inputFormatter.parseDateTime(partitionVal.toString()).getMillis();
- } else {
- throw new HoodieNotSupportedException(
- "Unexpected type for partition field: " +
partitionVal.getClass().getName());
+ long timeMs;
Review comment:
note to reviewer: removed the outer try catch and moved it to the
caller. Except that, no other code changes.
##########
File path:
hudi-spark/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java
##########
@@ -22,30 +22,27 @@
import org.apache.hudi.common.config.TypedProperties;
import org.apache.avro.generic.GenericRecord;
+import org.apache.spark.sql.Row;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
/**
- * Key generator for deletes using global indices. Global index deletes do not
require partition value
- * so this key generator avoids using partition value for generating HoodieKey.
+ * Key generator for deletes using global indices. Global index deletes do not
require partition value so this key generator avoids using partition value for
generating HoodieKey.
*/
public class GlobalDeleteKeyGenerator extends BuiltinKeyGenerator {
private static final String EMPTY_PARTITION = "";
- protected final List<String> recordKeyFields;
-
public GlobalDeleteKeyGenerator(TypedProperties config) {
super(config);
- this.recordKeyFields =
Arrays.stream(config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(",")).map(String::trim).collect(Collectors.toList());
+ this.recordKeyFields =
Arrays.asList(config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(","));
Review comment:
This line is different from what I see before this patch. It is an
optimization, but just to be safe, we can keep it as is.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
