jonvex commented on code in PR #12178:
URL: https://github.com/apache/hudi/pull/12178#discussion_r1821548824
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertDataInternalWriterHelper.java:
##########
@@ -100,22 +98,25 @@ public BulkInsertDataInternalWriterHelper(HoodieTable
hoodieTable, HoodieWriteCo
this.shouldPreserveHoodieMetadata = shouldPreserveHoodieMetadata;
this.arePartitionRecordsSorted = arePartitionRecordsSorted;
this.fileIdPrefix = UUID.randomUUID().toString();
+ this.shouldDropPartitionColumns = writeConfig.shouldDropPartitionColumns();
+ if (this.shouldDropPartitionColumns) {
+ // Drop the partition columns from the row
+ // Using the deprecated JavaConversions to be compatible with scala
versions < 2.12. Once hudi support for scala versions < 2.12 is
+ // stopped, can move this to JavaConverters.seqAsJavaList(...)
+ List<String> partitionCols =
JavaScalaConverters.convertScalaListToJavaList(HoodieDatasetBulkInsertHelper.getPartitionPathCols(this.writeConfig));
+ this.partitionIdx = new HashSet<>();
+ for (String col : partitionCols) {
+ partitionIdx.add(this.structType.fieldIndex(col));
+ }
+ } else {
+ this.partitionIdx = Collections.emptySet();
+ }
if (!populateMetaFields) {
this.keyGeneratorOpt =
HoodieSparkKeyGeneratorFactory.getKeyGenerator(writeConfig.getProps());
} else {
this.keyGeneratorOpt = Option.empty();
}
-
- if (keyGeneratorOpt.isPresent() && keyGeneratorOpt.get() instanceof
SimpleKeyGenerator) {
- this.simpleKeyGen = true;
Review Comment:
none of these are used by anything else
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
