deniskuzZ commented on code in PR #5418:
URL: https://github.com/apache/hive/pull/5418#discussion_r1741900109
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/WriterBuilder.java:
##########
@@ -78,75 +75,58 @@ public WriterBuilder queryId(String newQueryId) {
return this;
}
- public WriterBuilder poolSize(int newPoolSize) {
- this.poolSize = newPoolSize;
- return this;
- }
-
public WriterBuilder operation(Operation newOperation) {
this.operation = newOperation;
return this;
}
- public WriterBuilder isFanoutEnabled(boolean isFanoutEnabled) {
- this.fanoutEnabled = isFanoutEnabled;
+ public WriterBuilder hasOrdering(boolean inputOrdered) {
+ context.inputOrdered = inputOrdered;
+ if (IcebergTableUtil.isFanoutEnabled(table.properties()) && !inputOrdered)
{
+ context.useFanoutWriter = true;
+ }
return this;
}
- public WriterBuilder isMergeTask(boolean mergeTaskEnabled) {
- this.isMergeTask = mergeTaskEnabled;
+ public WriterBuilder isMergeTask(boolean isMergeTaskEnabled) {
+ context.isMergeTask = isMergeTaskEnabled;
return this;
}
public HiveIcebergWriter build() {
- Map<String, String> properties = table.properties();
-
- String dataFileFormatName = properties.getOrDefault(DEFAULT_FILE_FORMAT,
DEFAULT_FILE_FORMAT_DEFAULT);
- FileFormat dataFileFormat =
FileFormat.valueOf(dataFileFormatName.toUpperCase(Locale.ENGLISH));
-
- String deleteFileFormatName =
properties.getOrDefault(DELETE_DEFAULT_FILE_FORMAT, dataFileFormatName);
- FileFormat deleteFileFormat =
FileFormat.valueOf(deleteFileFormatName.toUpperCase(Locale.ENGLISH));
-
- long targetFileSize = PropertyUtil.propertyAsLong(table.properties(),
TableProperties.WRITE_TARGET_FILE_SIZE_BYTES,
- TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT);
-
- boolean skipRowData =
-
Boolean.parseBoolean(properties.getOrDefault(ICEBERG_DELETE_SKIPROWDATA,
ICEBERG_DELETE_SKIPROWDATA_DEFAULT));
-
- Schema dataSchema = table.schema();
- FileIO io = table.io();
- Map<Integer, PartitionSpec> specs = table.specs();
- int currentSpecId = table.spec().specId();
int partitionId = attemptID.getTaskID().getId();
int taskId = attemptID.getId();
String operationId = queryId + "-" + attemptID.getJobID() + "-" +
operationNum.incrementAndGet();
- OutputFileFactory outputFileFactory = OutputFileFactory.builderFor(table,
partitionId, taskId)
- .format(dataFileFormat)
- .operationId("data-" + operationId)
+
+ OutputFileFactory dataFileFactory = OutputFileFactory.builderFor(table,
partitionId, taskId)
+ .format(context.dataFileFormat())
+ .operationId(operationId)
.build();
- OutputFileFactory deleteOutputFileFactory =
OutputFileFactory.builderFor(table, partitionId, taskId)
- .format(deleteFileFormat)
- .operationId("delete-" + operationId)
+ OutputFileFactory deleteFileFactory = OutputFileFactory.builderFor(table,
partitionId, taskId)
+ .format(context.deleteFileFormat())
+ .operationId(operationId)
+ .suffix("deletes")
Review Comment:
yes, suffix+operationId
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
