ahmedabu98 commented on code in PR #41012:
URL: https://github.com/apache/beam/pull/41012#discussion_r4633908619
##########
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java:
##########
@@ -113,184 +81,60 @@ private static class WriteDoFn extends DoFn<KV<Row,
Iterable<Row>>, FileWriteRes
private final DynamicDestinations dynamicDestinations;
private final IcebergCatalogConfig catalogConfig;
private final String filePrefix;
- private final Schema dataSchema;
private final @Nullable Map<String, String> writeProperties;
private final @Nullable PCollectionView<Map<String,
SerializableTableSpec>> metadataView;
- private transient @MonotonicNonNull Map<TableIdentifier, Integer> specIds;
- private transient @MonotonicNonNull Map<TableIdentifier, Map<String,
PartitionField>>
- partitionFieldMaps;
WriteDoFn(
IcebergCatalogConfig catalogConfig,
DynamicDestinations dynamicDestinations,
String filePrefix,
- Schema dataSchema,
- @Nullable Map<String, String> writeProperties) {
- this(catalogConfig, dynamicDestinations, filePrefix, dataSchema,
writeProperties, null);
- }
-
- WriteDoFn(
- IcebergCatalogConfig catalogConfig,
- DynamicDestinations dynamicDestinations,
- String filePrefix,
- Schema dataSchema,
@Nullable Map<String, String> writeProperties,
@Nullable PCollectionView<Map<String, SerializableTableSpec>>
metadataView) {
this.catalogConfig = catalogConfig;
this.dynamicDestinations = dynamicDestinations;
this.filePrefix = filePrefix;
- this.dataSchema = dataSchema;
this.writeProperties = writeProperties;
this.metadataView = metadataView;
}
- @Setup
- public void setup() {
- partitionFieldMaps = Maps.newHashMap();
- specIds = Maps.newHashMap();
- }
-
@ProcessElement
public void processElement(
ProcessContext c,
@Element KV<Row, Iterable<Row>> element,
+ BoundedWindow window,
+ PaneInfo paneInfo,
OutputReceiver<FileWriteResult> out)
throws Exception {
String tableIdentifier =
checkStateNotNull(element.getKey().getString(DESTINATION));
- String partitionPath =
checkStateNotNull(element.getKey().getString(PARTITION));
IcebergDestination destination =
dynamicDestinations.instantiateDestination(tableIdentifier);
+ WindowedValue<IcebergDestination> windowedDestination =
+ WindowedValues.of(destination, window.maxTimestamp(), window,
paneInfo);
Map<String, SerializableTableSpec> sideInputs =
metadataView != null ? c.sideInput(metadataView) : null;
- Table table = getOrCreateTable(destination, dataSchema, sideInputs);
- partitionPath =
- getPartitionDataPath(
- partitionPath,
getPartitionFieldMap(destination.getTableIdentifier(), table));
-
- StructLike partitionData =
- table.spec().isPartitioned()
- ? DataFiles.data(table.spec(), partitionPath)
- : new PartitionKey(table.spec(), table.schema());
- String fileName =
- destination
- .getFileFormat()
- .addExtension(String.format("%s-%s", filePrefix,
UUID.randomUUID()));
-
- RecordWriter writer =
- new RecordWriter(
- table, destination.getFileFormat(), fileName, partitionData,
writeProperties);
- try {
+ RecordWriterManager writer;
+ try (RecordWriterManager openWriter =
Review Comment:
Each element is a batch of rows, so should be fine. We already have the same
pattern in `WriteGroupedRowsToFiles`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]