tejasiyer-dev commented on code in PR #40543:
URL: https://github.com/apache/beam/pull/40543#discussion_r4623186946
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1125,133 +1417,94 @@ private class WriteAutoShardedBundlesToTempFiles
private WriteAutoShardedBundlesToTempFiles(
Coder<DestinationT> destinationCoder,
Coder<FileResult<DestinationT>> fileResultCoder,
- Integer batchSize,
- Integer batchSizeBytes,
+ int batchSize,
+ int batchSizeBytes,
Duration maxBufferingDuration) {
this.destinationCoder = destinationCoder;
this.fileResultCoder = fileResultCoder;
- this.batchSize = batchSize != null ? batchSize :
FILE_TRIGGERING_RECORD_COUNT;
- this.batchSizeBytes = batchSizeBytes != null ? batchSizeBytes :
FILE_TRIGGERING_BYTE_COUNT;
- this.maxBufferingDuration =
- maxBufferingDuration != null
- ? maxBufferingDuration
- : FILE_TRIGGERING_RECORD_BUFFERING_DURATION;
+ this.batchSize = batchSize;
+ this.batchSizeBytes = batchSizeBytes;
+ this.maxBufferingDuration = maxBufferingDuration;
}
@Override
public PCollection<List<FileResult<DestinationT>>>
expand(PCollection<UserT> input) {
- // Auto-sharding is achieved via GroupIntoBatches.WithShardedKey which
shards, groups and at
- // the same time batches the input records. The sharding behavior
depends on runners. The
- // batching is per window and we also emit the batches if there are a
certain number of
- // records buffered or they have been buffered for a certain time,
controlled by
- // FILE_TRIGGERING_RECORD_COUNT and BUFFERING_DURATION respectively.
- //
- // TODO(https://github.com/apache/beam/issues/20928): The implementation
doesn't currently
- // work with merging windows.
- TupleTag<KV<Integer, UserT>> shardTag = new TupleTag<>("shardTag");
-
- PCollectionTuple shardedElements =
- input.apply(
- "KeyedByDestinationHash",
- ParDo.of(new KeyByDestinationHash(input.getCoder(),
destinationCoder))
- .withOutputTags(shardTag, TupleTagList.of(BAD_RECORD_TAG)));
- addErrorCollection(shardedElements);
-
- PCollection<KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>> shardedInput =
- shardedElements
- .get(shardTag)
- .setCoder(KvCoder.of(VarIntCoder.of(), input.getCoder()))
- .apply(
- "ShardAndBatch",
- GroupIntoBatches.<Integer, UserT>ofSize(batchSize)
- .withByteSize(batchSizeBytes)
- .withMaxBufferingDuration(maxBufferingDuration)
- .withShardedKey())
- .setCoder(
- KvCoder.of(
-
org.apache.beam.sdk.util.ShardedKey.Coder.of(VarIntCoder.of()),
- IterableCoder.of(input.getCoder())));
-
- TupleTag<FileResult<DestinationT>> writtenRecordsTag = new
TupleTag<>("writtenRecords");
- // Write grouped elements to temp files.
- PCollectionTuple writtenFiles =
- shardedInput
- .apply(
- "AddDummyShard",
- MapElements.via(
- new SimpleFunction<
- KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>,
- KV<ShardedKey<Integer>, Iterable<UserT>>>() {
- @Override
- public KV<ShardedKey<Integer>, Iterable<UserT>> apply(
- KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>
- input) {
- // Add dummy shard since it is required by
WriteShardsIntoTempFilesFn. It
- // will be dropped after we generate the temp files.
- return KV.of(
- ShardedKey.of(input.getKey().getKey(),
DUMMY_SHARDNUM),
- input.getValue());
- }
- }))
- .setCoder(
- KvCoder.of(
- ShardedKeyCoder.of(VarIntCoder.of()),
IterableCoder.of(input.getCoder())))
- .apply(
- "WriteShardsIntoTempFiles",
- ParDo.of(new WriteShardsIntoTempFilesFn(input.getCoder()))
- .withSideInputs(getSideInputs())
- .withOutputTags(writtenRecordsTag,
TupleTagList.of(BAD_RECORD_TAG)));
-
- addErrorCollection(writtenFiles);
-
PCollection<FileResult<DestinationT>> tempFiles =
- writtenFiles
- .get(writtenRecordsTag)
- .setCoder(fileResultCoder)
- .apply(
- "DropShardNum",
- ParDo.of(
- new DoFn<FileResult<DestinationT>,
FileResult<DestinationT>>() {
- @ProcessElement
- public void process(ProcessContext c) {
- c.output(c.element().withShard(UNKNOWN_SHARDNUM));
- }
- }))
- .setCoder(fileResultCoder);
+ writeAutoShardedRecordsToTempFiles(
+ input,
+ destinationCoder,
+ fileResultCoder,
+ batchSize,
+ batchSizeBytes,
+ maxBufferingDuration);
// Group temp file results by destinations again to gather all the
results in the same window.
// This is needed since we don't have shard idx associated with each
temp file so have to rely
// on the indexing within a bundle.
- return tempFiles
- .apply(
- "KeyedByDestination",
- WithKeys.of(
- new SimpleFunction<FileResult<DestinationT>, DestinationT>()
{
- @Override
- public DestinationT apply(FileResult<DestinationT> input) {
- return input.getDestination();
- }
- }))
- .setCoder(KvCoder.of(destinationCoder, fileResultCoder))
- .apply(GroupByKey.create())
- .apply(
- "ExtractValuesToList",
- ParDo.of(
- new DoFn<
- KV<DestinationT, Iterable<FileResult<DestinationT>>>,
- List<FileResult<DestinationT>>>() {
- @ProcessElement
- public void processElement(
- @Element KV<DestinationT,
Iterable<FileResult<DestinationT>>> element,
- ProcessContext c) {
- List<FileResult<DestinationT>> result = new
ArrayList<>();
- for (FileResult<DestinationT> e : element.getValue()) {
- result.add(e);
- }
- c.output(result);
- }
- }))
- .setCoder(ListCoder.of(fileResultCoder));
+ // "GroupByKey" matches the default name this step has always had
(update compatibility).
+ return groupTempFilesByDestination(
+ tempFiles, destinationCoder, fileResultCoder, "GroupByKey", /*
stampFiringPane= */ false);
+ }
+ }
+
+ /**
+ * Groups temp file results by destination into one list per destination and
window firing. Shared
+ * by the auto-sharded write and the direct-write path. Produces the steps
{@code
+ * KeyedByDestination}, {@code groupByKeyName} and {@code
ExtractValuesToList}.
+ *
+ * @param stampFiringPane if true, each file is re-stamped with the firing's
pane and an unknown
+ * shard number so shard numbers can be assigned in finalize across all
files of the firing.
+ */
+ private PCollection<List<FileResult<DestinationT>>>
groupTempFilesByDestination(
+ PCollection<FileResult<DestinationT>> tempFiles,
+ Coder<DestinationT> destinationCoder,
+ Coder<FileResult<DestinationT>> fileResultCoder,
+ String groupByKeyName,
+ boolean stampFiringPane) {
+ return tempFiles
+ .apply(
+ "KeyedByDestination",
+ WithKeys.of(
+ new SimpleFunction<FileResult<DestinationT>, DestinationT>() {
+ @Override
+ public DestinationT apply(FileResult<DestinationT> input) {
+ return input.getDestination();
+ }
+ }))
+ .setCoder(KvCoder.of(destinationCoder, fileResultCoder))
+ .apply(groupByKeyName, GroupByKey.create())
+ .apply("ExtractValuesToList", ParDo.of(new
ExtractValuesToListFn<>(stampFiringPane)))
+ .setCoder(ListCoder.of(fileResultCoder));
+ }
+
+ private static class ExtractValuesToListFn<DestinationT>
+ extends DoFn<
+ KV<DestinationT, Iterable<FileResult<DestinationT>>>,
List<FileResult<DestinationT>>> {
+ private final boolean stampFiringPane;
+
+ ExtractValuesToListFn(boolean stampFiringPane) {
+ this.stampFiringPane = stampFiringPane;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ List<FileResult<DestinationT>> result = new ArrayList<>();
+ for (FileResult<DestinationT> e : c.element().getValue()) {
+ if (stampFiringPane) {
+ // Stamp every file with this firing's pane so files from different
firings of the
+ // same window get distinct final names.
+ result.add(
+ new FileResult<>(
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]