arunpandianp commented on code in PR #40543:
URL: https://github.com/apache/beam/pull/40543#discussion_r4597417268


##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1114,6 +1208,301 @@ public PCollection<FileResult<DestinationT>> 
expand(PCollection<UserT> input) {
     }
   }
 
+  /**
+   * Streaming write that sends large bundles straight to temporary files.
+   *
+   * <p>Small bundles, and records that would exceed the per-bundle writer 
limit, take the usual
+   * grouped path ({@link WriteShardedBundlesToTempFiles} when a shard count 
is set, or the {@link
+   * GroupIntoBatches} auto-sharded path otherwise). All files then drop their 
shard number and are
+   * grouped per destination, so finalization assigns shard numbers across 
both paths.
+   */
+  private class WriteShardedBundlesWithDirectWrite
+      extends PTransform<PCollection<UserT>, 
PCollection<List<FileResult<DestinationT>>>> {
+    private final Coder<DestinationT> destinationCoder;
+    private final Coder<FileResult<DestinationT>> fileResultCoder;
+    private final @Nullable PCollectionView<Integer> numShardsView;
+    private final long thresholdBytes;
+
+    private WriteShardedBundlesWithDirectWrite(
+        Coder<DestinationT> destinationCoder,
+        Coder<FileResult<DestinationT>> fileResultCoder,
+        @Nullable PCollectionView<Integer> numShardsView,
+        long thresholdBytes) {
+      this.destinationCoder = destinationCoder;
+      this.fileResultCoder = fileResultCoder;
+      this.numShardsView = numShardsView;
+      this.thresholdBytes = thresholdBytes;
+    }
+
+    @Override
+    public PCollection<List<FileResult<DestinationT>>> 
expand(PCollection<UserT> input) {
+      TupleTag<UserT> smallBundlesTag = new TupleTag<>("smallBundles");
+      TupleTag<UserT> largeBundlesTag = new TupleTag<>("largeBundles");
+      TupleTag<FileResult<DestinationT>> directFilesTag = new 
TupleTag<>("directFiles");
+      TupleTag<UserT> spilledRecordsTag = new TupleTag<>("spilledRecords");
+
+      // Evaluate the threshold per window, pane and destination, i.e. per 
output file.
+      PCollectionTuple lifted =
+          input.apply(
+              "LiftLargeBundles",
+              BundleLifter.of(smallBundlesTag, largeBundlesTag, 
thresholdBytes, input.getCoder())
+                  .withKeyFn(new 
DestinationKeyFn<>(getDynamicDestinations())));
+
+      PCollectionTuple routed =
+          lifted
+              .get(largeBundlesTag)
+              .setCoder(input.getCoder())
+              .apply(
+                  "DirectWriteLargeBundles",
+                  ParDo.of(new DirectWriteRecordsFn(spilledRecordsTag, 
input.getCoder()))
+                      .withSideInputs(getSideInputs())
+                      .withOutputTags(
+                          directFilesTag,
+                          TupleTagList.of(ImmutableList.of(spilledRecordsTag, 
BAD_RECORD_TAG))));
+      addErrorCollection(routed);
+
+      PCollection<UserT> smallBundles =
+          
PCollectionList.of(lifted.get(smallBundlesTag).setCoder(input.getCoder()))
+              .and(routed.get(spilledRecordsTag).setCoder(input.getCoder()))
+              .apply("FlattenSmallBundles", Flatten.pCollections())
+              .setCoder(input.getCoder());
+      PCollection<FileResult<DestinationT>> groupedFiles;
+      if (getComputeNumShards() != null || getNumShardsProvider() != null) {
+        groupedFiles =
+            smallBundles
+                .apply(
+                    "WriteShardedBundlesToTempFiles",
+                    new WriteShardedBundlesToTempFiles(
+                        destinationCoder, fileResultCoder, numShardsView))
+                .apply(
+                    "DropShardNum",
+                    ParDo.of(
+                        new DoFn<FileResult<DestinationT>, 
FileResult<DestinationT>>() {
+                          @ProcessElement
+                          public void process(ProcessContext c) {
+                            c.output(c.element().withShard(UNKNOWN_SHARDNUM));
+                          }
+                        }))
+                .setCoder(fileResultCoder);

Review Comment:
   If we are supporting only autosharding, this block can be removed.



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1114,6 +1208,301 @@ public PCollection<FileResult<DestinationT>> 
expand(PCollection<UserT> input) {
     }
   }
 
+  /**
+   * Streaming write that sends large bundles straight to temporary files.
+   *
+   * <p>Small bundles, and records that would exceed the per-bundle writer 
limit, take the usual
+   * grouped path ({@link WriteShardedBundlesToTempFiles} when a shard count 
is set, or the {@link
+   * GroupIntoBatches} auto-sharded path otherwise). All files then drop their 
shard number and are
+   * grouped per destination, so finalization assigns shard numbers across 
both paths.
+   */
+  private class WriteShardedBundlesWithDirectWrite
+      extends PTransform<PCollection<UserT>, 
PCollection<List<FileResult<DestinationT>>>> {
+    private final Coder<DestinationT> destinationCoder;
+    private final Coder<FileResult<DestinationT>> fileResultCoder;
+    private final @Nullable PCollectionView<Integer> numShardsView;
+    private final long thresholdBytes;
+
+    private WriteShardedBundlesWithDirectWrite(
+        Coder<DestinationT> destinationCoder,
+        Coder<FileResult<DestinationT>> fileResultCoder,
+        @Nullable PCollectionView<Integer> numShardsView,
+        long thresholdBytes) {
+      this.destinationCoder = destinationCoder;
+      this.fileResultCoder = fileResultCoder;
+      this.numShardsView = numShardsView;
+      this.thresholdBytes = thresholdBytes;
+    }
+
+    @Override
+    public PCollection<List<FileResult<DestinationT>>> 
expand(PCollection<UserT> input) {
+      TupleTag<UserT> smallBundlesTag = new TupleTag<>("smallBundles");
+      TupleTag<UserT> largeBundlesTag = new TupleTag<>("largeBundles");
+      TupleTag<FileResult<DestinationT>> directFilesTag = new 
TupleTag<>("directFiles");
+      TupleTag<UserT> spilledRecordsTag = new TupleTag<>("spilledRecords");
+
+      // Evaluate the threshold per window, pane and destination, i.e. per 
output file.
+      PCollectionTuple lifted =
+          input.apply(
+              "LiftLargeBundles",
+              BundleLifter.of(smallBundlesTag, largeBundlesTag, 
thresholdBytes, input.getCoder())
+                  .withKeyFn(new 
DestinationKeyFn<>(getDynamicDestinations())));
+
+      PCollectionTuple routed =
+          lifted
+              .get(largeBundlesTag)
+              .setCoder(input.getCoder())
+              .apply(
+                  "DirectWriteLargeBundles",
+                  ParDo.of(new DirectWriteRecordsFn(spilledRecordsTag, 
input.getCoder()))
+                      .withSideInputs(getSideInputs())
+                      .withOutputTags(
+                          directFilesTag,
+                          TupleTagList.of(ImmutableList.of(spilledRecordsTag, 
BAD_RECORD_TAG))));
+      addErrorCollection(routed);
+
+      PCollection<UserT> smallBundles =
+          
PCollectionList.of(lifted.get(smallBundlesTag).setCoder(input.getCoder()))
+              .and(routed.get(spilledRecordsTag).setCoder(input.getCoder()))
+              .apply("FlattenSmallBundles", Flatten.pCollections())
+              .setCoder(input.getCoder());
+      PCollection<FileResult<DestinationT>> groupedFiles;
+      if (getComputeNumShards() != null || getNumShardsProvider() != null) {
+        groupedFiles =
+            smallBundles
+                .apply(
+                    "WriteShardedBundlesToTempFiles",
+                    new WriteShardedBundlesToTempFiles(
+                        destinationCoder, fileResultCoder, numShardsView))
+                .apply(
+                    "DropShardNum",
+                    ParDo.of(
+                        new DoFn<FileResult<DestinationT>, 
FileResult<DestinationT>>() {
+                          @ProcessElement
+                          public void process(ProcessContext c) {
+                            c.output(c.element().withShard(UNKNOWN_SHARDNUM));
+                          }
+                        }))
+                .setCoder(fileResultCoder);

Review Comment:
   If we are supporting only autosharding, i think this block can be removed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to