tejasiyer-dev commented on code in PR #40543:
URL: https://github.com/apache/beam/pull/40543#discussion_r4648309401
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -579,8 +633,141 @@ public WriteFilesResult<DestinationT>
expand(PCollection<UserT> input) {
(getComputeNumShards() == null) ? null :
input.apply(getComputeNumShards());
boolean fixedSharding = getComputeNumShards() != null ||
getNumShardsProvider() != null;
+ // Records that take the grouped path, and files written directly (if any).
+ PCollection<UserT> groupedInput = input;
+ @Nullable PCollection<FileResult<DestinationT>> directFiles = null;
+ Long directWriteThreshold = getDirectWriteByteThreshold();
+ if (directWriteThreshold != null) {
+ List<String> disabledReasons = new ArrayList<>();
+ // Direct writes are only supported for unbounded inputs as of now.
+ if (input.isBounded() == IsBounded.BOUNDED) {
+ disabledReasons.add("the input is bounded");
+ }
+ // WriteFiles allows merging windows when a fixed shard count is set
(merged by
+ // GroupIntoShards's GroupByKey), whereas direct writes run before any
+ // GroupByKey and would
+ // use unmerged windows.
+ if (input.getWindowingStrategy().needsMerge()) {
+ disabledReasons.add("the windowing strategy uses merging windows");
+ }
+ // Accumulating panes re-emit earlier records in every firing, but
direct writes
+ // write each
+ // record only once.
+ if (input.getWindowingStrategy().getMode() ==
AccumulationMode.ACCUMULATING_FIRED_PANES) {
+ disabledReasons.add("the windowing strategy accumulates fired panes");
+ }
+ if (disabledReasons.isEmpty()) {
+ TupleTag<KV<DestinationT, UserT>> keyedTag = new
TupleTag<>("keyedRecords");
+ TupleTag<KV<DestinationT, UserT>> smallBundlesTag = new
TupleTag<>("smallBundles");
+ TupleTag<KV<DestinationT, UserT>> largeBundlesTag = new
TupleTag<>("largeBundles");
+ TupleTag<FileResult<DestinationT>> directFilesTag = new
TupleTag<>("directFiles");
+
+ PCollectionTuple keyed =
Review Comment:
Done. I grouped KeyByDestination, LiftLargeBundles, DirectWriteLargeBundles,
and DropDestinationKey under a WriteFilesDirectly PTransform
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -579,8 +633,141 @@ public WriteFilesResult<DestinationT>
expand(PCollection<UserT> input) {
(getComputeNumShards() == null) ? null :
input.apply(getComputeNumShards());
boolean fixedSharding = getComputeNumShards() != null ||
getNumShardsProvider() != null;
+ // Records that take the grouped path, and files written directly (if any).
+ PCollection<UserT> groupedInput = input;
+ @Nullable PCollection<FileResult<DestinationT>> directFiles = null;
+ Long directWriteThreshold = getDirectWriteByteThreshold();
+ if (directWriteThreshold != null) {
+ List<String> disabledReasons = new ArrayList<>();
+ // Direct writes are only supported for unbounded inputs as of now.
+ if (input.isBounded() == IsBounded.BOUNDED) {
+ disabledReasons.add("the input is bounded");
+ }
+ // WriteFiles allows merging windows when a fixed shard count is set
(merged by
+ // GroupIntoShards's GroupByKey), whereas direct writes run before any
+ // GroupByKey and would
+ // use unmerged windows.
+ if (input.getWindowingStrategy().needsMerge()) {
+ disabledReasons.add("the windowing strategy uses merging windows");
+ }
+ // Accumulating panes re-emit earlier records in every firing, but
direct writes
+ // write each
+ // record only once.
+ if (input.getWindowingStrategy().getMode() ==
AccumulationMode.ACCUMULATING_FIRED_PANES) {
+ disabledReasons.add("the windowing strategy accumulates fired panes");
+ }
+ if (disabledReasons.isEmpty()) {
+ TupleTag<KV<DestinationT, UserT>> keyedTag = new
TupleTag<>("keyedRecords");
+ TupleTag<KV<DestinationT, UserT>> smallBundlesTag = new
TupleTag<>("smallBundles");
+ TupleTag<KV<DestinationT, UserT>> largeBundlesTag = new
TupleTag<>("largeBundles");
+ TupleTag<FileResult<DestinationT>> directFilesTag = new
TupleTag<>("directFiles");
+
+ PCollectionTuple keyed =
+ input.apply(
+ "KeyByDestination",
+ ParDo.of(new KeyByDestinationFn(input.getCoder()))
+ .withSideInputs(getSideInputs())
+ .withOutputTags(keyedTag,
TupleTagList.of(BAD_RECORD_TAG)));
+ addErrorCollection(keyed);
+ PCollection<KV<DestinationT, UserT>> keyedInput =
+ keyed.get(keyedTag).setCoder(KvCoder.of(destinationCoder,
input.getCoder()));
+
+ // Evaluate the threshold per window, pane and destination, i.e. per
output
+ // file. Lifting
+ // at most maxNumWritersPerBundle keys means the direct write never
runs out of
+ // writers.
+ BundleLifter<KV<DestinationT, UserT>> lifter =
+ BundleLifter.of(
+ smallBundlesTag,
+ largeBundlesTag,
+ directWriteThreshold,
+ new ValueCoderSizer<>(input.getCoder()))
+ .withKeyFn(KV::getKey);
+ if (getMaxNumWritersPerBundle() > 0) {
+ lifter = lifter.withMaxLiftedKeys(getMaxNumWritersPerBundle());
+ }
+ PCollectionTuple lifted = keyedInput.apply("LiftLargeBundles", lifter);
+
+ PCollectionTuple written =
+ lifted
+ .get(largeBundlesTag)
+ .apply(
+ "DirectWriteLargeBundles",
+ ParDo.of(
+ new WriteKeyedUnshardedTempFilesFn(
+ destinationCoder,
+ input.getCoder(),
+ Metrics.counter(WriteFiles.class,
"directWriteBundles")))
+ .withSideInputs(getSideInputs())
+ .withOutputTags(directFilesTag,
TupleTagList.of(BAD_RECORD_TAG)));
+ addErrorCollection(written);
+ directFiles = written.get(directFilesTag).setCoder(fileResultCoder);
+ groupedInput =
+ lifted
+ .get(smallBundlesTag)
+ .apply("DropDestinationKey", Values.create())
+ .setCoder(input.getCoder());
Review Comment:
Removed. It does just propagate from Values.create()
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -579,8 +633,141 @@ public WriteFilesResult<DestinationT>
expand(PCollection<UserT> input) {
(getComputeNumShards() == null) ? null :
input.apply(getComputeNumShards());
boolean fixedSharding = getComputeNumShards() != null ||
getNumShardsProvider() != null;
+ // Records that take the grouped path, and files written directly (if any).
+ PCollection<UserT> groupedInput = input;
+ @Nullable PCollection<FileResult<DestinationT>> directFiles = null;
+ Long directWriteThreshold = getDirectWriteByteThreshold();
+ if (directWriteThreshold != null) {
+ List<String> disabledReasons = new ArrayList<>();
+ // Direct writes are only supported for unbounded inputs as of now.
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]