tejasiyer-dev commented on code in PR #41468:
URL: https://github.com/apache/beam/pull/41468#discussion_r4627887281


##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.
+    private long bufferedBytes;
+    // Set once the memory cap is exceeded or the lifted key limit is reached 
in this bundle: keys
+    // that are neither buffered nor lifted are no longer tracked and go to 
the small output
+    // immediately.
+    private boolean keysFinalized;
 
     BundleLiftDoFn(
         TupleTag<T> smallBatchTag,
         TupleTag<T> largeBatchTag,
-        int threshold,
-        SerializableFunction<T, Integer> elementSizer) {
+        long threshold,
+        long maxBufferedBytes,
+        int maxLiftedKeys,
+        SerializableFunction<T, Integer> elementSizer,
+        @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
       this.smallBatchTag = smallBatchTag;
       this.largeBatchTag = largeBatchTag;
       this.threshold = threshold;
+      this.maxBufferedBytes = maxBufferedBytes;
+      this.maxLiftedKeys = maxLiftedKeys;
       this.elementSizer = elementSizer;
+      this.keyFn = keyFn;
+    }
+
+    @Setup
+    public void setup() {
+      buffers = new LinkedHashMap<>();
+      liftedKeys = new HashSet<>();
     }
 
     @StartBundle
     public void startBundle() {
-      buffer = new ArrayList<>();
-      receiver = null;
-      bundleSizeBytes = 0L;
+      checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+      checkStateNotNull(liftedKeys, "Lifted keys should be set by 
setup.").clear();
+      bufferedBytes = 0L;
+      keysFinalized = false;
     }
 
     @ProcessElement
-    public void processElement(@Element T element, MultiOutputReceiver mor) {
-      if (receiver == null) {
-        receiver = mor;
+    public void processElement(
+        @Element T element,
+        @Timestamp Instant timestamp,
+        BoundedWindow window,
+        PaneInfo pane,
+        MultiOutputReceiver receiver) {
+      Map<Object, KeyBuffer<T>> buffers =
+          checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+      Set<Object> liftedKeys =
+          checkStateNotNull(this.liftedKeys, "Lifted keys should be set by 
setup.");
+
+      Object key;
+      if (keyFn == null) {
+        key = UNKEYED;
+      } else {
+        @Nullable Object userKey = keyFn.apply(element);
+        key = new WindowedKey(window, pane, userKey == null ? UNKEYED : 
userKey);

Review Comment:
   Agreed, we should always group by (window, pane) because downstream 
transforms process each window and pane separately anyway, so this avoids 
needing a separate config option. Updated BundleLifter to always group by 
window and pane.
   
   I think one issue with a GroupedBundleLifter emitting KV<K, Iterable<V>> is 
that it makes it harder to preserve per-element timestamps as well as element 
counts, and in WriteFiles (#40543) both downstream paths 
(WriteUnshardedTempFilesFn and GroupIntoBatches) expect individual UserT 
records, so changing to Iterable would require unwrapping them or changing 
those functions.
   
   But let me know if you would still prefer a separate GroupedBundleLifter 
class.
   
   



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.
+    private long bufferedBytes;
+    // Set once the memory cap is exceeded or the lifted key limit is reached 
in this bundle: keys
+    // that are neither buffered nor lifted are no longer tracked and go to 
the small output
+    // immediately.
+    private boolean keysFinalized;
 
     BundleLiftDoFn(
         TupleTag<T> smallBatchTag,
         TupleTag<T> largeBatchTag,
-        int threshold,
-        SerializableFunction<T, Integer> elementSizer) {
+        long threshold,
+        long maxBufferedBytes,
+        int maxLiftedKeys,
+        SerializableFunction<T, Integer> elementSizer,
+        @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
       this.smallBatchTag = smallBatchTag;
       this.largeBatchTag = largeBatchTag;
       this.threshold = threshold;
+      this.maxBufferedBytes = maxBufferedBytes;
+      this.maxLiftedKeys = maxLiftedKeys;
       this.elementSizer = elementSizer;
+      this.keyFn = keyFn;
+    }
+
+    @Setup
+    public void setup() {
+      buffers = new LinkedHashMap<>();

Review Comment:
   Yes, normal hashmap is sufficient. Switched. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to