scwhittle commented on code in PR #41468:
URL: https://github.com/apache/beam/pull/41468#discussion_r4626011682


##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;

Review Comment:
   // Once a key reaches this threshold all elements for the key are output to 
largeBatchTag.



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */

Review Comment:
   nit: rm comment, doesn't add much



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.

Review Comment:
   nit: remove second part of the comment.



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -149,33 +409,116 @@ private BundleLifter(
     if (threshold <= 0) {
       throw new IllegalArgumentException("Threshold must be a positive 
integer");
     }
+    if (maxBufferedBytes < threshold) {
+      throw new IllegalArgumentException(
+          "maxBufferedBytes must be at least the threshold, so a key can reach 
the threshold");
+    }
+    if (maxLiftedKeys <= 0) {
+      throw new IllegalArgumentException("maxLiftedKeys must be positive");
+    }
 
     this.smallBatchTag = smallBatchTag;
     this.largeBatchTag = largeBatchTag;
     this.threshold = threshold;
+    this.maxBufferedBytes = maxBufferedBytes;
+    this.maxLiftedKeys = maxLiftedKeys;
     this.elementSizer = elementSizer;
+    this.keyFn = keyFn;
   }
 
+  /** Routes by element count: each element counts as one toward the 
threshold. */
   public static <T> BundleLifter<T> of(
-      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, int threshold) {
-    return new BundleLifter<>(smallBatchTag, largeBatchTag, threshold);
+      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, long threshold) {
+    return of(smallBatchTag, largeBatchTag, threshold, x -> 1);
   }
 
+  /** Routes by the sizes returned by {@code elementSizer}. */
   public static <T> BundleLifter<T> of(
       TupleTag<T> smallBatchTag,
       TupleTag<T> largeBatchTag,
-      int threshold,
+      long threshold,

Review Comment:
   sizeThreshold



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -149,33 +409,116 @@ private BundleLifter(
     if (threshold <= 0) {
       throw new IllegalArgumentException("Threshold must be a positive 
integer");
     }
+    if (maxBufferedBytes < threshold) {
+      throw new IllegalArgumentException(
+          "maxBufferedBytes must be at least the threshold, so a key can reach 
the threshold");
+    }
+    if (maxLiftedKeys <= 0) {
+      throw new IllegalArgumentException("maxLiftedKeys must be positive");
+    }
 
     this.smallBatchTag = smallBatchTag;
     this.largeBatchTag = largeBatchTag;
     this.threshold = threshold;
+    this.maxBufferedBytes = maxBufferedBytes;
+    this.maxLiftedKeys = maxLiftedKeys;
     this.elementSizer = elementSizer;
+    this.keyFn = keyFn;
   }
 
+  /** Routes by element count: each element counts as one toward the 
threshold. */
   public static <T> BundleLifter<T> of(
-      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, int threshold) {
-    return new BundleLifter<>(smallBatchTag, largeBatchTag, threshold);
+      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, long threshold) {

Review Comment:
   rename threshold to elementCountThreshold?



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -149,33 +409,116 @@ private BundleLifter(
     if (threshold <= 0) {
       throw new IllegalArgumentException("Threshold must be a positive 
integer");
     }
+    if (maxBufferedBytes < threshold) {
+      throw new IllegalArgumentException(
+          "maxBufferedBytes must be at least the threshold, so a key can reach 
the threshold");
+    }
+    if (maxLiftedKeys <= 0) {
+      throw new IllegalArgumentException("maxLiftedKeys must be positive");
+    }
 
     this.smallBatchTag = smallBatchTag;
     this.largeBatchTag = largeBatchTag;
     this.threshold = threshold;
+    this.maxBufferedBytes = maxBufferedBytes;
+    this.maxLiftedKeys = maxLiftedKeys;
     this.elementSizer = elementSizer;
+    this.keyFn = keyFn;
   }
 
+  /** Routes by element count: each element counts as one toward the 
threshold. */
   public static <T> BundleLifter<T> of(
-      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, int threshold) {
-    return new BundleLifter<>(smallBatchTag, largeBatchTag, threshold);
+      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, long threshold) {
+    return of(smallBatchTag, largeBatchTag, threshold, x -> 1);
   }
 
+  /** Routes by the sizes returned by {@code elementSizer}. */
   public static <T> BundleLifter<T> of(
       TupleTag<T> smallBatchTag,
       TupleTag<T> largeBatchTag,
-      int threshold,
+      long threshold,
       SerializableFunction<T, Integer> elementSizer) {
-    return new BundleLifter<>(smallBatchTag, largeBatchTag, threshold, 
elementSizer);
+    return new BundleLifter<>(
+        smallBatchTag,
+        largeBatchTag,
+        threshold,
+        defaultMaxBufferedBytes(threshold),
+        Integer.MAX_VALUE,
+        elementSizer,
+        null);
+  }
+
+  /** Routes by each element's encoded size under {@code coder}. */
+  public static <T> BundleLifter<T> of(
+      TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag, long threshold, 
Coder<T> coder) {

Review Comment:
   sizeThreshold



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.
+    private long bufferedBytes;
+    // Set once the memory cap is exceeded or the lifted key limit is reached 
in this bundle: keys
+    // that are neither buffered nor lifted are no longer tracked and go to 
the small output
+    // immediately.
+    private boolean keysFinalized;
 
     BundleLiftDoFn(
         TupleTag<T> smallBatchTag,
         TupleTag<T> largeBatchTag,
-        int threshold,
-        SerializableFunction<T, Integer> elementSizer) {
+        long threshold,
+        long maxBufferedBytes,
+        int maxLiftedKeys,
+        SerializableFunction<T, Integer> elementSizer,
+        @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
       this.smallBatchTag = smallBatchTag;
       this.largeBatchTag = largeBatchTag;
       this.threshold = threshold;
+      this.maxBufferedBytes = maxBufferedBytes;
+      this.maxLiftedKeys = maxLiftedKeys;
       this.elementSizer = elementSizer;
+      this.keyFn = keyFn;
+    }
+
+    @Setup
+    public void setup() {
+      buffers = new LinkedHashMap<>();

Review Comment:
   is normal hashmap sufficient?



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;

Review Comment:
   // Keys which have been emitted to the largeBundleTag and for which future 
elements should also be routed to the largeBundleTag.



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.
+    private long bufferedBytes;
+    // Set once the memory cap is exceeded or the lifted key limit is reached 
in this bundle: keys
+    // that are neither buffered nor lifted are no longer tracked and go to 
the small output
+    // immediately.
+    private boolean keysFinalized;
 
     BundleLiftDoFn(
         TupleTag<T> smallBatchTag,
         TupleTag<T> largeBatchTag,
-        int threshold,
-        SerializableFunction<T, Integer> elementSizer) {
+        long threshold,
+        long maxBufferedBytes,
+        int maxLiftedKeys,
+        SerializableFunction<T, Integer> elementSizer,
+        @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
       this.smallBatchTag = smallBatchTag;
       this.largeBatchTag = largeBatchTag;
       this.threshold = threshold;
+      this.maxBufferedBytes = maxBufferedBytes;
+      this.maxLiftedKeys = maxLiftedKeys;
       this.elementSizer = elementSizer;
+      this.keyFn = keyFn;
+    }
+
+    @Setup
+    public void setup() {
+      buffers = new LinkedHashMap<>();
+      liftedKeys = new HashSet<>();
     }
 
     @StartBundle
     public void startBundle() {
-      buffer = new ArrayList<>();
-      receiver = null;
-      bundleSizeBytes = 0L;
+      checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+      checkStateNotNull(liftedKeys, "Lifted keys should be set by 
setup.").clear();
+      bufferedBytes = 0L;
+      keysFinalized = false;
     }
 
     @ProcessElement
-    public void processElement(@Element T element, MultiOutputReceiver mor) {
-      if (receiver == null) {
-        receiver = mor;
+    public void processElement(
+        @Element T element,
+        @Timestamp Instant timestamp,
+        BoundedWindow window,
+        PaneInfo pane,
+        MultiOutputReceiver receiver) {
+      Map<Object, KeyBuffer<T>> buffers =
+          checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+      Set<Object> liftedKeys =
+          checkStateNotNull(this.liftedKeys, "Lifted keys should be set by 
setup.");
+
+      Object key;
+      if (keyFn == null) {
+        key = UNKEYED;
+      } else {
+        @Nullable Object userKey = keyFn.apply(element);
+        key = new WindowedKey(window, pane, userKey == null ? UNKEYED : 
userKey);

Review Comment:
   perhaps whether or not to group by window and pane should be specified 
separately than the user key? Or should we always group by window and pane?
   
   An idea would be to move the changes to this class to a new 
GroupedBundleLifter class that accepts KV<K, V> and emits KV<K, Iterable<V>>?  
Benefits of this would be that for uses like WriteFiles we wouldn't have to 
reapply the keying function (and hope it is consistent) and the KV<K, 
Iterable<V>> could be more efficient than looking up the key repeatedly in 
write files map (though once we've spilled a large tag and passing through we'd 
have to determine if we want to buffer locally and flush when limits are 
reached so there are larger iterables or pass through as single-element 
iterables).



##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +90,316 @@ public class BundleLifter<T> extends 
PTransform<PCollection<T>, PCollectionTuple
 
   final TupleTag<T> smallBatchTag;
   final TupleTag<T> largeBatchTag;
-  final int threshold;
+  final long threshold;
+  final long maxBufferedBytes;
+  final int maxLiftedKeys;
   final SerializableFunction<T, Integer> elementSizer;
+  final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+  /** The default memory cap: twice the threshold. */
+  static long defaultMaxBufferedBytes(long threshold) {
+    return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+  }
 
   /**
-   * A DoFn that buffers elements within a bundle and outputs them to 
different tags in
-   * finish_bundle based on the total bundle size.
+   * A DoFn that buffers elements per key within a bundle and outputs them to 
the large tag once a
+   * key reaches the threshold, or to the small tag otherwise.
    *
    * @param <T> The type of elements being processed.
    */
   static class BundleLiftDoFn<T> extends DoFn<T, Void> {
     private static final Logger LOG = 
LoggerFactory.getLogger(BundleLiftDoFn.class);
 
+    /** The key shared by all elements without a key, or whose key function 
returns null. */
+    private static final Object UNKEYED = new Object();
+
     final TupleTag<T> smallBatchTag;
     final TupleTag<T> largeBatchTag;
-    final int threshold;
+    final long threshold;
+    final long maxBufferedBytes;
+    final int maxLiftedKeys;
     final SerializableFunction<T, Integer> elementSizer;
+    final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
 
-    private transient @MonotonicNonNull List<T> buffer;
-    private transient long bundleSizeBytes;
-    private transient @Nullable MultiOutputReceiver receiver;
+    // Per-bundle state, allocated once in setup and cleared at the start of 
each bundle.
+    private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+    private transient @MonotonicNonNull Set<Object> liftedKeys;
+    // Reset in startBundle; not transient so SpotBugs doesn't flag them as 
unrestored state.
+    private long bufferedBytes;
+    // Set once the memory cap is exceeded or the lifted key limit is reached 
in this bundle: keys
+    // that are neither buffered nor lifted are no longer tracked and go to 
the small output
+    // immediately.
+    private boolean keysFinalized;
 
     BundleLiftDoFn(
         TupleTag<T> smallBatchTag,
         TupleTag<T> largeBatchTag,
-        int threshold,
-        SerializableFunction<T, Integer> elementSizer) {
+        long threshold,
+        long maxBufferedBytes,
+        int maxLiftedKeys,
+        SerializableFunction<T, Integer> elementSizer,
+        @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
       this.smallBatchTag = smallBatchTag;
       this.largeBatchTag = largeBatchTag;
       this.threshold = threshold;
+      this.maxBufferedBytes = maxBufferedBytes;
+      this.maxLiftedKeys = maxLiftedKeys;
       this.elementSizer = elementSizer;
+      this.keyFn = keyFn;
+    }
+
+    @Setup
+    public void setup() {
+      buffers = new LinkedHashMap<>();
+      liftedKeys = new HashSet<>();
     }
 
     @StartBundle
     public void startBundle() {
-      buffer = new ArrayList<>();
-      receiver = null;
-      bundleSizeBytes = 0L;
+      checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+      checkStateNotNull(liftedKeys, "Lifted keys should be set by 
setup.").clear();
+      bufferedBytes = 0L;
+      keysFinalized = false;
     }
 
     @ProcessElement
-    public void processElement(@Element T element, MultiOutputReceiver mor) {
-      if (receiver == null) {
-        receiver = mor;
+    public void processElement(
+        @Element T element,
+        @Timestamp Instant timestamp,
+        BoundedWindow window,
+        PaneInfo pane,
+        MultiOutputReceiver receiver) {
+      Map<Object, KeyBuffer<T>> buffers =
+          checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+      Set<Object> liftedKeys =
+          checkStateNotNull(this.liftedKeys, "Lifted keys should be set by 
setup.");
+
+      Object key;
+      if (keyFn == null) {
+        key = UNKEYED;
+      } else {
+        @Nullable Object userKey = keyFn.apply(element);
+        key = new WindowedKey(window, pane, userKey == null ? UNKEYED : 
userKey);
+      }
+
+      if (liftedKeys.contains(key)) {
+        // This key already reached the threshold in this bundle; stream it 
through.
+        receiver
+            .get(largeBatchTag)
+            .outputWindowedValue(element, timestamp, 
Collections.singletonList(window), pane);
+        return;
+      }
+
+      KeyBuffer<T> buffer = buffers.get(key);
+      if (buffer == null) {
+        if (keysFinalized) {
+          // The memory cap was exceeded earlier in this bundle; don't track 
new keys.
+          receiver
+              .get(smallBatchTag)
+              .outputWindowedValue(element, timestamp, 
Collections.singletonList(window), pane);
+          return;
+        }
+        buffer = new KeyBuffer<>();
+        buffers.put(key, buffer);
+      }
+      long elementBytes = elementSizer.apply(element);
+      buffer.add(new BufferedElement<>(element, timestamp, window, pane), 
elementBytes);
+      bufferedBytes += elementBytes;
+
+      if (buffer.sizeBytes >= threshold) {
+        buffers.remove(key);
+        bufferedBytes -= buffer.sizeBytes;
+        liftedKeys.add(key);
+        LOG.debug(
+            "Key reached {} estimated bytes; emitting {} buffered elements to 
tag: '{}'",
+            buffer.sizeBytes,
+            buffer.elements.size(),
+            largeBatchTag.getId());
+        emit(buffer, receiver.get(largeBatchTag));
+        buffer.elements.clear();
+        if (liftedKeys.size() >= maxLiftedKeys) {
+          // No other key can be lifted in this bundle; release the remaining 
buffers.
+          keysFinalized = true;
+          OutputReceiver<T> small = receiver.get(smallBatchTag);
+          Iterator<KeyBuffer<T>> it = buffers.values().iterator();
+          while (it.hasNext()) {
+            KeyBuffer<T> other = it.next();
+            it.remove();
+            emit(other, small);
+            other.elements.clear();
+          }
+          bufferedBytes = 0L;
+        }
+        return;
+      }
+
+      // Anything that isn't in lifted keys or buffers is sent to small keys 
immediately.
+      if (bufferedBytes > maxBufferedBytes) {
+        keysFinalized = true;
+        evictSmallestKeys(buffers, receiver.get(smallBatchTag));
       }
-      checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
-      buffer.add(element);
-      bundleSizeBytes += elementSizer.apply(element);
     }
 
     @FinishBundle
-    public void finishBundle() {
-      checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
-      if (buffer.isEmpty()) {
-        return;
+    public void finishBundle(FinishBundleContext context) {
+      Map<Object, KeyBuffer<T>> buffers =
+          checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+      Iterator<KeyBuffer<T>> it = buffers.values().iterator();
+      while (it.hasNext()) {
+        KeyBuffer<T> buffer = it.next();
+        it.remove();
+        LOG.debug(
+            "Emitting {} elements of {} estimated bytes to tag: '{}'",
+            buffer.elements.size(),
+            buffer.sizeBytes,
+            smallBatchTag.getId());
+        // FinishBundleContext.output has no PaneInfo overload, and 
MultiOutputReceiver from

Review Comment:
   I think we should add an outputWindowedValue to FinishBundleContext so that 
we can output consistently.  From some quick peeking it seems that the existing 
impls would be easily able to support this.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to