scwhittle commented on code in PR #40543:
URL: https://github.com/apache/beam/pull/40543#discussion_r4611875383
##########
runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java:
##########
@@ -82,7 +82,6 @@
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
Review Comment:
can this change be in a separate PR since it affects a lot of WriteFiles
options and would be nice to be documented in isolation
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +89,340 @@ public class BundleLifter<T> extends
PTransform<PCollection<T>, PCollectionTuple
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+ /** The default memory cap: twice the threshold. */
+ static long defaultMaxBufferedBytes(long threshold) {
+ return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+ }
/**
- * A DoFn that buffers elements within a bundle and outputs them to
different tags in
- * finish_bundle based on the total bundle size.
+ * A DoFn that buffers elements per key within a bundle and outputs them to
the large tag once a
+ * key reaches the threshold, or to the small tag otherwise.
*
* @param <T> The type of elements being processed.
*/
static class BundleLiftDoFn<T> extends DoFn<T, Void> {
private static final Logger LOG =
LoggerFactory.getLogger(BundleLiftDoFn.class);
+ /** The key shared by all elements without a key, or whose key function
returns null. */
+ private static final Object UNKEYED = new Object();
+
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
- private transient @MonotonicNonNull List<T> buffer;
- private transient long bundleSizeBytes;
- private transient @Nullable MultiOutputReceiver receiver;
+ // Per-bundle state, allocated once in setup and cleared at the start of
each bundle.
+ private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+ private transient @MonotonicNonNull Set<Object> liftedKeys;
+ // Reset in startBundle; not transient so SpotBugs doesn't flag them as
unrestored state.
+ private long bufferedBytes;
+ // Set once the memory cap is exceeded or the lifted key limit is reached
in this bundle: keys
+ // that are neither buffered nor lifted are no longer tracked and go to
the small output
+ // immediately.
+ private boolean keysFinalized;
BundleLiftDoFn(
TupleTag<T> smallBatchTag,
TupleTag<T> largeBatchTag,
- int threshold,
+ long threshold,
SerializableFunction<T, Integer> elementSizer) {
+ this(smallBatchTag, largeBatchTag, threshold, elementSizer, null);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ defaultMaxBufferedBytes(threshold),
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ maxBufferedBytes,
+ Integer.MAX_VALUE,
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ int maxLiftedKeys,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
Review Comment:
can we just have this constructor? The BundleLifter can always pass all the
params and tests can have internal helper methods if they want.
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1125,133 +1417,94 @@ private class WriteAutoShardedBundlesToTempFiles
private WriteAutoShardedBundlesToTempFiles(
Coder<DestinationT> destinationCoder,
Coder<FileResult<DestinationT>> fileResultCoder,
- Integer batchSize,
- Integer batchSizeBytes,
+ int batchSize,
+ int batchSizeBytes,
Duration maxBufferingDuration) {
this.destinationCoder = destinationCoder;
this.fileResultCoder = fileResultCoder;
- this.batchSize = batchSize != null ? batchSize :
FILE_TRIGGERING_RECORD_COUNT;
- this.batchSizeBytes = batchSizeBytes != null ? batchSizeBytes :
FILE_TRIGGERING_BYTE_COUNT;
- this.maxBufferingDuration =
- maxBufferingDuration != null
- ? maxBufferingDuration
- : FILE_TRIGGERING_RECORD_BUFFERING_DURATION;
+ this.batchSize = batchSize;
+ this.batchSizeBytes = batchSizeBytes;
+ this.maxBufferingDuration = maxBufferingDuration;
}
@Override
public PCollection<List<FileResult<DestinationT>>>
expand(PCollection<UserT> input) {
- // Auto-sharding is achieved via GroupIntoBatches.WithShardedKey which
shards, groups and at
- // the same time batches the input records. The sharding behavior
depends on runners. The
- // batching is per window and we also emit the batches if there are a
certain number of
- // records buffered or they have been buffered for a certain time,
controlled by
- // FILE_TRIGGERING_RECORD_COUNT and BUFFERING_DURATION respectively.
- //
- // TODO(https://github.com/apache/beam/issues/20928): The implementation
doesn't currently
- // work with merging windows.
- TupleTag<KV<Integer, UserT>> shardTag = new TupleTag<>("shardTag");
-
- PCollectionTuple shardedElements =
- input.apply(
- "KeyedByDestinationHash",
- ParDo.of(new KeyByDestinationHash(input.getCoder(),
destinationCoder))
- .withOutputTags(shardTag, TupleTagList.of(BAD_RECORD_TAG)));
- addErrorCollection(shardedElements);
-
- PCollection<KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>> shardedInput =
- shardedElements
- .get(shardTag)
- .setCoder(KvCoder.of(VarIntCoder.of(), input.getCoder()))
- .apply(
- "ShardAndBatch",
- GroupIntoBatches.<Integer, UserT>ofSize(batchSize)
- .withByteSize(batchSizeBytes)
- .withMaxBufferingDuration(maxBufferingDuration)
- .withShardedKey())
- .setCoder(
- KvCoder.of(
-
org.apache.beam.sdk.util.ShardedKey.Coder.of(VarIntCoder.of()),
- IterableCoder.of(input.getCoder())));
-
- TupleTag<FileResult<DestinationT>> writtenRecordsTag = new
TupleTag<>("writtenRecords");
- // Write grouped elements to temp files.
- PCollectionTuple writtenFiles =
- shardedInput
- .apply(
- "AddDummyShard",
- MapElements.via(
- new SimpleFunction<
- KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>,
- KV<ShardedKey<Integer>, Iterable<UserT>>>() {
- @Override
- public KV<ShardedKey<Integer>, Iterable<UserT>> apply(
- KV<org.apache.beam.sdk.util.ShardedKey<Integer>,
Iterable<UserT>>
- input) {
- // Add dummy shard since it is required by
WriteShardsIntoTempFilesFn. It
- // will be dropped after we generate the temp files.
- return KV.of(
- ShardedKey.of(input.getKey().getKey(),
DUMMY_SHARDNUM),
- input.getValue());
- }
- }))
- .setCoder(
- KvCoder.of(
- ShardedKeyCoder.of(VarIntCoder.of()),
IterableCoder.of(input.getCoder())))
- .apply(
- "WriteShardsIntoTempFiles",
- ParDo.of(new WriteShardsIntoTempFilesFn(input.getCoder()))
- .withSideInputs(getSideInputs())
- .withOutputTags(writtenRecordsTag,
TupleTagList.of(BAD_RECORD_TAG)));
-
- addErrorCollection(writtenFiles);
-
PCollection<FileResult<DestinationT>> tempFiles =
- writtenFiles
- .get(writtenRecordsTag)
- .setCoder(fileResultCoder)
- .apply(
- "DropShardNum",
- ParDo.of(
- new DoFn<FileResult<DestinationT>,
FileResult<DestinationT>>() {
- @ProcessElement
- public void process(ProcessContext c) {
- c.output(c.element().withShard(UNKNOWN_SHARDNUM));
- }
- }))
- .setCoder(fileResultCoder);
+ writeAutoShardedRecordsToTempFiles(
+ input,
+ destinationCoder,
+ fileResultCoder,
+ batchSize,
+ batchSizeBytes,
+ maxBufferingDuration);
// Group temp file results by destinations again to gather all the
results in the same window.
// This is needed since we don't have shard idx associated with each
temp file so have to rely
// on the indexing within a bundle.
- return tempFiles
- .apply(
- "KeyedByDestination",
- WithKeys.of(
- new SimpleFunction<FileResult<DestinationT>, DestinationT>()
{
- @Override
- public DestinationT apply(FileResult<DestinationT> input) {
- return input.getDestination();
- }
- }))
- .setCoder(KvCoder.of(destinationCoder, fileResultCoder))
- .apply(GroupByKey.create())
- .apply(
- "ExtractValuesToList",
- ParDo.of(
- new DoFn<
- KV<DestinationT, Iterable<FileResult<DestinationT>>>,
- List<FileResult<DestinationT>>>() {
- @ProcessElement
- public void processElement(
- @Element KV<DestinationT,
Iterable<FileResult<DestinationT>>> element,
- ProcessContext c) {
- List<FileResult<DestinationT>> result = new
ArrayList<>();
- for (FileResult<DestinationT> e : element.getValue()) {
- result.add(e);
- }
- c.output(result);
- }
- }))
- .setCoder(ListCoder.of(fileResultCoder));
+ // "GroupByKey" matches the default name this step has always had
(update compatibility).
+ return groupTempFilesByDestination(
+ tempFiles, destinationCoder, fileResultCoder, "GroupByKey", /*
stampFiringPane= */ false);
+ }
+ }
+
+ /**
+ * Groups temp file results by destination into one list per destination and
window firing. Shared
+ * by the auto-sharded write and the direct-write path. Produces the steps
{@code
+ * KeyedByDestination}, {@code groupByKeyName} and {@code
ExtractValuesToList}.
+ *
+ * @param stampFiringPane if true, each file is re-stamped with the firing's
pane and an unknown
+ * shard number so shard numbers can be assigned in finalize across all
files of the firing.
+ */
+ private PCollection<List<FileResult<DestinationT>>>
groupTempFilesByDestination(
+ PCollection<FileResult<DestinationT>> tempFiles,
+ Coder<DestinationT> destinationCoder,
+ Coder<FileResult<DestinationT>> fileResultCoder,
+ String groupByKeyName,
+ boolean stampFiringPane) {
+ return tempFiles
+ .apply(
+ "KeyedByDestination",
+ WithKeys.of(
+ new SimpleFunction<FileResult<DestinationT>, DestinationT>() {
+ @Override
+ public DestinationT apply(FileResult<DestinationT> input) {
+ return input.getDestination();
+ }
+ }))
+ .setCoder(KvCoder.of(destinationCoder, fileResultCoder))
+ .apply(groupByKeyName, GroupByKey.create())
+ .apply("ExtractValuesToList", ParDo.of(new
ExtractValuesToListFn<>(stampFiringPane)))
+ .setCoder(ListCoder.of(fileResultCoder));
+ }
+
+ private static class ExtractValuesToListFn<DestinationT>
+ extends DoFn<
+ KV<DestinationT, Iterable<FileResult<DestinationT>>>,
List<FileResult<DestinationT>>> {
+ private final boolean stampFiringPane;
+
+ ExtractValuesToListFn(boolean stampFiringPane) {
+ this.stampFiringPane = stampFiringPane;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ List<FileResult<DestinationT>> result = new ArrayList<>();
+ for (FileResult<DestinationT> e : c.element().getValue()) {
+ if (stampFiringPane) {
+ // Stamp every file with this firing's pane so files from different
firings of the
+ // same window get distinct final names.
+ result.add(
+ new FileResult<>(
Review Comment:
maybe we should add a withPane factory similar to withShard and then just do
result.add(e.withPane(c.pane()).withShard(UNKNOWN_SHARDNUM))
this will ensure that if more fields are added that they are propagated
correctly
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +89,340 @@ public class BundleLifter<T> extends
PTransform<PCollection<T>, PCollectionTuple
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+ /** The default memory cap: twice the threshold. */
+ static long defaultMaxBufferedBytes(long threshold) {
+ return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+ }
/**
- * A DoFn that buffers elements within a bundle and outputs them to
different tags in
- * finish_bundle based on the total bundle size.
+ * A DoFn that buffers elements per key within a bundle and outputs them to
the large tag once a
+ * key reaches the threshold, or to the small tag otherwise.
*
* @param <T> The type of elements being processed.
*/
static class BundleLiftDoFn<T> extends DoFn<T, Void> {
private static final Logger LOG =
LoggerFactory.getLogger(BundleLiftDoFn.class);
+ /** The key shared by all elements without a key, or whose key function
returns null. */
+ private static final Object UNKEYED = new Object();
+
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
- private transient @MonotonicNonNull List<T> buffer;
- private transient long bundleSizeBytes;
- private transient @Nullable MultiOutputReceiver receiver;
+ // Per-bundle state, allocated once in setup and cleared at the start of
each bundle.
+ private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+ private transient @MonotonicNonNull Set<Object> liftedKeys;
+ // Reset in startBundle; not transient so SpotBugs doesn't flag them as
unrestored state.
+ private long bufferedBytes;
+ // Set once the memory cap is exceeded or the lifted key limit is reached
in this bundle: keys
+ // that are neither buffered nor lifted are no longer tracked and go to
the small output
+ // immediately.
+ private boolean keysFinalized;
BundleLiftDoFn(
TupleTag<T> smallBatchTag,
TupleTag<T> largeBatchTag,
- int threshold,
+ long threshold,
SerializableFunction<T, Integer> elementSizer) {
+ this(smallBatchTag, largeBatchTag, threshold, elementSizer, null);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ defaultMaxBufferedBytes(threshold),
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ maxBufferedBytes,
+ Integer.MAX_VALUE,
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ int maxLiftedKeys,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
this.smallBatchTag = smallBatchTag;
this.largeBatchTag = largeBatchTag;
this.threshold = threshold;
+ this.maxBufferedBytes = maxBufferedBytes;
+ this.maxLiftedKeys = maxLiftedKeys;
this.elementSizer = elementSizer;
+ this.keyFn = keyFn;
+ }
+
+ @Setup
+ public void setup() {
+ buffers = new LinkedHashMap<>();
+ liftedKeys = new HashSet<>();
}
@StartBundle
public void startBundle() {
- buffer = new ArrayList<>();
- receiver = null;
- bundleSizeBytes = 0L;
+ checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+ checkStateNotNull(liftedKeys, "Lifted keys should be set by
setup.").clear();
+ bufferedBytes = 0L;
+ keysFinalized = false;
}
@ProcessElement
- public void processElement(@Element T element, MultiOutputReceiver mor) {
- if (receiver == null) {
- receiver = mor;
+ public void processElement(
+ @Element T element,
+ @Timestamp Instant timestamp,
+ BoundedWindow window,
+ PaneInfo pane,
+ MultiOutputReceiver receiver) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ Set<Object> liftedKeys =
+ checkStateNotNull(this.liftedKeys, "Lifted keys should be set by
setup.");
+
+ Object key;
+ if (keyFn == null) {
+ key = UNKEYED;
+ } else {
+ @Nullable Object userKey = keyFn.apply(element);
+ key = new WindowedKey(window, pane, userKey == null ? UNKEYED :
userKey);
+ }
+
+ if (liftedKeys.contains(key)) {
+ // This key already reached the threshold in this bundle; stream it
through.
+ receiver.get(largeBatchTag).output(element);
+ return;
+ }
+
+ KeyBuffer<T> buffer = buffers.get(key);
+ if (buffer == null) {
+ if (keysFinalized) {
+ // The memory cap was exceeded earlier in this bundle; don't track
new keys.
+ receiver.get(smallBatchTag).output(element);
+ return;
+ }
+ buffer = new KeyBuffer<>();
+ buffers.put(key, buffer);
+ }
+ long elementBytes = elementSizer.apply(element);
+ buffer.add(new BufferedElement<>(element, timestamp, window, pane),
elementBytes);
+ bufferedBytes += elementBytes;
+
+ if (buffer.sizeBytes >= threshold) {
+ buffers.remove(key);
+ bufferedBytes -= buffer.sizeBytes;
+ liftedKeys.add(key);
+ LOG.debug(
+ "Key reached {} estimated bytes; emitting {} buffered elements to
tag: '{}'",
+ buffer.sizeBytes,
+ buffer.elements.size(),
+ largeBatchTag.getId());
+ emit(buffer, receiver.get(largeBatchTag));
+ if (liftedKeys.size() >= maxLiftedKeys) {
+ // No other key can be lifted in this bundle; release the remaining
buffers.
+ keysFinalized = true;
+ OutputReceiver<T> small = receiver.get(smallBatchTag);
+ for (KeyBuffer<T> other : buffers.values()) {
+ emit(other, small);
+ }
+ buffers.clear();
+ bufferedBytes = 0L;
+ }
+ return;
+ }
+
+ // Anything that isn't in lifted keys or buffers is sent to small keys
immediately.
+ if (bufferedBytes > maxBufferedBytes) {
+ keysFinalized = true;
+ evictSmallestKeys(buffers, receiver.get(smallBatchTag));
}
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- buffer.add(element);
- bundleSizeBytes += elementSizer.apply(element);
}
@FinishBundle
- public void finishBundle() {
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- if (buffer.isEmpty()) {
- return;
+ public void finishBundle(FinishBundleContext context) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ for (KeyBuffer<T> buffer : buffers.values()) {
+ LOG.debug(
+ "Emitting {} elements of {} estimated bytes to tag: '{}'",
+ buffer.elements.size(),
+ buffer.sizeBytes,
+ smallBatchTag.getId());
+ for (BufferedElement<T> e : buffer.elements) {
+ context.output(smallBatchTag, e.value, e.timestamp, e.window);
+ }
Review Comment:
clear each buffer as you go to free up memory
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -17,30 +17,62 @@
*/
package org.apache.beam.sdk.util;
-import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
Review Comment:
this has gotten big, can we separate it to it's own PR?
##########
runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java:
##########
@@ -2673,28 +2672,15 @@ static class StreamingShardedWriteFactory<UserT,
DestinationT, OutputT>
numShards = DEFAULT_NUM_SHARDS;
}
- try {
- List<PCollectionView<?>> sideInputs =
- WriteFilesTranslation.getDynamicDestinationSideInputs(transform);
- FileBasedSink sink = WriteFilesTranslation.getSink(transform);
- WriteFiles<UserT, DestinationT, OutputT> replacement =
- WriteFiles.to(sink).withSideInputs(sideInputs);
- if (WriteFilesTranslation.isWindowedWrites(transform)) {
- replacement = replacement.withWindowedWrites();
- }
-
- if (WriteFilesTranslation.isAutoSharded(transform)) {
- replacement = replacement.withAutoSharding();
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
replacement);
- }
-
+ // Keep every option of the original transform and only set the shard
count.
+ WriteFiles<UserT, DestinationT, OutputT> original =
transform.getTransform();
+ if (original.getWithAutoSharding()) {
Review Comment:
Can we modify the matcher for the override so that it doesn't trigger if
autosharding is enabled?
Alternatively do we need to avoid setting the number of shards? It seems
like autosharding can just ignore it if desired.
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +89,340 @@ public class BundleLifter<T> extends
PTransform<PCollection<T>, PCollectionTuple
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+ /** The default memory cap: twice the threshold. */
+ static long defaultMaxBufferedBytes(long threshold) {
+ return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+ }
/**
- * A DoFn that buffers elements within a bundle and outputs them to
different tags in
- * finish_bundle based on the total bundle size.
+ * A DoFn that buffers elements per key within a bundle and outputs them to
the large tag once a
+ * key reaches the threshold, or to the small tag otherwise.
*
* @param <T> The type of elements being processed.
*/
static class BundleLiftDoFn<T> extends DoFn<T, Void> {
private static final Logger LOG =
LoggerFactory.getLogger(BundleLiftDoFn.class);
+ /** The key shared by all elements without a key, or whose key function
returns null. */
+ private static final Object UNKEYED = new Object();
+
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
+ final int maxLiftedKeys;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
- private transient @MonotonicNonNull List<T> buffer;
- private transient long bundleSizeBytes;
- private transient @Nullable MultiOutputReceiver receiver;
+ // Per-bundle state, allocated once in setup and cleared at the start of
each bundle.
+ private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+ private transient @MonotonicNonNull Set<Object> liftedKeys;
+ // Reset in startBundle; not transient so SpotBugs doesn't flag them as
unrestored state.
+ private long bufferedBytes;
+ // Set once the memory cap is exceeded or the lifted key limit is reached
in this bundle: keys
+ // that are neither buffered nor lifted are no longer tracked and go to
the small output
+ // immediately.
+ private boolean keysFinalized;
BundleLiftDoFn(
TupleTag<T> smallBatchTag,
TupleTag<T> largeBatchTag,
- int threshold,
+ long threshold,
SerializableFunction<T, Integer> elementSizer) {
+ this(smallBatchTag, largeBatchTag, threshold, elementSizer, null);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ defaultMaxBufferedBytes(threshold),
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ maxBufferedBytes,
+ Integer.MAX_VALUE,
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ int maxLiftedKeys,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
this.smallBatchTag = smallBatchTag;
this.largeBatchTag = largeBatchTag;
this.threshold = threshold;
+ this.maxBufferedBytes = maxBufferedBytes;
+ this.maxLiftedKeys = maxLiftedKeys;
this.elementSizer = elementSizer;
+ this.keyFn = keyFn;
+ }
+
+ @Setup
+ public void setup() {
+ buffers = new LinkedHashMap<>();
+ liftedKeys = new HashSet<>();
}
@StartBundle
public void startBundle() {
- buffer = new ArrayList<>();
- receiver = null;
- bundleSizeBytes = 0L;
+ checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+ checkStateNotNull(liftedKeys, "Lifted keys should be set by
setup.").clear();
+ bufferedBytes = 0L;
+ keysFinalized = false;
}
@ProcessElement
- public void processElement(@Element T element, MultiOutputReceiver mor) {
- if (receiver == null) {
- receiver = mor;
+ public void processElement(
+ @Element T element,
+ @Timestamp Instant timestamp,
+ BoundedWindow window,
+ PaneInfo pane,
+ MultiOutputReceiver receiver) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ Set<Object> liftedKeys =
+ checkStateNotNull(this.liftedKeys, "Lifted keys should be set by
setup.");
+
+ Object key;
+ if (keyFn == null) {
+ key = UNKEYED;
+ } else {
+ @Nullable Object userKey = keyFn.apply(element);
+ key = new WindowedKey(window, pane, userKey == null ? UNKEYED :
userKey);
+ }
+
+ if (liftedKeys.contains(key)) {
+ // This key already reached the threshold in this bundle; stream it
through.
+ receiver.get(largeBatchTag).output(element);
+ return;
+ }
+
+ KeyBuffer<T> buffer = buffers.get(key);
+ if (buffer == null) {
+ if (keysFinalized) {
+ // The memory cap was exceeded earlier in this bundle; don't track
new keys.
+ receiver.get(smallBatchTag).output(element);
+ return;
+ }
+ buffer = new KeyBuffer<>();
+ buffers.put(key, buffer);
+ }
+ long elementBytes = elementSizer.apply(element);
+ buffer.add(new BufferedElement<>(element, timestamp, window, pane),
elementBytes);
+ bufferedBytes += elementBytes;
+
+ if (buffer.sizeBytes >= threshold) {
+ buffers.remove(key);
+ bufferedBytes -= buffer.sizeBytes;
+ liftedKeys.add(key);
+ LOG.debug(
+ "Key reached {} estimated bytes; emitting {} buffered elements to
tag: '{}'",
+ buffer.sizeBytes,
+ buffer.elements.size(),
+ largeBatchTag.getId());
+ emit(buffer, receiver.get(largeBatchTag));
+ if (liftedKeys.size() >= maxLiftedKeys) {
+ // No other key can be lifted in this bundle; release the remaining
buffers.
+ keysFinalized = true;
+ OutputReceiver<T> small = receiver.get(smallBatchTag);
+ for (KeyBuffer<T> other : buffers.values()) {
+ emit(other, small);
+ }
+ buffers.clear();
+ bufferedBytes = 0L;
+ }
+ return;
+ }
+
+ // Anything that isn't in lifted keys or buffers is sent to small keys
immediately.
+ if (bufferedBytes > maxBufferedBytes) {
+ keysFinalized = true;
+ evictSmallestKeys(buffers, receiver.get(smallBatchTag));
}
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- buffer.add(element);
- bundleSizeBytes += elementSizer.apply(element);
}
@FinishBundle
- public void finishBundle() {
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- if (buffer.isEmpty()) {
- return;
+ public void finishBundle(FinishBundleContext context) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ for (KeyBuffer<T> buffer : buffers.values()) {
+ LOG.debug(
+ "Emitting {} elements of {} estimated bytes to tag: '{}'",
+ buffer.elements.size(),
+ buffer.sizeBytes,
+ smallBatchTag.getId());
+ for (BufferedElement<T> e : buffer.elements) {
+ context.output(smallBatchTag, e.value, e.timestamp, e.window);
Review Comment:
why is this different than emit? we don't seem to output the pane here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]