arunpandianp commented on code in PR #40543:
URL: https://github.com/apache/beam/pull/40543#discussion_r4559680678
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1114,6 +1182,318 @@ public PCollection<FileResult<DestinationT>>
expand(PCollection<UserT> input) {
}
}
+ /**
+ * Streaming write that sends large bundles straight to temporary files.
+ *
+ * <p>Small bundles, and records that would exceed the per-bundle writer
limit, take the usual
+ * grouped path ({@link WriteShardedBundlesToTempFiles} when a shard count
is set, or the {@link
+ * GroupIntoBatches} auto-sharded path otherwise). All files then drop their
shard number and are
+ * grouped per destination, so finalization assigns shard numbers across
both paths.
+ */
+ private class WriteShardedBundlesWithDirectWrite
+ extends PTransform<PCollection<UserT>,
PCollection<List<FileResult<DestinationT>>>> {
+ private final Coder<DestinationT> destinationCoder;
+ private final Coder<FileResult<DestinationT>> fileResultCoder;
+ private final @Nullable PCollectionView<Integer> numShardsView;
+ private final long thresholdBytes;
+
+ private WriteShardedBundlesWithDirectWrite(
+ Coder<DestinationT> destinationCoder,
+ Coder<FileResult<DestinationT>> fileResultCoder,
+ @Nullable PCollectionView<Integer> numShardsView,
+ long thresholdBytes) {
+ this.destinationCoder = destinationCoder;
+ this.fileResultCoder = fileResultCoder;
+ this.numShardsView = numShardsView;
+ this.thresholdBytes = thresholdBytes;
+ }
+
+ @Override
+ public PCollection<List<FileResult<DestinationT>>>
expand(PCollection<UserT> input) {
+ TupleTag<UserT> smallBundlesTag = new TupleTag<>("smallBundles");
+ TupleTag<UserT> largeBundlesTag = new TupleTag<>("largeBundles");
+ TupleTag<FileResult<DestinationT>> directFilesTag = new
TupleTag<>("directFiles");
+ TupleTag<UserT> spilledRecordsTag = new TupleTag<>("spilledRecords");
+
+ // Evaluate the threshold per window, pane and destination, i.e. per
output file.
+ PCollectionTuple lifted =
+ input.apply(
+ "LiftLargeBundles",
+ BundleLifter.of(smallBundlesTag, largeBundlesTag,
thresholdBytes, input.getCoder())
+ .withKeyFn(new
DestinationKeyFn<>(getDynamicDestinations())));
+
+ PCollectionTuple routed =
+ lifted
+ .get(largeBundlesTag)
+ .setCoder(input.getCoder())
+ .apply(
+ "DirectWriteLargeBundles",
+ ParDo.of(new DirectWriteRecordsFn(spilledRecordsTag,
input.getCoder()))
+ .withSideInputs(getSideInputs())
+ .withOutputTags(
+ directFilesTag,
+ TupleTagList.of(ImmutableList.of(spilledRecordsTag,
BAD_RECORD_TAG))));
+ addErrorCollection(routed);
+
+ PCollection<UserT> smallBundles =
+
PCollectionList.of(lifted.get(smallBundlesTag).setCoder(input.getCoder()))
+ .and(routed.get(spilledRecordsTag).setCoder(input.getCoder()))
+ .apply("FlattenSmallBundles", Flatten.pCollections())
+ .apply("CountGroupedBundles", ParDo.of(new
CountGroupedBundlesFn<>()))
+ .setCoder(input.getCoder());
+ PCollection<FileResult<DestinationT>> groupedFiles;
+ if (getComputeNumShards() != null || getNumShardsProvider() != null) {
+ groupedFiles =
+ smallBundles
+ .apply(
+ "WriteShardedBundlesToTempFiles",
+ new WriteShardedBundlesToTempFiles(
+ destinationCoder, fileResultCoder, numShardsView))
+ .apply(
+ "DropShardNum",
+ ParDo.of(
+ new DoFn<FileResult<DestinationT>,
FileResult<DestinationT>>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ c.output(c.element().withShard(UNKNOWN_SHARDNUM));
+ }
+ }))
+ .setCoder(fileResultCoder);
+ } else {
+ groupedFiles =
+ writeAutoShardedRecordsToTempFiles(
+ smallBundles,
+ destinationCoder,
+ fileResultCoder,
+ getBatchSizeOrDefault(),
+ getBatchSizeBytesOrDefault(),
+ getBatchMaxBufferingDurationOrDefault());
+ }
+
+ // When the small path goes through GroupIntoShards (GBK), its trigger
transitions to the
+ // continuation trigger. Set the direct path to the same windowing
strategy so
+ // FlattenTempFiles does not reject mismatched triggers when user
pipelines use non-default
+ // triggers.
+ PCollection<FileResult<DestinationT>> directFiles =
+ routed
+ .get(directFilesTag)
+ .setCoder(fileResultCoder)
+
.setWindowingStrategyInternal(groupedFiles.getWindowingStrategy());
+
+ // Group all temp files per destination so each window/pane firing is
finalized together
+ // and shard numbers are assigned once across both paths. Every file is
stamped with this
+ // firing's pane so files from different firings of the same window get
distinct names.
+ PCollection<FileResult<DestinationT>> allTempFiles =
+ PCollectionList.of(directFiles)
+ .and(groupedFiles)
+ .apply("FlattenTempFiles", Flatten.pCollections())
+ .setCoder(fileResultCoder);
+ return groupTempFilesByDestination(
+ allTempFiles,
+ destinationCoder,
+ fileResultCoder,
+ "GroupTempFilesByDestination",
+ /* stampFiringPane= */ true);
+ }
+ }
+
+ /**
+ * Key function for {@link BundleLifter}: the record's destination, so that
together with the
+ * window and pane the threshold is evaluated per output file. Returns
{@code null} if the
+ * destination cannot be computed, which sends the record to the grouped
path where the usual
+ * error handling applies.
+ */
+ private static class DestinationKeyFn<UserT, DestinationT>
+ implements SerializableFunction<UserT, @Nullable Object> {
+ private final DynamicDestinations<UserT, DestinationT, ?>
dynamicDestinations;
+
+ DestinationKeyFn(DynamicDestinations<UserT, DestinationT, ?>
dynamicDestinations) {
+ this.dynamicDestinations = dynamicDestinations;
+ }
+
+ @Override
+ public @Nullable Object apply(UserT record) {
+ try {
+ // Wrapped so that a null destination (e.g. Void) is still a valid key.
+ return
java.util.Optional.ofNullable(dynamicDestinations.getDestination(record));
Review Comment:
import?
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1114,6 +1208,301 @@ public PCollection<FileResult<DestinationT>>
expand(PCollection<UserT> input) {
}
}
+ /**
+ * Streaming write that sends large bundles straight to temporary files.
+ *
+ * <p>Small bundles, and records that would exceed the per-bundle writer
limit, take the usual
+ * grouped path ({@link WriteShardedBundlesToTempFiles} when a shard count
is set, or the {@link
+ * GroupIntoBatches} auto-sharded path otherwise). All files then drop their
shard number and are
+ * grouped per destination, so finalization assigns shard numbers across
both paths.
+ */
+ private class WriteShardedBundlesWithDirectWrite
+ extends PTransform<PCollection<UserT>,
PCollection<List<FileResult<DestinationT>>>> {
+ private final Coder<DestinationT> destinationCoder;
+ private final Coder<FileResult<DestinationT>> fileResultCoder;
+ private final @Nullable PCollectionView<Integer> numShardsView;
+ private final long thresholdBytes;
+
+ private WriteShardedBundlesWithDirectWrite(
+ Coder<DestinationT> destinationCoder,
+ Coder<FileResult<DestinationT>> fileResultCoder,
+ @Nullable PCollectionView<Integer> numShardsView,
+ long thresholdBytes) {
+ this.destinationCoder = destinationCoder;
+ this.fileResultCoder = fileResultCoder;
+ this.numShardsView = numShardsView;
+ this.thresholdBytes = thresholdBytes;
+ }
+
+ @Override
+ public PCollection<List<FileResult<DestinationT>>>
expand(PCollection<UserT> input) {
+ TupleTag<UserT> smallBundlesTag = new TupleTag<>("smallBundles");
+ TupleTag<UserT> largeBundlesTag = new TupleTag<>("largeBundles");
+ TupleTag<FileResult<DestinationT>> directFilesTag = new
TupleTag<>("directFiles");
+ TupleTag<UserT> spilledRecordsTag = new TupleTag<>("spilledRecords");
Review Comment:
I think we can add a maxLiftedKeys param to BundleLifter and get rid of
spilledRecordsTag and the spill logic in DirectWriteRecordsFn.
##########
runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java:
##########
@@ -2683,6 +2685,11 @@ static class StreamingShardedWriteFactory<UserT,
DestinationT, OutputT>
replacement = replacement.withWindowedWrites();
}
+ Long directWriteThreshold = getDirectWriteThreshold(transform);
+ if (directWriteThreshold != null) {
+ replacement =
replacement.withDirectWriteByteThreshold(directWriteThreshold);
Review Comment:
why do we need to copy directWriteThreshold this here? WriteFiles have many
other options (eg: batchSize) that are not here.
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -923,30 +991,56 @@ public void finishBundle(FinishBundleContext c) throws
Exception {
BoundedWindow window = evictedResult.getWindow();
c.output(evictedResult, window.maxTimestamp(), window);
}
- for (Map.Entry<WriterKey<DestinationT>, Writer<DestinationT, OutputT>>
entry :
- writers.entrySet()) {
- WriterKey<DestinationT> key = entry.getKey();
- Writer<DestinationT, OutputT> writer = entry.getValue();
- try {
- writer.close();
- } catch (Exception e) {
- // If anything goes wrong, make sure to delete the temporary file.
- writer.cleanup();
- throw e;
- }
- BoundedWindow window = key.window;
- c.output(
- new FileResult<>(
- writer.getOutputFile(), UNKNOWN_SHARDNUM, window,
key.paneInfo, key.destination),
- window.maxTimestamp(),
- window);
- }
+ closeAllWritersAndOutputFileResults(writers, c);
writers.clear();
evictedFileResults.clear();
}
}
}
+ private Writer<DestinationT, OutputT> openAndRegisterWriter(
+ Map<WriterKey<DestinationT>, Writer<DestinationT, OutputT>> writers,
Review Comment:
do we need to pass in writers? writers is a class variable, can we access it
directly without passing it in?
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java:
##########
@@ -1114,6 +1208,301 @@ public PCollection<FileResult<DestinationT>>
expand(PCollection<UserT> input) {
}
}
+ /**
+ * Streaming write that sends large bundles straight to temporary files.
+ *
+ * <p>Small bundles, and records that would exceed the per-bundle writer
limit, take the usual
+ * grouped path ({@link WriteShardedBundlesToTempFiles} when a shard count
is set, or the {@link
+ * GroupIntoBatches} auto-sharded path otherwise). All files then drop their
shard number and are
+ * grouped per destination, so finalization assigns shard numbers across
both paths.
+ */
+ private class WriteShardedBundlesWithDirectWrite
+ extends PTransform<PCollection<UserT>,
PCollection<List<FileResult<DestinationT>>>> {
+ private final Coder<DestinationT> destinationCoder;
+ private final Coder<FileResult<DestinationT>> fileResultCoder;
+ private final @Nullable PCollectionView<Integer> numShardsView;
+ private final long thresholdBytes;
+
+ private WriteShardedBundlesWithDirectWrite(
+ Coder<DestinationT> destinationCoder,
+ Coder<FileResult<DestinationT>> fileResultCoder,
+ @Nullable PCollectionView<Integer> numShardsView,
+ long thresholdBytes) {
+ this.destinationCoder = destinationCoder;
+ this.fileResultCoder = fileResultCoder;
+ this.numShardsView = numShardsView;
+ this.thresholdBytes = thresholdBytes;
+ }
+
+ @Override
+ public PCollection<List<FileResult<DestinationT>>>
expand(PCollection<UserT> input) {
+ TupleTag<UserT> smallBundlesTag = new TupleTag<>("smallBundles");
+ TupleTag<UserT> largeBundlesTag = new TupleTag<>("largeBundles");
+ TupleTag<FileResult<DestinationT>> directFilesTag = new
TupleTag<>("directFiles");
+ TupleTag<UserT> spilledRecordsTag = new TupleTag<>("spilledRecords");
+
+ // Evaluate the threshold per window, pane and destination, i.e. per
output file.
+ PCollectionTuple lifted =
+ input.apply(
+ "LiftLargeBundles",
+ BundleLifter.of(smallBundlesTag, largeBundlesTag,
thresholdBytes, input.getCoder())
+ .withKeyFn(new
DestinationKeyFn<>(getDynamicDestinations())));
+
+ PCollectionTuple routed =
+ lifted
+ .get(largeBundlesTag)
+ .setCoder(input.getCoder())
+ .apply(
+ "DirectWriteLargeBundles",
+ ParDo.of(new DirectWriteRecordsFn(spilledRecordsTag,
input.getCoder()))
+ .withSideInputs(getSideInputs())
+ .withOutputTags(
+ directFilesTag,
+ TupleTagList.of(ImmutableList.of(spilledRecordsTag,
BAD_RECORD_TAG))));
+ addErrorCollection(routed);
+
+ PCollection<UserT> smallBundles =
+
PCollectionList.of(lifted.get(smallBundlesTag).setCoder(input.getCoder()))
+ .and(routed.get(spilledRecordsTag).setCoder(input.getCoder()))
+ .apply("FlattenSmallBundles", Flatten.pCollections())
+ .setCoder(input.getCoder());
+ PCollection<FileResult<DestinationT>> groupedFiles;
+ if (getComputeNumShards() != null || getNumShardsProvider() != null) {
+ groupedFiles =
+ smallBundles
+ .apply(
+ "WriteShardedBundlesToTempFiles",
+ new WriteShardedBundlesToTempFiles(
+ destinationCoder, fileResultCoder, numShardsView))
+ .apply(
+ "DropShardNum",
+ ParDo.of(
+ new DoFn<FileResult<DestinationT>,
FileResult<DestinationT>>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ c.output(c.element().withShard(UNKNOWN_SHARDNUM));
+ }
+ }))
+ .setCoder(fileResultCoder);
+ } else {
+ groupedFiles =
+ writeAutoShardedRecordsToTempFiles(
+ smallBundles,
+ destinationCoder,
+ fileResultCoder,
+ getBatchSizeOrDefault(),
+ getBatchSizeBytesOrDefault(),
+ getBatchMaxBufferingDurationOrDefault());
+ }
+
+ // When the small path goes through GroupIntoShards (GBK), its trigger
transitions to the
+ // continuation trigger. Set the direct path to the same windowing
strategy so
+ // FlattenTempFiles does not reject mismatched triggers when user
pipelines use non-default
+ // triggers.
+ PCollection<FileResult<DestinationT>> directFiles =
+ routed
+ .get(directFilesTag)
+ .setCoder(fileResultCoder)
+
.setWindowingStrategyInternal(groupedFiles.getWindowingStrategy());
+
+ // Group all temp files per destination so each window/pane firing is
finalized together
+ // and shard numbers are assigned once across both paths. Every file is
stamped with this
+ // firing's pane so files from different firings of the same window get
distinct names.
+ PCollection<FileResult<DestinationT>> allTempFiles =
+ PCollectionList.of(directFiles)
+ .and(groupedFiles)
+ .apply("FlattenTempFiles", Flatten.pCollections())
+ .setCoder(fileResultCoder);
+ return groupTempFilesByDestination(
+ allTempFiles,
+ destinationCoder,
+ fileResultCoder,
+ "GroupTempFilesByDestination",
+ /* stampFiringPane= */ true);
+ }
+ }
+
+ /**
+ * Key function for {@link BundleLifter}: the record's destination, so that
together with the
+ * window and pane the threshold is evaluated per output file. Returns
{@code null} if the
+ * destination cannot be computed; such records share one key, and the write
DoFn applies the
+ * usual error handling to them.
+ */
+ private static class DestinationKeyFn<UserT, DestinationT>
+ implements SerializableFunction<UserT, @Nullable Object> {
+ private final DynamicDestinations<UserT, DestinationT, ?>
dynamicDestinations;
+
+ DestinationKeyFn(DynamicDestinations<UserT, DestinationT, ?>
dynamicDestinations) {
+ this.dynamicDestinations = dynamicDestinations;
+ }
+
+ @Override
+ public @Nullable Object apply(UserT record) {
+ try {
+ return dynamicDestinations.getDestination(record);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Writes the records lifted by {@link BundleLifter} directly to temporary
files, one writer per
+ * window, pane and destination. Records that would exceed the per-bundle
writer limit are emitted
+ * to {@code spilledRecordsTag} for the grouped path.
+ */
+ private class DirectWriteRecordsFn extends DoFn<UserT,
FileResult<DestinationT>> {
Review Comment:
after removing the spill logic, we probably can replace it with
WriteUnshardedTempFilesFn?
##########
sdks/java/core/src/main/java/org/apache/beam/sdk/util/BundleLifter.java:
##########
@@ -57,89 +85,312 @@ public class BundleLifter<T> extends
PTransform<PCollection<T>, PCollectionTuple
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
+
+ /** The default memory cap: twice the threshold. */
+ static long defaultMaxBufferedBytes(long threshold) {
+ return threshold > Long.MAX_VALUE / 2 ? Long.MAX_VALUE : 2 * threshold;
+ }
/**
- * A DoFn that buffers elements within a bundle and outputs them to
different tags in
- * finish_bundle based on the total bundle size.
+ * A DoFn that buffers elements per key within a bundle and outputs them to
the large tag once a
+ * key reaches the threshold, or to the small tag otherwise.
*
* @param <T> The type of elements being processed.
*/
static class BundleLiftDoFn<T> extends DoFn<T, Void> {
private static final Logger LOG =
LoggerFactory.getLogger(BundleLiftDoFn.class);
+ /** The key shared by all elements without a key, or whose key function
returns null. */
+ private static final Object UNKEYED = new Object();
+
final TupleTag<T> smallBatchTag;
final TupleTag<T> largeBatchTag;
- final int threshold;
+ final long threshold;
+ final long maxBufferedBytes;
final SerializableFunction<T, Integer> elementSizer;
+ final @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn;
- private transient @MonotonicNonNull List<T> buffer;
- private transient long bundleSizeBytes;
- private transient @Nullable MultiOutputReceiver receiver;
+ // Per-bundle state, allocated once in setup and cleared at the start of
each bundle.
+ private transient @MonotonicNonNull Map<Object, KeyBuffer<T>> buffers;
+ private transient @MonotonicNonNull Set<Object> liftedKeys;
+ // Reset in startBundle; not transient so SpotBugs doesn't flag them as
unrestored state.
+ private long bufferedBytes;
+ // Set once the memory cap has been exceeded in this bundle: keys that are
neither buffered nor
+ // lifted are no longer tracked and go to the small output immediately.
+ private boolean keysFinalized;
BundleLiftDoFn(
TupleTag<T> smallBatchTag,
TupleTag<T> largeBatchTag,
- int threshold,
+ long threshold,
SerializableFunction<T, Integer> elementSizer) {
+ this(smallBatchTag, largeBatchTag, threshold, elementSizer, null);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
+ this(
+ smallBatchTag,
+ largeBatchTag,
+ threshold,
+ defaultMaxBufferedBytes(threshold),
+ elementSizer,
+ keyFn);
+ }
+
+ BundleLiftDoFn(
+ TupleTag<T> smallBatchTag,
+ TupleTag<T> largeBatchTag,
+ long threshold,
+ long maxBufferedBytes,
+ SerializableFunction<T, Integer> elementSizer,
+ @Nullable SerializableFunction<T, ? extends @Nullable Object> keyFn) {
this.smallBatchTag = smallBatchTag;
this.largeBatchTag = largeBatchTag;
this.threshold = threshold;
+ this.maxBufferedBytes = maxBufferedBytes;
this.elementSizer = elementSizer;
+ this.keyFn = keyFn;
+ }
+
+ @Setup
+ public void setup() {
+ buffers = new LinkedHashMap<>();
+ liftedKeys = new HashSet<>();
}
@StartBundle
public void startBundle() {
- buffer = new ArrayList<>();
- receiver = null;
- bundleSizeBytes = 0L;
+ checkStateNotNull(buffers, "Buffers should be set by setup.").clear();
+ checkStateNotNull(liftedKeys, "Lifted keys should be set by
setup.").clear();
+ bufferedBytes = 0L;
+ keysFinalized = false;
}
@ProcessElement
- public void processElement(@Element T element, MultiOutputReceiver mor) {
- if (receiver == null) {
- receiver = mor;
+ public void processElement(
+ @Element T element,
+ @Timestamp Instant timestamp,
+ BoundedWindow window,
+ PaneInfo pane,
+ MultiOutputReceiver receiver) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ Set<Object> liftedKeys =
+ checkStateNotNull(this.liftedKeys, "Lifted keys should be set by
setup.");
+
+ Object key;
+ if (keyFn == null) {
+ key = UNKEYED;
+ } else {
+ @Nullable Object userKey = keyFn.apply(element);
+ key = new WindowedKey(window, pane, userKey == null ? UNKEYED :
userKey);
+ }
+
+ if (liftedKeys.contains(key)) {
+ // This key already reached the threshold in this bundle; stream it
through.
+ receiver.get(largeBatchTag).output(element);
+ return;
+ }
+
+ KeyBuffer<T> buffer = buffers.get(key);
+ if (buffer == null) {
+ if (keysFinalized) {
+ // The memory cap was exceeded earlier in this bundle; don't track
new keys.
+ receiver.get(smallBatchTag).output(element);
+ return;
+ }
+ buffer = new KeyBuffer<>();
+ buffers.put(key, buffer);
+ }
+ long elementBytes = elementSizer.apply(element);
+ buffer.add(new BufferedElement<>(element, timestamp, window, pane),
elementBytes);
+ bufferedBytes += elementBytes;
+
+ if (buffer.sizeBytes >= threshold) {
+ buffers.remove(key);
+ bufferedBytes -= buffer.sizeBytes;
+ liftedKeys.add(key);
+ LOG.debug(
+ "Key reached {} estimated bytes; emitting {} buffered elements to
tag: '{}'",
+ buffer.sizeBytes,
+ buffer.elements.size(),
+ largeBatchTag.getId());
+ emit(buffer, receiver.get(largeBatchTag));
+ return;
+ }
+
+ // Anything that isn't in lifted keys or buffers is sent to small keys
immediately.
+ if (bufferedBytes > maxBufferedBytes) {
+ keysFinalized = true;
+ evictSmallestKeys(buffers, receiver.get(smallBatchTag));
}
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- buffer.add(element);
- bundleSizeBytes += elementSizer.apply(element);
}
@FinishBundle
- public void finishBundle() {
- checkArgumentNotNull(buffer, "Buffer should be set by startBundle.");
- if (buffer.isEmpty()) {
- return;
+ public void finishBundle(FinishBundleContext context) {
+ Map<Object, KeyBuffer<T>> buffers =
+ checkStateNotNull(this.buffers, "Buffers should be set by setup.");
+ for (KeyBuffer<T> buffer : buffers.values()) {
+ LOG.debug(
+ "Emitting {} elements of {} estimated bytes to tag: '{}'",
+ buffer.elements.size(),
+ buffer.sizeBytes,
+ smallBatchTag.getId());
+ for (BufferedElement<T> e : buffer.elements) {
+ context.output(smallBatchTag, e.value, e.timestamp, e.window);
+ }
}
+ buffers.clear();
+ checkStateNotNull(this.liftedKeys).clear();
+ bufferedBytes = 0L;
+ keysFinalized = false;
+ }
- // Select the target tag based on the bundle size
- TupleTag<T> targetTag;
- targetTag = (bundleSizeBytes < threshold) ? smallBatchTag :
largeBatchTag;
- LOG.debug(
- "Emitting {} elements of {} estimated bytes to tag: '{}'",
- buffer.size(),
- bundleSizeBytes,
- targetTag.getId());
+ /**
+ * Buffered elements are re-emitted from {@link ProcessElement} with their
own timestamps, which
+ * may be earlier than the current element's. This is safe because they
belong to the same
+ * bundle, so the input watermark is still held at or before them.
+ */
+ @Override
+ public Duration getAllowedTimestampSkew() {
+ return Duration.millis(Long.MAX_VALUE);
+ }
- checkArgumentNotNull(receiver, "Receiver should be set by startBundle.");
- OutputReceiver<T> taggedOutput = receiver.get(targetTag);
+ /**
+ * Emits the smallest keys to the small output until at most half of
{@code maxBufferedBytes}
+ * remains buffered, so each call frees a large chunk and the sort's cost
stays amortized.
+ */
+ private void evictSmallestKeys(Map<Object, KeyBuffer<T>> buffers,
OutputReceiver<T> small) {
+ long target = maxBufferedBytes / 2;
+ List<Map.Entry<Object, KeyBuffer<T>>> bySize = new
ArrayList<>(buffers.entrySet());
+ bySize.sort(Comparator.comparingLong(entry ->
entry.getValue().sizeBytes));
+ for (Map.Entry<Object, KeyBuffer<T>> entry : bySize) {
+ if (bufferedBytes <= target) {
+ break;
+ }
+ KeyBuffer<T> buffer = entry.getValue();
+ buffers.remove(entry.getKey());
+ bufferedBytes -= buffer.sizeBytes;
+ emit(buffer, small);
+ }
+ }
- for (T element : buffer) {
- taggedOutput.output(element);
+ private static <T> void emit(KeyBuffer<T> buffer, OutputReceiver<T>
receiver) {
+ for (BufferedElement<T> e : buffer.elements) {
+ receiver.outputWindowedValue(
+ e.value, e.timestamp, Collections.singletonList(e.window), e.pane);
}
}
}
- private BundleLifter(TupleTag<T> smallBatchTag, TupleTag<T> largeBatchTag,
int threshold) {
- this(smallBatchTag, largeBatchTag, threshold, x -> 1);
+ /** The buffered elements of one key, in arrival order. */
+ private static class KeyBuffer<T> {
+ private final List<BufferedElement<T>> elements = new ArrayList<>();
+ private long sizeBytes;
+
+ void add(BufferedElement<T> element, long elementBytes) {
+ elements.add(element);
+ sizeBytes += elementBytes;
+ }
+ }
+
+ /** A buffered element with the metadata needed to re-emit it. */
+ private static class BufferedElement<T> {
+ private final T value;
+ private final Instant timestamp;
+ private final BoundedWindow window;
+ private final PaneInfo pane;
+
+ BufferedElement(T value, Instant timestamp, BoundedWindow window, PaneInfo
pane) {
+ this.value = value;
+ this.timestamp = timestamp;
+ this.window = window;
+ this.pane = pane;
+ }
+ }
+
+ /** A user key scoped to a window and pane. */
+ private static class WindowedKey {
+ private final BoundedWindow window;
+ private final PaneInfo pane;
+ private final Object key;
+
+ WindowedKey(BoundedWindow window, PaneInfo pane, Object key) {
+ this.window = window;
+ this.pane = pane;
+ this.key = key;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof WindowedKey)) {
+ return false;
+ }
+ WindowedKey other = (WindowedKey) o;
+ return window.equals(other.window) && pane.equals(other.pane) &&
key.equals(other.key);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(window, pane, key);
+ }
+ }
+
+ /** Sizes elements by their encoded size under the given coder. */
+ private static class CoderSizer<T> implements SerializableFunction<T,
Integer> {
+ private final Coder<T> coder;
+
+ CoderSizer(Coder<T> coder) {
+ this.coder = coder;
+ }
+
+ @Override
+ public Integer apply(T element) {
+ try {
+ long bytes;
+ if (coder.isRegisterByteSizeObserverCheap(element)) {
+ SizeObserver observer = new SizeObserver();
+ coder.registerByteSizeObserver(element, observer);
+ observer.advance();
+ bytes = observer.bytes;
+ } else {
+ bytes = CoderUtils.encodeToByteArray(coder, element).length;
Review Comment:
using SizeObserver should be cheaper than encodeToByteArray. The
SizeObserver won't allocate the byte array.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]