This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from cdf314d30b5 [FLINK-34116][test] Enhance
GlobalConfigurationTest.testInvalidStandardYamlFile for JDK compatibility.
add 2839d06559c [FLINK-33743][runtime] Distinguish channel and subpartition
add c65d5f18ad5 [FLINK-33743][runtime] Replace consumer-side
SubpartitionId with InputChannelId
add 0e8b9808839 [FLINK-33743][runtime] Modify parameter subpartitionIndex
to subpartitionIndexSet
add 6b44f29a1ce [FLINK-33743][runtime] Identify subpartitionId in
notifyRequiredSegment
add f0436e53313 [FLINK-33743][runtime] Disable tier shuffle during recovery
add 439d1091daa [FLINK-33743][runtime] Acquire subpartition id for input
channel dynamically in tiered shuffle
add 4e6796dd147 [FLINK-33743][runtime] Change numCreditsAvailable to
isCreditAvailable in ResultSubpartitionView#getAvailabilityAndBacklog
add 8651f734b16 [FLINK-33743][runtime] Add ResultSubpartitionView
parameter to notifyDataAvailable() method
add 1199c7106a4 [FLINK-33743][runtime] Support consuming multiple
subpartition in one inputchannel
add f2417a74bd7 [FLINK-33743][runtime] Rename InputChannelStatus to
SubpartitionStatus
add 32de7521ac7 [FLINK-33743][runtime] Align watermark at subpartition
granularity
add 8358e3aa2ee [FLINK-33743][runtime] Ignore RecordAttributes when
adaptive parallelism is used
add be127310958 [FLINK-33743][runtime] Flush one accumulated buffer at a
time
add a886339dbb3 [FLINK-33743][runtime] Optimize partial record split logic
No new revisions were added by this update.
Summary of changes:
.../ResultPartitionDeploymentDescriptor.java | 4 +
.../flink/runtime/executiongraph/IndexRange.java | 4 +-
.../IntermediateResultPartition.java | 8 +
.../io/network/NetworkSequenceViewReader.java | 15 +-
.../runtime/io/network/PartitionRequestClient.java | 11 +-
.../runtime/io/network/api/RecoveryMetadata.java | 72 ++++
.../network/api/serialization/EventSerializer.java | 13 +
.../api/writer/ChannelSelectorRecordWriter.java | 10 +-
.../io/network/api/writer/RecordWriter.java | 30 +-
.../network/api/writer/ResultPartitionWriter.java | 8 +-
.../flink/runtime/io/network/buffer/Buffer.java | 50 ++-
.../runtime/io/network/buffer/BufferBuilder.java | 10 +
.../metrics/ExclusiveBuffersUsageGauge.java | 4 +-
.../network/metrics/FloatingBuffersUsageGauge.java | 2 +-
.../io/network/metrics/InputGateMetrics.java | 12 +-
.../CreditBasedPartitionRequestClientHandler.java | 5 +-
.../CreditBasedSequenceNumberingViewReader.java | 56 ++-
.../runtime/io/network/netty/NettyMessage.java | 58 ++-
.../network/netty/NettyPartitionRequestClient.java | 21 +-
.../netty/NettyPartitionRequestListener.java | 13 +-
.../io/network/netty/PartitionRequestQueue.java | 8 +-
.../netty/PartitionRequestServerHandler.java | 5 +-
...edBlockingSubpartitionDirectTransferReader.java | 11 +-
.../BoundedBlockingSubpartitionReader.java | 11 +-
.../partition/BufferAvailabilityListener.java | 8 +-
.../network/partition/BufferReaderWriterUtil.java | 18 +
...ithChannel.java => BufferWithSubpartition.java} | 14 +-
.../partition/BufferWritingResultPartition.java | 9 +-
.../runtime/io/network/partition/DataBuffer.java | 19 +-
.../io/network/partition/DeduplicatedQueue.java | 70 ++++
.../io/network/partition/HashBasedDataBuffer.java | 60 ++--
.../partition/NoOpResultSubpartitionView.java | 7 +-
.../network/partition/PartitionedFileWriter.java | 39 ++-
.../network/partition/PipelinedSubpartition.java | 4 +-
.../partition/PipelinedSubpartitionView.java | 11 +-
.../io/network/partition/ResultPartition.java | 38 ++
.../network/partition/ResultPartitionFactory.java | 11 +-
.../network/partition/ResultPartitionManager.java | 13 +-
.../network/partition/ResultPartitionProvider.java | 6 +-
.../partition/ResultSubpartitionIndexSet.java | 77 ++++
.../network/partition/ResultSubpartitionView.java | 15 +-
.../partition/RoundRobinSubpartitionSelector.java | 73 ++++
.../io/network/partition/SortBasedDataBuffer.java | 12 +-
.../runtime/io/network/partition/SortBuffer.java | 34 +-
.../partition/SortMergeResultPartition.java | 44 +--
.../partition/SortMergeSubpartitionReader.java | 11 +-
.../io/network/partition/SubpartitionSelector.java | 56 +++
.../partition/UnionResultSubpartitionView.java | 258 ++++++++++++++
.../network/partition/consumer/InputChannel.java | 60 +++-
.../partition/consumer/LocalInputChannel.java | 57 ++-
.../consumer/LocalRecoveredInputChannel.java | 7 +-
.../partition/consumer/RecoveredInputChannel.java | 13 +-
.../partition/consumer/RemoteInputChannel.java | 81 ++++-
.../consumer/RemoteRecoveredInputChannel.java | 7 +-
.../partition/consumer/SingleInputGate.java | 389 ++++++++++++++-------
.../partition/consumer/SingleInputGateFactory.java | 150 +++++---
.../partition/consumer/UnknownInputChannel.java | 16 +-
.../partition/hybrid/HsResultPartition.java | 10 +-
.../partition/hybrid/HsSubpartitionConsumer.java | 13 +-
.../tiered/common/TieredStorageInputChannelId.java | 58 +++
.../hybrid/tiered/netty/NettyConnectionReader.java | 10 +-
.../tiered/netty/NettyConnectionReaderImpl.java | 25 +-
.../tiered/netty/NettyConnectionWriterImpl.java | 12 +-
.../netty/TieredStorageNettyServiceImpl.java | 37 +-
.../netty/TieredStorageResultSubpartitionView.java | 18 +-
.../tiered/shuffle/TieredResultPartition.java | 4 +-
.../shuffle/TieredResultPartitionFactory.java | 20 +-
.../tiered/storage/AvailabilityNotifier.java | 11 +
.../hybrid/tiered/storage/BufferAccumulator.java | 9 +-
.../tiered/storage/HashBufferAccumulator.java | 24 +-
.../storage/HashSubpartitionBufferAccumulator.java | 44 ++-
.../HashSubpartitionBufferAccumulatorContext.java | 10 +-
.../tiered/storage/SortBufferAccumulator.java | 55 ++-
.../storage/TieredStorageConsumerClient.java | 22 ++
.../tiered/storage/TieredStorageConsumerSpec.java | 21 +-
.../storage/TieredStorageProducerClient.java | 80 ++---
.../tiered/storage/TieredStorageSortBuffer.java | 46 ++-
.../hybrid/tiered/tier/TierConsumerAgent.java | 13 +
.../hybrid/tiered/tier/TierProducerAgent.java | 17 +-
.../hybrid/tiered/tier/disk/DiskCacheManager.java | 11 +-
.../tiered/tier/disk/DiskTierConsumerAgent.java | 38 +-
.../tiered/tier/disk/DiskTierProducerAgent.java | 23 +-
.../tier/memory/MemoryTierConsumerAgent.java | 38 +-
.../tier/memory/MemoryTierProducerAgent.java | 18 +-
.../tier/remote/RemoteTierConsumerAgent.java | 85 ++++-
.../tiered/tier/remote/RemoteTierFactory.java | 5 +-
.../tier/remote/RemoteTierProducerAgent.java | 14 +-
.../flink/runtime/shuffle/PartitionDescriptor.java | 13 +-
.../SequentialChannelStateReaderImplTest.java | 4 +-
.../ResultPartitionDeploymentDescriptorTest.java | 3 +-
.../io/network/TestingPartitionRequestClient.java | 6 +-
.../api/serialization/EventSerializerTest.java | 14 +-
.../api/writer/BroadcastRecordWriterTest.java | 44 +--
.../api/writer/RecordWriterDelegateTest.java | 9 +-
.../io/network/api/writer/RecordWriterTest.java | 71 ++--
.../network/netty/CancelPartitionRequestTest.java | 28 +-
.../netty/ClientTransportErrorHandlingTest.java | 7 +-
...editBasedPartitionRequestClientHandlerTest.java | 24 +-
...CreditBasedSequenceNumberingViewReaderTest.java | 12 +-
.../NettyMessageClientDecoderDelegateTest.java | 4 +-
.../NettyMessageClientSideSerializationTest.java | 3 +-
.../NettyMessageServerSideSerializationTest.java | 6 +-
.../netty/NettyPartitionRequestClientTest.java | 14 +-
.../runtime/io/network/netty/NettyTestUtil.java | 1 +
.../network/netty/PartitionRequestQueueTest.java | 70 ++--
.../netty/PartitionRequestRegistrationTest.java | 13 +-
.../netty/PartitionRequestServerHandlerTest.java | 3 +-
.../netty/ServerTransportErrorHandlingTest.java | 9 +-
.../AwaitableBufferAvailablityListener.java | 2 +-
...oundedBlockingSubpartitionAvailabilityTest.java | 6 +-
.../BoundedBlockingSubpartitionWriteReadTest.java | 9 +-
.../partition/CountingAvailabilityListener.java | 2 +-
.../io/network/partition/DataBufferTest.java | 32 +-
.../partition/FileChannelBoundedDataTest.java | 5 +-
.../network/partition/InputChannelTestUtils.java | 5 +-
.../network/partition/InputGateFairnessTest.java | 20 +-
.../partition/MockResultPartitionWriter.java | 3 +-
.../partition/NoOpBufferAvailablityListener.java | 2 +-
.../PartitionRequestListenerManagerTest.java | 18 +-
.../io/network/partition/PartitionTestUtils.java | 11 +-
.../partition/PartitionedFileWriteReadTest.java | 39 ++-
.../PipelinedApproximateSubpartitionTest.java | 4 +-
.../PipelinedSubpartitionWithReadViewTest.java | 24 +-
.../network/partition/ResultPartitionBuilder.java | 3 +-
.../partition/ResultPartitionManagerTest.java | 27 +-
.../io/network/partition/ResultPartitionTest.java | 25 +-
.../SortMergeResultPartitionReadSchedulerTest.java | 6 +-
.../partition/SortMergeResultPartitionTest.java | 53 +--
.../partition/SortMergeSubpartitionReaderTest.java | 12 +-
.../io/network/partition/SubpartitionTestBase.java | 2 +-
.../partition/TestingPartitionRequestListener.java | 2 +-
.../network/partition/TestingResultPartition.java | 2 +-
.../partition/TestingResultPartitionProvider.java | 16 +-
.../TestingSubpartitionCreatedViewReader.java | 19 +-
.../partition/UnionResultSubpartitionViewTest.java | 185 ++++++++++
.../consumer/InputBuffersMetricsTest.java | 2 +-
.../partition/consumer/InputChannelBuilder.java | 18 +-
.../partition/consumer/InputChannelTest.java | 10 +-
.../partition/consumer/LocalInputChannelTest.java | 61 ++--
.../consumer/RecoveredInputChannelTest.java | 5 +-
.../partition/consumer/RemoteInputChannelTest.java | 83 ++---
.../partition/consumer/SingleInputGateBuilder.java | 16 +-
.../partition/consumer/SingleInputGateTest.java | 152 ++++----
.../partition/consumer/TestInputChannel.java | 12 +-
.../partition/hybrid/HsResultPartitionTest.java | 39 ++-
.../partition/hybrid/HsSubpartitionViewTest.java | 18 +-
.../hybrid/tiered/TestingBufferAccumulator.java | 17 +-
.../hybrid/tiered/TestingTierProducerAgent.java | 8 +-
.../tiered/netty/NettyConnectionReaderTest.java | 17 +-
.../tiered/netty/NettyConnectionWriterTest.java | 20 +-
.../tiered/netty/TestingNettyConnectionReader.java | 10 +-
.../tiered/netty/TestingTierConsumerAgent.java | 10 +
.../netty/TieredStorageConsumerClientTest.java | 14 +-
.../TieredStorageResultSubpartitionViewTest.java | 10 +-
.../tiered/shuffle/TieredResultPartitionTest.java | 7 +-
.../tiered/storage/HashBufferAccumulatorTest.java | 54 ++-
.../tiered/storage/SortBufferAccumulatorTest.java | 58 ++-
.../storage/TieredStorageSortBufferTest.java | 44 ++-
.../tiered/tier/disk/DiskCacheManagerTest.java | 8 +-
.../tier/disk/DiskTierProducerAgentTest.java | 25 +-
.../tier/memory/MemoryTierProducerAgentTest.java | 16 +-
.../tier/remote/RemoteTierConsumerAgentTest.java | 29 +-
.../tier/remote/TestingAvailabilityNotifier.java | 7 +
.../runtime/io/network/util/TestBufferFactory.java | 18 +-
.../io/network/util/TestSubpartitionConsumer.java | 2 +-
.../runtime/shuffle/NettyShuffleUtilsTest.java | 5 +-
.../shuffle/PartitionDescriptorBuilder.java | 3 +-
.../runtime/state/ChannelPersistenceITCase.java | 10 +-
.../streaming/api/watermark/InternalWatermark.java | 55 +++
.../flink/streaming/api/watermark/Watermark.java | 7 +-
.../streaming/runtime/io/RecordWriterOutput.java | 36 +-
.../io/StreamMultipleInputProcessorFactory.java | 3 +-
.../runtime/io/StreamTwoInputProcessorFactory.java | 6 +-
.../runtime/streamrecord/StreamElement.java | 2 +-
.../streamrecord/StreamElementSerializer.java | 18 +-
.../runtime/tasks/OneInputStreamTask.java | 3 +-
.../watermarkstatus/StatusWatermarkValve.java | 316 +++++++++++------
.../benchmark/SingleInputGateBenchmarkFactory.java | 25 +-
.../checkpointing/AlternatingCheckpointsTest.java | 100 +++---
.../checkpointing/CheckpointedInputGateTest.java | 8 +-
.../io/checkpointing/UnalignedCheckpointsTest.java | 1 +
.../streamrecord/StreamElementSerializerTest.java | 4 +
.../watermarkstatus/StatusWatermarkValveTest.java | 8 +-
.../flink/test/runtime/BatchShuffleITCaseBase.java | 58 ++-
.../flink/test/runtime/BlockingShuffleITCase.java | 12 +-
.../flink/test/runtime/HybridShuffleITCase.java | 28 +-
186 files changed, 3805 insertions(+), 1504 deletions(-)
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/RecoveryMetadata.java
rename
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/{BufferWithChannel.java
=> BufferWithSubpartition.java} (77%)
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/DeduplicatedQueue.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionIndexSet.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/RoundRobinSubpartitionSelector.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SubpartitionSelector.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/UnionResultSubpartitionView.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/common/TieredStorageInputChannelId.java
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/UnionResultSubpartitionViewTest.java
create mode 100644
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/InternalWatermark.java
