This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 1d8cbe9065e [FLINK-34483][docs] Improve the documentation of
'state.checkpoints.dir' and 'state.checkpoint-storage' (#24401)
add 012b8936e11 [FLINK-33668][runtime] Redefine the redistribution logic
of the NetworkBufferPool
add 2e257896661 [FLINK-33668][runtime] Decouple the network memory and job
topology on input gate side
add 88138d08e73 [FLINK-33668][runtime] Decouple the network memory and job
topology on result partition side
add ac2ec349640 [FLINK-33668][runtime] Make the SortBuffer be able to
return unused memory segments
add c1ea971129a [FLINK-33668][runtime] Close the producer client in case
of the result partition is not finished properly
add 4171b980717 [FLINK-33668][runtime] Avoid the potential deadlock during
reserving segments
No new revisions were added by this update.
Summary of changes:
.../generated/all_taskmanager_network_section.html | 6 +
.../netty_shuffle_environment_configuration.html | 6 +
.../NettyShuffleEnvironmentOptions.java | 13 +
.../runtime/io/network/buffer/BufferPool.java | 8 +-
.../io/network/buffer/BufferPoolFactory.java | 18 +-
.../runtime/io/network/buffer/LocalBufferPool.java | 105 +++--
.../io/network/buffer/NetworkBufferPool.java | 205 ++++++---
.../metrics/CreditBasedInputBuffersUsageGauge.java | 37 +-
.../metrics/ExclusiveBuffersUsageGauge.java | 2 +-
.../network/metrics/FloatingBuffersUsageGauge.java | 40 +-
.../network/metrics/NettyShuffleMetricFactory.java | 10 +-
.../network/netty/NettyPartitionRequestClient.java | 2 +-
.../partition/BufferWritingResultPartition.java | 2 +-
.../network/partition/ResultPartitionFactory.java | 21 +-
.../runtime/io/network/partition/SortBuffer.java | 27 +-
.../partition/SortMergeResultPartition.java | 2 +-
.../network/partition/consumer/BufferManager.java | 162 ++++++-
.../partition/consumer/GateBuffersSpec.java | 34 +-
.../partition/consumer/InputGateSpecUtils.java | 65 ++-
.../consumer/LocalRecoveredInputChannel.java | 15 +
.../partition/consumer/RecoveredInputChannel.java | 10 +-
.../partition/consumer/RemoteInputChannel.java | 30 +-
.../consumer/RemoteRecoveredInputChannel.java | 1 +
.../partition/consumer/SingleInputGate.java | 13 +
.../partition/consumer/SingleInputGateFactory.java | 49 +-
.../tiered/common/TieredStorageConfiguration.java | 94 ++++
.../tiered/shuffle/TieredResultPartition.java | 4 +
.../shuffle/TieredResultPartitionFactory.java | 35 +-
.../tiered/storage/SortBufferAccumulator.java | 88 +++-
.../tiered/storage/TieredStorageMemoryManager.java | 3 +
.../storage/TieredStorageMemoryManagerImpl.java | 63 +--
.../flink/runtime/shuffle/NettyShuffleUtils.java | 46 +-
.../NettyShuffleEnvironmentConfiguration.java | 4 +
.../InputChannelRecoveredStateHandlerTest.java | 6 +-
.../SequentialChannelStateReaderImplTest.java | 2 +-
.../io/network/NettyShuffleEnvironmentTest.java | 21 +-
.../api/writer/RecordWriterDelegateTest.java | 2 +-
.../io/network/api/writer/RecordWriterTest.java | 2 +-
.../io/network/buffer/BufferPoolFactoryTest.java | 186 ++++++--
.../io/network/buffer/LocalBufferPoolTest.java | 37 +-
.../io/network/buffer/NetworkBufferPoolTest.java | 38 +-
.../runtime/io/network/buffer/NoOpBufferPool.java | 131 ------
.../io/network/buffer/TestingBufferPool.java | 505 +++++++++++++++++++++
.../io/network/buffer/UnpooledBufferPool.java | 7 +-
...editBasedPartitionRequestClientHandlerTest.java | 34 +-
.../NettyMessageClientDecoderDelegateTest.java | 8 +-
.../NettyMessageClientSideSerializationTest.java | 8 +-
.../netty/NettyPartitionRequestClientTest.java | 8 +-
.../io/network/partition/DataBufferTest.java | 6 +-
.../network/partition/InputGateFairnessTest.java | 15 +-
.../io/network/partition/PartitionTestUtils.java | 16 +
.../PipelinedSubpartitionWithReadViewTest.java | 3 +
.../network/partition/ResultPartitionBuilder.java | 2 +-
.../io/network/partition/ResultPartitionTest.java | 10 +-
.../partition/SortMergeResultPartitionTest.java | 31 +-
.../partition/consumer/GateBuffersSpecTest.java | 118 +++--
.../consumer/InputBuffersMetricsTest.java | 36 +-
.../partition/consumer/LocalInputChannelTest.java | 4 +-
.../partition/consumer/RemoteInputChannelTest.java | 103 +++--
.../partition/consumer/SingleInputGateBuilder.java | 20 +-
.../partition/consumer/SingleInputGateTest.java | 48 +-
.../partition/hybrid/HsMemoryDataManagerTest.java | 8 +-
.../partition/hybrid/HsResultPartitionTest.java | 26 +-
.../partition/hybrid/HsSubpartitionViewTest.java | 2 +-
.../tiered/TestingTieredStorageMemoryManager.java | 15 +
.../tiered/shuffle/TieredResultPartitionTest.java | 28 +-
.../tiered/storage/HashBufferAccumulatorTest.java | 3 +-
.../tiered/storage/SortBufferAccumulatorTest.java | 13 +-
.../TieredStorageMemoryManagerImplTest.java | 13 +-
.../storage/TieredStorageSortBufferTest.java | 37 +-
.../runtime/shuffle/NettyShuffleUtilsTest.java | 12 +-
.../runtime/state/ChannelPersistenceITCase.java | 3 +-
.../AlignedCheckpointsMassiveRandomTest.java | 4 +-
.../checkpointing/CheckpointedInputGateTest.java | 6 +-
.../util/TestCheckpointedInputGateBuilder.java | 3 +-
.../flink/test/runtime/BatchShuffleITCaseBase.java | 22 +-
.../HybridShuffleMemoryDecouplingITCase.java | 114 +++++
77 files changed, 2150 insertions(+), 786 deletions(-)
delete mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NoOpBufferPool.java
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/TestingBufferPool.java
create mode 100644
flink-tests/src/test/java/org/apache/flink/test/runtime/HybridShuffleMemoryDecouplingITCase.java
