This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from e1ac6c9 [FLINK-17132][metrics] Bump Prometheus to 0.8.1
add 9bed566 [hotfix][task] Avoid unnecessary actionExecutor for
notifyCheckpointComplete.
add 7327ffd [hotfix][network] Remove unused InterruptedExceptions.
add c172e7a [hotfix][network] Move uncritical code out of lock in
RemoteInputChannel#onBuffer.
add 4319b01 [hotfix][task] Rename CheckpointedInputGate#cleanup to close
and implement Closeable.
add 9649d38 [hotfix][task] Narrowing checkpointing related exceptions to
IOException.
add ef6e2c4 [hotfix][task] Generalizing MailboxExecutor#execute to
ThrowingRunnable.
add b63797c [FLINK-16587][checkpointing] Make
ChannelStateWriterImpl#getWriteResult idempotent.
add b9b761d [FLINK-16587][coordinator] Ensuring channel deployment
descriptors are lists.
add 1007232 [FLINK-16587][checkpointing] Introducing IndexedInputGate for
abstracting single channel InputGates with a specific index.
add bde57b2 [FLINK-16587][checkpointing] Switch to positional input
channel and subpartitions.
add aed6c1a [FLINK-16587][checkpointing] Adding unaligned checkpoint
config flag.
add dbdaa51 [FLINK-16587][checkpointing] Introduce BufferReceivedListener
for notifying of received buffers and barriers from network channel.
add 4538321 [FLINK-16587][checkpointing] Implement
CheckpointBarrierUnaligner to trigger checkpoint when receiving the first
barrier.
add 825cb25 [FLINK-16587][task] Moving event creation from OperatorChain
to SubtaskCheckpointCoordinatorImpl.
add 9b0477f [FLINK-16587][checkpointing] Implement checkpoint barrier
overtake in output partitions.
add d041efe [FLINK-16587][checkpointing] Provide the method for getting
unconsumed buffer from RecordDeserializer.
add f661d18 [FLINK-16587][checkpointing] Provide the method of getting
in-flight buffers from input gate.
add 5cebfb7 [FLINK-16587][checkpointing] Spill the in-flight input and
output buffers during checkpointing.
add 77dce77 [FLINK-16587][checkpointing] Fix abortion of
ChannelStateWriter and add test/IT cases for unaligned checkpoint.
No new revisions were added by this update.
Summary of changes:
.../execution_checkpointing_configuration.html | 6 +
.../state/api/runtime/SavepointEnvironment.java | 6 +-
.../channel/ChannelStateCheckpointWriter.java | 1 -
.../checkpoint/channel/ChannelStateWriter.java | 9 +-
.../checkpoint/channel/ChannelStateWriterImpl.java | 7 +-
.../flink/runtime/concurrent/FutureUtils.java | 16 +
.../flink/runtime/execution/Environment.java | 6 +-
.../io/network/NettyShuffleEnvironment.java | 21 +-
.../io/network/NetworkSequenceViewReader.java | 2 +-
.../api/serialization/RecordDeserializer.java | 9 +
...SpillingAdaptiveSpanningRecordDeserializer.java | 57 ++
.../io/network/api/writer/RecordWriter.java | 6 +-
.../network/api/writer/ResultPartitionWriter.java | 23 +-
.../io/network/buffer/BufferReceivedListener.java | 47 ++
.../CreditBasedSequenceNumberingViewReader.java | 2 +-
.../partition/BoundedBlockingSubpartition.java | 8 +-
.../BoundedBlockingSubpartitionReader.java | 2 +-
.../partition/BufferAvailabilityListener.java | 13 +
.../network/partition/PipelinedSubpartition.java | 66 ++-
.../partition/PipelinedSubpartitionView.java | 8 +-
.../io/network/partition/ResultPartition.java | 14 +-
.../io/network/partition/ResultSubpartition.java | 38 +-
.../network/partition/ResultSubpartitionView.java | 2 +-
.../network/partition/consumer/BufferOrEvent.java | 2 +-
.../partition/consumer/IndexedInputGate.java | 14 +-
.../network/partition/consumer/InputChannel.java | 45 +-
.../io/network/partition/consumer/InputGate.java | 8 +
.../partition/consumer/LocalInputChannel.java | 26 +-
.../partition/consumer/RemoteInputChannel.java | 59 +-
.../partition/consumer/SingleInputGate.java | 45 +-
.../partition/consumer/SingleInputGateFactory.java | 2 +-
.../network/partition/consumer/UnionInputGate.java | 82 +--
.../partition/consumer/UnknownInputChannel.java | 4 +-
.../runtime/jobgraph/tasks/AbstractInvokable.java | 20 +-
.../apache/flink/runtime/operators/BatchTask.java | 6 +-
.../flink/runtime/shuffle/ShuffleEnvironment.java | 7 +-
.../runtime/shuffle/ShuffleServiceFactory.java | 4 +-
...bleNotifyingResultPartitionWriterDecorator.java | 13 +-
.../runtime/taskmanager/InputGateWithMetrics.java | 24 +-
.../runtime/taskmanager/RuntimeEnvironment.java | 10 +-
.../org/apache/flink/runtime/taskmanager/Task.java | 16 +-
.../channel/ChannelStateWriterImplTest.java | 13 +
.../checkpoint/channel/MockChannelStateWriter.java | 119 ++++
.../channel/RecordingChannelStateWriter.java | 88 +++
.../io/network/NettyShuffleEnvironmentTest.java | 36 +-
.../SpanningRecordSerializationTest.java | 124 ++++
.../AbstractCollectingResultPartitionWriter.java | 5 +-
.../io/network/api/writer/RecordWriterTest.java | 4 +-
...editBasedPartitionRequestClientHandlerTest.java | 48 +-
.../NettyMessageClientDecoderDelegateTest.java | 9 +-
.../NettyMessageClientSideSerializationTest.java | 1 +
.../netty/NettyPartitionRequestClientTest.java | 5 +-
.../AwaitableBufferAvailablityListener.java | 21 +
.../network/partition/InputChannelTestUtils.java | 8 +-
.../network/partition/InputGateFairnessTest.java | 12 +-
.../partition/MockResultPartitionWriter.java | 12 +-
.../PipelinedSubpartitionWithReadViewTest.java | 79 ++-
.../TaskExecutorPartitionTrackerImplTest.java | 4 +-
.../consumer/InputBuffersMetricsTest.java | 16 +-
.../partition/consumer/InputChannelBuilder.java | 20 +-
.../partition/consumer/InputGateTestBase.java | 10 +
.../IteratorWrappingTestSingleInputGate.java | 10 +-
.../partition/consumer/LocalInputChannelTest.java | 10 +-
.../partition/consumer/RemoteInputChannelTest.java | 60 +-
.../partition/consumer/SingleInputGateTest.java | 147 ++++-
.../partition/consumer/TestInputChannel.java | 3 +-
.../partition/consumer/TestSingleInputGate.java | 10 +-
.../partition/consumer/UnionInputGateTest.java | 8 +-
.../operators/testutils/DummyEnvironment.java | 6 +-
.../operators/testutils/MockEnvironment.java | 19 +-
.../runtime/shuffle/ShuffleServiceLoaderTest.java | 6 +-
.../runtime/taskmanager/TaskAsyncCallTest.java | 2 +-
.../api/environment/CheckpointConfig.java | 47 ++
.../environment/ExecutionCheckpointingOptions.java | 19 +
.../flink/streaming/api/graph/StreamConfig.java | 9 +
.../api/graph/StreamingJobGraphGenerator.java | 1 +
.../streaming/api/operators/MailboxExecutor.java | 5 +-
.../runtime/io/CheckpointBarrierHandler.java | 29 +-
.../runtime/io/CheckpointBarrierUnaligner.java | 431 ++++++++++++++
.../runtime/io/CheckpointedInputGate.java | 26 +-
.../flink/streaming/runtime/io/InputGateUtil.java | 9 +-
.../streaming/runtime/io/InputProcessorUtil.java | 70 ++-
.../streaming/runtime/io/RecordWriterOutput.java | 4 +-
.../streaming/runtime/io/StreamInputProcessor.java | 5 +
.../runtime/io/StreamMultipleInputProcessor.java | 18 +
.../runtime/io/StreamOneInputProcessor.java | 8 +
.../streaming/runtime/io/StreamTaskInput.java | 8 +
.../runtime/io/StreamTaskNetworkInput.java | 28 +-
.../runtime/io/StreamTaskSourceInput.java | 8 +
.../runtime/io/StreamTwoInputProcessor.java | 10 +
.../runtime/tasks/AbstractTwoInputStreamTask.java | 12 +-
.../runtime/tasks/AsyncCheckpointRunnable.java | 6 +
.../runtime/tasks/MultipleInputStreamTask.java | 12 +-
.../runtime/tasks/OneInputStreamTask.java | 6 +-
.../streaming/runtime/tasks/OperatorChain.java | 16 +-
.../flink/streaming/runtime/tasks/StreamTask.java | 59 +-
.../tasks/SubtaskCheckpointCoordinator.java | 3 +-
.../tasks/SubtaskCheckpointCoordinatorImpl.java | 64 +-
.../runtime/tasks/TwoInputStreamTask.java | 12 +-
.../streaming/runtime/tasks/mailbox/Mail.java | 10 +-
.../runtime/tasks/mailbox/MailboxExecutorImpl.java | 8 +-
.../runtime/tasks/mailbox/MailboxProcessor.java | 9 +
.../consumer/StreamTestSingleInputGate.java | 12 +-
.../CheckpointConfigFromConfigurationTest.java | 8 +-
...CheckpointBarrierAlignerAlignmentLimitTest.java | 10 +-
.../CheckpointBarrierAlignerMassiveRandomTest.java | 11 +
.../io/CheckpointBarrierAlignerTestBase.java | 6 +-
.../runtime/io/CheckpointBarrierUnalignerTest.java | 642 +++++++++++++++++++++
.../runtime/io/CheckpointSequenceValidator.java | 2 +-
.../flink/streaming/runtime/io/MockInputGate.java | 11 +
.../runtime/io/StreamTaskNetworkInputTest.java | 2 +-
.../benchmark/SingleInputGateBenchmarkFactory.java | 6 +-
.../StreamNetworkBenchmarkEnvironment.java | 7 +-
.../runtime/tasks/LocalStateForwardingTest.java | 2 +
.../MultipleInputStreamTaskTestHarnessBuilder.java | 7 +-
.../tasks/OneInputStreamTaskTestHarness.java | 6 +-
.../runtime/tasks/StreamMockEnvironment.java | 16 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 13 +
.../tasks/TwoInputStreamTaskTestHarness.java | 14 +-
.../checkpointing/UnalignedCheckpointITCase.java | 175 ++++++
120 files changed, 3121 insertions(+), 452 deletions(-)
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferReceivedListener.java
copy
flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/common/Acknowledger.java
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/IndexedInputGate.java
(74%)
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/MockChannelStateWriter.java
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecordingChannelStateWriter.java
create mode 100644
flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnaligner.java
create mode 100644
flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnalignerTest.java
create mode 100644
flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java
