This is an automated email from the ASF dual-hosted git repository. echauchot pushed a commit to branch spark-runner_structured-streaming in repository https://gitbox.apache.org/repos/asf/beam.git
commit 6ba3d1cb0d09ea0aa7f6a5f308fd59ccceae9d47 Author: Etienne Chauchot <[email protected]> AuthorDate: Wed May 15 14:24:11 2019 +0200 Add comment about checkpoint mark --- .../translation/streaming/DatasetSourceStreaming.java | 1 + 1 file changed, 1 insertion(+) diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java index 234eecf..1cf52ba 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java @@ -134,6 +134,7 @@ class DatasetSourceStreaming implements DataSourceV2, MicroBatchReadSupport { // offsets are ignored see javadoc for (DatasetPartitionReader partitionReader : partitionReaders) { try { + // TODO: is checkpointMark stored in reliable storage ? partitionReader.reader.getCheckpointMark().finalizeCheckpoint(); } catch (IOException e) { throw new RuntimeException(
