This closes #1926: Allow unbounded windowed PCollections for FileBasedSinks
Add windowing support to FileBasedSink Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bc907c58 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bc907c58 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bc907c58 Branch: refs/heads/master Commit: bc907c58b1da97e53dd0f4b6bda0834b41bb6e66 Parents: 8e5cfde 6addc95 Author: Kenneth Knowles <[email protected]> Authored: Wed Apr 5 10:23:20 2017 -0700 Committer: Kenneth Knowles <[email protected]> Committed: Wed Apr 5 10:23:20 2017 -0700 ---------------------------------------------------------------------- examples/java/pom.xml | 1 - .../apache/beam/examples/WindowedWordCount.java | 34 +- .../examples/common/WriteOneFilePerWindow.java | 91 ++++ .../examples/common/WriteWindowedFilesDoFn.java | 77 ---- .../beam/examples/WindowedWordCountIT.java | 41 +- .../core/construction/PTransformMatchers.java | 3 +- .../direct/WriteWithShardingFactory.java | 6 +- .../streaming/io/UnboundedFlinkSink.java | 20 +- .../beam/runners/flink/WriteSinkITCase.java | 23 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 157 +++++-- .../org/apache/beam/sdk/io/FileBasedSink.java | 429 +++++++++++++++---- .../main/java/org/apache/beam/sdk/io/Sink.java | 55 ++- .../java/org/apache/beam/sdk/io/TextIO.java | 98 ++++- .../main/java/org/apache/beam/sdk/io/Write.java | 377 +++++++++------- .../java/org/apache/beam/sdk/io/XmlSink.java | 6 +- .../beam/sdk/testing/TestPipelineOptions.java | 5 + .../beam/sdk/util/FileIOChannelFactory.java | 23 +- .../beam/sdk/util/GcsIOChannelFactory.java | 3 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 21 +- .../apache/beam/sdk/util/IOChannelFactory.java | 3 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 146 ++++++- .../apache/beam/sdk/io/FileBasedSinkTest.java | 94 ++-- .../java/org/apache/beam/sdk/io/WriteTest.java | 49 ++- .../org/apache/beam/sdk/io/XmlSinkTest.java | 8 +- .../beam/sdk/testing/TestPipelineTest.java | 17 - .../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 24 +- .../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 2 +- 27 files changed, 1295 insertions(+), 518 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/bc907c58/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java ----------------------------------------------------------------------
