This closes #3541: [BEAM-92] Supports DynamicDestinations in AvroIO.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/540fa9b4 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/540fa9b4 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/540fa9b4 Branch: refs/heads/master Commit: 540fa9b4246f2e680b6550537c4dda575e5cf71f Parents: 1f2634d 9f2622f Author: Eugene Kirpichov <[email protected]> Authored: Fri Jul 28 17:28:29 2017 -0700 Committer: Eugene Kirpichov <[email protected]> Committed: Fri Jul 28 17:28:29 2017 -0700 ---------------------------------------------------------------------- .../core/construction/ParDoTranslation.java | 2 +- .../construction/WriteFilesTranslation.java | 81 ++-- .../construction/PTransformMatchersTest.java | 10 +- .../construction/WriteFilesTranslationTest.java | 26 +- .../direct/WriteWithShardingFactory.java | 10 +- .../direct/WriteWithShardingFactoryTest.java | 8 +- .../beam/runners/dataflow/DataflowRunner.java | 8 +- .../runners/dataflow/DataflowRunnerTest.java | 10 +- .../src/main/proto/beam_runner_api.proto | 2 + .../java/org/apache/beam/sdk/io/AvroIO.java | 436 +++++++++++++++---- .../java/org/apache/beam/sdk/io/AvroSink.java | 93 ++-- .../beam/sdk/io/ConstantAvroDestination.java | 130 ++++++ .../beam/sdk/io/DefaultFilenamePolicy.java | 1 - .../beam/sdk/io/DynamicAvroDestinations.java | 46 ++ .../beam/sdk/io/DynamicFileDestinations.java | 59 ++- .../org/apache/beam/sdk/io/FileBasedSink.java | 121 +++-- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 23 +- .../java/org/apache/beam/sdk/io/TextIO.java | 228 ++++++---- .../java/org/apache/beam/sdk/io/TextSink.java | 14 +- .../java/org/apache/beam/sdk/io/WriteFiles.java | 116 ++--- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 156 ++++++- .../apache/beam/sdk/io/FileBasedSinkTest.java | 6 +- .../java/org/apache/beam/sdk/io/SimpleSink.java | 10 +- .../org/apache/beam/sdk/io/TextIOWriteTest.java | 23 +- .../org/apache/beam/sdk/io/WriteFilesTest.java | 74 ++-- .../java/org/apache/beam/sdk/io/xml/XmlIO.java | 4 +- .../org/apache/beam/sdk/io/xml/XmlSink.java | 8 +- 27 files changed, 1214 insertions(+), 491 deletions(-) ----------------------------------------------------------------------
