This is an automated email from the ASF dual-hosted git repository. jooykim pushed a commit to branch maven-artifact in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit 49266b5b70aa7bc317807d20e2dd7a1cfb170ffd Merge: b2787ca eea3f57 Author: Joo Yeon Kim <[email protected]> AuthorDate: Wed Oct 31 11:05:38 2018 +0900 Merge branch 'master' into maven-artifact .gitignore | 1 + .travis.yml | 4 +- checkstyle.xml | 2 +- .../java/org/apache/nemo/client/JobLauncher.java | 15 +- ...Collector.java => BoundedIteratorReadable.java} | 43 ++-- .../org/apache/nemo/common/ir/OutputCollector.java | 7 + .../java/org/apache/nemo/common/ir/Readable.java | 38 +++- .../edge/executionproperty/DataStoreProperty.java | 1 + .../nemo/common/ir/vertex/CachedSourceVertex.java | 40 +++- .../common/ir/vertex/InMemorySourceVertex.java | 25 ++- .../apache/nemo/common/ir/vertex/SourceVertex.java | 2 + .../vertex/transform/AggregateMetricTransform.java | 2 +- .../vertex/transform/MetricCollectTransform.java | 2 +- .../vertex/transform/NoWatermarkEmitTransform.java | 24 +-- .../common/ir/vertex/transform/RelayTransform.java | 6 + .../nemo/common/ir/vertex/transform/Transform.java | 10 + .../apache/nemo/common/punctuation/Finishmark.java | 23 +-- .../Watermark.java} | 48 +++-- .../org/apache/nemo/common/test/ArgBuilder.java | 9 + .../apache/nemo/common/test/EmptyComponents.java | 37 +++- .../compiler/frontend/beam/PipelineTranslator.java | 21 +- .../beam/source/BeamBoundedSourceVertex.java | 73 +++++-- .../beam/source/BeamUnboundedSourceVertex.java | 128 ++++++------ .../beam/transform/AbstractDoFnTransform.java | 6 +- .../beam/transform/CreateViewTransform.java | 4 +- .../frontend/beam/transform/DoFnTransform.java | 8 + .../frontend/beam/transform/FlattenTransform.java | 6 + .../GroupByKeyAndWindowDoFnTransform.java | 10 +- .../beam/transform/GroupByKeyTransform.java | 4 +- .../frontend/beam/transform/WindowFnTransform.java | 6 + compiler/frontend/spark/pom.xml | 19 ++ .../source/SparkDatasetBoundedSourceVertex.java | 29 ++- .../source/SparkTextFileBoundedSourceVertex.java | 34 +++- .../frontend/spark/transform/CollectTransform.java | 5 +- .../frontend/spark/transform/FlatMapTransform.java | 6 + .../spark/transform/GroupByKeyTransform.java | 3 +- .../spark/transform/HDFSTextFileTransform.java | 3 +- .../spark/transform/LocalTextFileTransform.java | 4 +- .../spark/transform/MapToPairTransform.java | 6 + .../frontend/spark/transform/MapTransform.java | 6 + .../spark/transform/ReduceByKeyTransform.java | 4 +- .../frontend/spark/transform/ReduceTransform.java | 6 + .../annotating/PipeTransferForAllEdgesPass.java | 41 ++-- .../frontend/beam/transform/DoFnTransformTest.java | 6 + .../main/java/org/apache/nemo/conf/JobConf.java | 8 + .../nemo/examples/beam/WriteOneFilePerWindow.java | 2 +- .../examples/beam/WindowedWordCountITCase.java | 48 ++++- .../policy/StreamingPolicyParallelismFive.java | 57 ++++++ pom.xml | 9 + .../runtime/common/message/MessageEnvironment.java | 1 + .../common/message/ncs/NcsMessageEnvironment.java | 7 + .../nemo/runtime/common/plan/RuntimeEdge.java | 15 ++ runtime/common/src/main/proto/ControlMessage.proto | 33 +++- .../org/apache/nemo/runtime/executor/Executor.java | 10 +- .../executor/bytetransfer/ByteTransfer.java | 12 +- .../executor/bytetransfer/ByteTransferContext.java | 10 +- .../ByteTransportChannelInitializer.java | 11 +- .../bytetransfer/ClosableBlockingQueue.java | 1 + .../executor/bytetransfer/ContextManager.java | 40 +++- .../executor/bytetransfer/ControlFrameEncoder.java | 1 + .../runtime/executor/data/BlockManagerWorker.java | 10 +- .../executor/data/BroadcastManagerWorker.java | 3 +- .../nemo/runtime/executor/data/PipeContainer.java | 150 ++++++++++++++ .../runtime/executor/data/PipeManagerWorker.java | 199 +++++++++++++++++++ .../runtime/executor/data/SerializerManager.java | 2 +- .../data/partitioner/DataSkewHashPartitioner.java | 3 - .../executor/data/stores/GlusterFileStore.java | 3 - .../executor/data/stores/LocalFileStore.java | 3 - .../{InputReader.java => BlockInputReader.java} | 125 ++++-------- .../{OutputWriter.java => BlockOutputWriter.java} | 90 +++------ .../datatransfer/DataFetcherOutputCollector.java | 6 + .../datatransfer/DynOptDataOutputCollector.java | 6 + .../runtime/executor/datatransfer/InputReader.java | 155 +-------------- ...Factory.java => IntermediateDataIOFactory.java} | 35 +++- .../OperatorVertexOutputCollector.java | 20 ++ .../executor/datatransfer/OutputWriter.java | 158 ++++----------- .../executor/datatransfer/PipeInputReader.java | 82 ++++++++ .../executor/datatransfer/PipeOutputWriter.java | 136 +++++++++++++ .../nemo/runtime/executor/task/DataFetcher.java | 5 +- .../executor/task/ParentTaskDataFetcher.java | 12 +- .../executor/task/SourceVertexDataFetcher.java | 81 ++++++-- .../nemo/runtime/executor/task/TaskExecutor.java | 203 ++++++++++++++----- .../executor/datatransfer/DataTransferTest.java | 77 +++++--- .../executor/task/ParentTaskDataFetcherTest.java | 13 +- .../runtime/executor/task/TaskExecutorTest.java | 219 +++++++++++++++++++-- .../apache/nemo/runtime/master/MetricStore.java | 2 + .../nemo/runtime/master/PipeManagerMaster.java | 157 +++++++++++++++ .../apache/nemo/runtime/master/RuntimeMaster.java | 1 + .../master/resource/ExecutorRepresenter.java | 6 +- .../master/resource/ResourceSpecification.java | 2 +- .../nemo/runtime/master/scheduler/Scheduler.java | 2 - .../master/scheduler/StreamingScheduler.java | 29 ++- .../runtime/master/scheduler/TaskDispatcher.java | 10 +- .../master/scheduler/StreamingSchedulerTest.java | 8 +- .../runtime/master/scheduler/TaskRetryTest.java | 2 +- 95 files changed, 2214 insertions(+), 843 deletions(-)
