This is an automated email from the ASF dual-hosted git repository. jooykim pushed a commit to branch upgrade-to-0.2-SNAPSHOT in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit f3f8f3b449c5736a15a615c261c027b8fa2cc47d Merge: 06c1e11 4e40550 Author: Joo Yeon Kim <[email protected]> AuthorDate: Fri Nov 30 17:32:04 2018 +0900 Merge branch 'master' into upgrade-to-0.2-SNAPSHOT DISCLAIMER | 6 + .../org/apache/nemo/common/dag/DAGBuilder.java | 9 -- .../apache/nemo/common/punctuation/Watermark.java | 2 +- .../ExampleTestArgs.java} | 32 ++--- .../frontend/beam/InMemorySideInputReader.java | 108 ++++++++++++++ .../frontend/beam/PipelineTranslationContext.java | 109 ++++++++------ .../compiler/frontend/beam/PipelineTranslator.java | 69 ++++++--- .../compiler/frontend/beam/SideInputElement.java | 36 +++-- .../frontend/beam/coder/SideInputCoder.java | 74 ++++++++++ .../beam/source/BeamBoundedSourceVertex.java | 12 +- .../beam/source/BeamUnboundedSourceVertex.java | 12 +- .../beam/transform/AbstractDoFnTransform.java | 99 +++++++++---- .../BroadcastVariableSideInputReader.java | 63 -------- .../beam/transform/CreateViewTransform.java | 28 ++-- .../frontend/beam/transform/DoFnTransform.java | 26 +--- .../GroupByKeyAndWindowDoFnTransform.java | 41 ++---- .../beam/transform/GroupByKeyTransform.java | 8 -- .../beam/transform/PushBackDoFnTransform.java | 160 +++++++++++++++++++++ .../beam/transform/SideInputTransform.java | 72 ++++++++++ .../frontend/beam/transform/WindowFnTransform.java | 7 +- .../org/apache/nemo/compiler/CompilerTestUtil.java | 10 +- .../frontend/beam/BeamFrontendALSTest.java | 2 +- .../frontend/beam/BeamFrontendMLRTest.java | 4 +- .../beam/transform/CreateViewTransformTest.java | 2 + .../frontend/beam/transform/DoFnTransformTest.java | 121 ++++++++++------ .../GroupByKeyAndWindowDoFnTransformTest.java | 19 ++- .../reshaping/LoopInvariantCodeMotionPassTest.java | 2 +- .../nemo/examples/beam/AlternatingLeastSquare.java | 2 - .../beam/AlternatingLeastSquareInefficient.java | 2 - .../nemo/examples/beam}/FloatArrayCoder.java | 2 +- .../apache/nemo/examples/beam}/IntArrayCoder.java | 2 +- .../nemo/examples/beam/WindowedBroadcast.java | 90 ++++++++++++ .../nemo/examples/beam/WindowedWordCount.java | 2 +- .../beam/AlternatingLeastSquareITCase.java | 23 ++- .../nemo/examples/beam/BeamSimpleSumSQLITCase.java | 15 +- .../apache/nemo/examples/beam/BroadcastITCase.java | 21 ++- .../beam/MultinomialLogisticRegressionITCase.java | 12 +- .../examples/beam/NetworkTraceAnalysisITCase.java | 27 ++-- .../beam/PartitionWordsByLengthITCase.java | 27 ++-- .../nemo/examples/beam/PerKeyMedianITCase.java | 19 ++- .../examples/beam/PerPercentileAverageITCase.java | 19 ++- ...SQLITCase.java => WindowedBroadcastITCase.java} | 48 +++---- .../examples/beam/WindowedWordCountITCase.java | 53 ++++--- .../apache/nemo/examples/beam/WordCountITCase.java | 33 +++-- .../beam_test_executor_resources.json | 0 .../beam_test_one_executor_resources.json | 0 .../beam_test_poisoned_executor_resources.json | 0 .../spark_test_executor_resources.json | 0 examples/resources/{ => inputs}/test_input_als | 0 .../{ => inputs}/test_input_employees.json | 0 examples/resources/{ => inputs}/test_input_median | 0 examples/resources/{ => inputs}/test_input_mlr | 0 .../resources/{ => inputs}/test_input_network0 | 0 .../resources/{ => inputs}/test_input_network1 | 0 .../resources/{ => inputs}/test_input_partition | 0 .../resources/{ => inputs}/test_input_people.json | 0 .../resources/{ => inputs}/test_input_people.txt | 0 examples/resources/{ => inputs}/test_input_tag | 0 .../{ => inputs}/test_input_windowed_wordcount | 0 .../resources/{ => inputs}/test_input_wordcount | 0 .../{ => inputs}/test_input_wordcount_spark | 0 .../resources/{ => outputs}/expected_output_als | 0 .../{ => outputs}/expected_output_broadcast | 0 .../resources/{ => outputs}/expected_output_median | 0 .../{ => outputs}/expected_output_network | 0 .../{ => outputs}/expected_output_partition_0 | 0 .../{ => outputs}/expected_output_partition_1 | 0 .../{ => outputs}/expected_output_partition_2 | 0 .../{ => outputs}/expected_output_partition_3 | 0 .../{ => outputs}/expected_output_partition_4 | 0 .../{ => outputs}/expected_output_partition_5 | 0 .../{ => outputs}/expected_output_partition_6 | 0 .../{ => outputs}/expected_output_partition_7 | 0 .../{ => outputs}/expected_output_partition_8 | 0 .../{ => outputs}/expected_output_partition_9 | 0 .../expected_output_reversed_wordcount_spark | 0 .../{ => outputs}/expected_output_simplesql | 0 .../expected_output_sliding_windowed_wordcount | 0 .../{ => outputs}/expected_output_tag_long | 0 .../{ => outputs}/expected_output_tag_short | 0 .../{ => outputs}/expected_output_tag_very_long | 0 .../expected_output_tag_very_very_long | 0 .../expected_output_windowed_wordcount | 0 .../expected_output_word_and_line_count | 0 .../{ => outputs}/expected_output_wordcount | 0 .../{ => outputs}/expected_output_wordcount_spark | 0 .../org/apache/nemo/examples/spark/MRJava.java | 33 +++-- .../org/apache/nemo/examples/spark/SparkJava.java | 29 ++-- .../org/apache/nemo/examples/spark/SparkScala.java | 49 ++++--- pom.xml | 32 +++-- .../org/apache/nemo/runtime/executor/Executor.java | 2 +- .../executor/data/BroadcastManagerWorker.java | 68 +++------ .../DedicatedKeyPerElementPartitioner.java | 1 + .../datatransfer/MultiInputWatermarkManager.java | 1 - .../OperatorVertexOutputCollector.java | 3 - .../executor/datatransfer/PipeOutputWriter.java | 15 +- .../datatransfer/SingleInputWatermarkManager.java | 3 + .../nemo/runtime/executor/task/DataFetcher.java | 8 ++ .../task/MultiThreadParentTaskDataFetcher.java | 6 +- .../executor/task/ParentTaskDataFetcher.java | 3 +- .../nemo/runtime/executor/task/TaskExecutor.java | 143 +++++++++--------- .../executor/task/ParentTaskDataFetcherTest.java | 7 +- .../runtime/executor/task/TaskExecutorTest.java | 25 +--- 103 files changed, 1201 insertions(+), 757 deletions(-)
