Merge branch '0.14.0'
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/db2f121e Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/db2f121e Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/db2f121e Branch: refs/heads/master Commit: db2f121e22ad2ae28f914dd1510d251e713d2e7c Parents: 6e6e53b 052a057 Author: Xinyu Liu <[email protected]> Authored: Tue Oct 3 15:10:13 2017 -0700 Committer: Xinyu Liu <[email protected]> Committed: Tue Oct 3 15:10:13 2017 -0700 ---------------------------------------------------------------------- .../versioned/jobs/configuration-table.html | 32 +++ .../samza/checkpoint/CheckpointManager.java | 4 + .../operators/functions/WatermarkFunction.java | 58 ++++++ .../org/apache/samza/system/ControlMessage.java | 46 +++++ .../apache/samza/system/EndOfStreamMessage.java | 38 ++++ .../samza/system/IncomingMessageEnvelope.java | 9 +- .../org/apache/samza/system/MessageType.java | 45 ++++ .../org/apache/samza/system/StreamSpec.java | 68 +++++- .../org/apache/samza/system/SystemAdmin.java | 42 +--- .../apache/samza/system/WatermarkMessage.java | 46 +++++ ...inglePartitionWithoutOffsetsSystemAdmin.java | 16 -- .../apache/samza/config/ApplicationConfig.java | 15 ++ .../apache/samza/config/JavaStorageConfig.java | 5 + .../apache/samza/container/TaskContextImpl.java | 131 ++++++++++++ .../apache/samza/execution/ExecutionPlan.java | 7 + .../samza/execution/ExecutionPlanner.java | 20 +- .../org/apache/samza/execution/JobGraph.java | 2 +- .../org/apache/samza/execution/JobNode.java | 14 +- .../org/apache/samza/execution/StreamEdge.java | 50 ++++- .../apache/samza/execution/StreamManager.java | 76 +++++++ .../apache/samza/message/ControlMessage.java | 52 ----- .../samza/message/EndOfStreamMessage.java | 36 ---- .../org/apache/samza/message/MessageType.java | 46 ----- .../apache/samza/message/WatermarkMessage.java | 43 ---- .../operators/impl/ControlMessageSender.java | 56 +++++ .../samza/operators/impl/EndOfStreamStates.java | 98 +++++++++ .../samza/operators/impl/OperatorImpl.java | 191 ++++++++++++++++- .../samza/operators/impl/OperatorImplGraph.java | 96 ++++++++- .../operators/impl/PartitionByOperatorImpl.java | 26 +++ .../samza/operators/impl/WatermarkStates.java | 119 +++++++++++ .../samza/operators/spec/InputOperatorSpec.java | 6 + .../samza/operators/spec/JoinOperatorSpec.java | 6 + .../samza/operators/spec/OperatorSpec.java | 3 + .../operators/spec/OutputOperatorSpec.java | 7 + .../operators/spec/PartitionByOperatorSpec.java | 5 + .../samza/operators/spec/SinkOperatorSpec.java | 6 + .../operators/spec/StreamOperatorSpec.java | 6 + .../operators/spec/WindowOperatorSpec.java | 8 + .../apache/samza/processor/StreamProcessor.java | 18 +- .../runtime/AbstractApplicationRunner.java | 42 +++- .../samza/runtime/LocalApplicationRunner.java | 5 + .../samza/runtime/LocalContainerRunner.java | 6 +- .../samza/runtime/RemoteApplicationRunner.java | 27 ++- .../serializers/IntermediateMessageSerde.java | 6 +- .../PassthroughCoordinationUtilsFactory.java | 30 +++ .../org/apache/samza/task/AsyncRunLoop.java | 12 +- .../apache/samza/task/StreamOperatorTask.java | 27 ++- .../samza/zk/ScheduleAfterDebounceTime.java | 172 +++++++++++----- .../org/apache/samza/zk/ZkJobCoordinator.java | 27 ++- .../apache/samza/checkpoint/OffsetManager.scala | 7 +- .../org/apache/samza/config/StreamConfig.scala | 11 +- .../apache/samza/container/SamzaContainer.scala | 13 +- .../apache/samza/container/TaskInstance.scala | 88 ++++---- .../samza/coordinator/JobModelManager.scala | 10 +- .../scala/org/apache/samza/job/JobRunner.scala | 9 +- .../samza/job/local/ThreadJobFactory.scala | 6 +- .../apache/samza/serializers/SerdeManager.scala | 5 +- .../samza/storage/TaskStorageManager.scala | 12 +- .../samza/system/StreamMetadataCache.scala | 11 + .../MockCoordinatorStreamSystemFactory.java | 13 +- .../samza/execution/TestExecutionPlanner.java | 21 +- .../org/apache/samza/execution/TestJobNode.java | 7 +- .../apache/samza/execution/TestStreamEdge.java | 84 ++++++++ .../samza/execution/TestStreamManager.java | 147 +++++++++++++ .../samza/operators/TestJoinOperator.java | 3 +- .../samza/operators/TestWindowOperator.java | 6 +- .../impl/TestControlMessageSender.java | 73 +++++++ .../operators/impl/TestEndOfStreamStates.java | 78 +++++++ .../samza/operators/impl/TestOperatorImpl.java | 46 ++++- .../operators/impl/TestOperatorImplGraph.java | 187 +++++++++++++++-- .../operators/impl/TestWatermarkStates.java | 102 +++++++++ .../samza/processor/TestStreamProcessor.java | 4 +- .../runtime/TestLocalApplicationRunner.java | 162 ++++----------- .../TestIntermediateMessageSerde.java | 12 +- .../org/apache/samza/task/TestAsyncRunLoop.java | 2 +- .../samza/task/TestStreamOperatorTask.java | 30 +++ .../samza/zk/TestScheduleAfterDebounceTime.java | 74 ++++++- .../samza/checkpoint/TestOffsetManager.scala | 12 -- .../samza/container/TestTaskInstance.scala | 22 +- .../samza/coordinator/TestJobCoordinator.scala | 12 -- .../samza/serializers/TestSerdeManager.scala | 10 +- .../samza/storage/TestTaskStorageManager.scala | 6 +- .../elasticsearch/ElasticsearchSystemAdmin.java | 15 -- .../samza/system/hdfs/HdfsSystemAdmin.java | 15 -- .../samza/system/kafka/KafkaStreamSpec.java | 2 +- .../kafka/KafkaCheckpointManager.scala | 19 +- .../kafka/KafkaCheckpointManagerFactory.scala | 14 +- .../samza/system/kafka/KafkaSystemAdmin.scala | 96 +++++---- .../scala/org/apache/samza/util/KafkaUtil.scala | 12 +- .../samza/system/kafka/TestKafkaStreamSpec.java | 2 +- .../system/kafka/TestKafkaSystemAdminJava.java | 166 +++++++-------- .../system/kafka/TestKafkaSystemAdmin.scala | 5 +- .../samza/system/mock/MockSystemAdmin.java | 15 -- .../processor/TestStreamProcessorUtil.java | 28 +++ .../EndOfStreamIntegrationTest.java | 112 ++++++++++ .../samza/test/controlmessages/TestData.java | 85 ++++++++ .../WatermarkIntegrationTest.java | 206 +++++++++++++++++++ .../samza/test/util/ArraySystemConsumer.java | 82 ++++++++ .../samza/test/util/ArraySystemFactory.java | 50 +++++ .../samza/test/util/Base64Serializer.java | 66 ++++++ .../samza/test/util/SimpleSystemAdmin.java | 75 +++++++ .../samza/test/util/TestStreamConsumer.java | 52 +++++ .../apache/samza/job/yarn/MockSystemAdmin.scala | 12 -- 103 files changed, 3451 insertions(+), 869 deletions(-) ----------------------------------------------------------------------
