Merge branch 'master' into gearpump-runner # Conflicts: # runners/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0c362282 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0c362282 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0c362282 Branch: refs/heads/gearpump-runner Commit: 0c362282d3f2ca224c937c813f467723d7b1e908 Parents: 323ec11 99505e1 Author: manuzhang <owenzhang1...@gmail.com> Authored: Tue Nov 8 10:30:33 2016 +0800 Committer: manuzhang <owenzhang1...@gmail.com> Committed: Tue Nov 8 10:30:33 2016 +0800 ---------------------------------------------------------------------- .travis.yml | 22 +- examples/java/pom.xml | 92 +-- .../beam/examples/DebuggingWordCount.java | 2 +- .../apache/beam/examples/MinimalWordCount.java | 2 +- .../org/apache/beam/examples/WordCount.java | 2 +- .../beam/examples/common/ExampleUtils.java | 3 +- .../apache/beam/examples/complete/TfIdf.java | 8 +- .../examples/complete/TopWikipediaSessions.java | 2 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../beam/examples/cookbook/DeDupExample.java | 10 +- .../beam/examples/cookbook/FilterExamples.java | 2 +- .../beam/examples/cookbook/JoinExamples.java | 2 +- .../examples/cookbook/MaxPerKeyExamples.java | 2 +- .../org/apache/beam/examples/cookbook/README.md | 2 +- .../beam/examples/WindowedWordCountIT.java | 15 +- .../org/apache/beam/examples/WordCountIT.java | 18 +- .../org/apache/beam/examples/WordCountTest.java | 2 +- .../examples/complete/AutoCompleteTest.java | 6 +- .../beam/examples/complete/TfIdfTest.java | 6 +- .../complete/TopWikipediaSessionsTest.java | 2 +- .../examples/cookbook/BigQueryTornadoesIT.java | 15 +- .../examples/cookbook/DeDupExampleTest.java | 82 -- .../examples/cookbook/DistinctExampleTest.java | 82 ++ .../examples/cookbook/JoinExamplesTest.java | 2 +- .../examples/cookbook/TriggerExampleTest.java | 2 +- examples/java8/pom.xml | 32 +- .../beam/examples/MinimalWordCountJava8.java | 2 +- .../beam/examples/complete/game/GameStats.java | 55 +- .../examples/complete/game/HourlyTeamScore.java | 30 +- .../examples/complete/game/LeaderBoard.java | 45 +- .../beam/examples/complete/game/UserScore.java | 18 +- .../complete/game/utils/WriteToBigQuery.java | 49 +- .../game/utils/WriteWindowedToBigQuery.java | 14 +- .../examples/MinimalWordCountJava8Test.java | 2 +- .../examples/complete/game/GameStatsTest.java | 2 +- .../complete/game/HourlyTeamScoreTest.java | 2 +- .../examples/complete/game/LeaderBoardTest.java | 10 +- .../examples/complete/game/UserScoreTest.java | 6 +- examples/pom.xml | 18 +- pom.xml | 123 ++- runners/core-java/pom.xml | 77 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 3 +- .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 3 +- .../core/GroupByKeyViaGroupByKeyOnly.java | 13 +- .../core/PushbackSideInputDoFnRunner.java | 28 +- .../beam/runners/core/SimpleDoFnRunner.java | 12 + .../beam/runners/core/SplittableParDo.java | 12 + .../beam/runners/core/ReduceFnTester.java | 6 +- .../UnboundedReadFromBoundedSourceTest.java | 4 +- runners/direct-java/pom.xml | 115 +-- .../runners/direct/AggregatorContainer.java | 20 +- ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +- .../beam/runners/direct/DirectGroupByKey.java | 42 +- .../direct/DirectGroupByKeyOverrideFactory.java | 25 +- .../beam/runners/direct/DirectRunner.java | 7 +- .../runners/direct/DirectTimerInternals.java | 13 + .../beam/runners/direct/EvaluationContext.java | 18 +- .../direct/ExecutorServiceParallelExecutor.java | 41 +- .../GroupAlsoByWindowEvaluatorFactory.java | 249 ++++-- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 31 +- .../direct/ImmutableListBundleFactory.java | 21 +- .../direct/PTransformOverrideFactory.java | 8 +- .../beam/runners/direct/ParDoEvaluator.java | 28 +- .../runners/direct/ParDoEvaluatorFactory.java | 126 +++ .../direct/ParDoMultiEvaluatorFactory.java | 107 --- .../direct/ParDoMultiEvaluatorHooks.java | 55 ++ .../runners/direct/ParDoOverrideFactory.java | 14 +- .../direct/ParDoSingleEvaluatorFactory.java | 110 --- .../direct/ParDoSingleEvaluatorHooks.java | 58 ++ .../direct/TestStreamEvaluatorFactory.java | 14 +- .../direct/TransformEvaluatorRegistry.java | 10 +- .../direct/UncommittedBundleOutputManager.java | 50 -- .../runners/direct/ViewEvaluatorFactory.java | 19 +- .../direct/WatermarkCallbackExecutor.java | 3 +- .../beam/runners/direct/WatermarkManager.java | 243 ++++-- .../direct/WriteWithShardingFactory.java | 15 +- .../runners/direct/EvaluationContextTest.java | 23 +- .../direct/GroupByKeyEvaluatorFactoryTest.java | 29 +- .../GroupByKeyOnlyEvaluatorFactoryTest.java | 31 +- .../direct/ImmutableListBundleFactoryTest.java | 15 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 16 +- .../direct/ParDoMultiEvaluatorFactoryTest.java | 439 ---------- .../direct/ParDoMultiEvaluatorHooksTest.java | 439 ++++++++++ .../direct/ParDoSingleEvaluatorFactoryTest.java | 335 -------- .../direct/ParDoSingleEvaluatorHooksTest.java | 335 ++++++++ .../runners/direct/WatermarkManagerTest.java | 102 +-- .../direct/WriteWithShardingFactoryTest.java | 26 +- runners/flink/examples/pom.xml | 17 - .../beam/runners/flink/examples/TFIDF.java | 6 +- .../flink/examples/streaming/AutoComplete.java | 8 +- .../flink/examples/streaming/JoinExamples.java | 12 +- .../examples/streaming/KafkaIOExamples.java | 2 +- .../examples/streaming/WindowedWordCount.java | 8 +- runners/flink/pom.xml | 9 + runners/flink/runner/pom.xml | 17 - .../wrappers/streaming/DoFnOperator.java | 10 +- .../wrappers/streaming/WindowDoFnOperator.java | 13 + runners/google-cloud-dataflow-java/pom.xml | 142 +--- .../dataflow/BlockingDataflowRunner.java | 181 ----- .../dataflow/DataflowJobCancelledException.java | 39 - .../dataflow/DataflowJobExecutionException.java | 35 - .../dataflow/DataflowJobUpdatedException.java | 51 -- .../runners/dataflow/DataflowPipelineJob.java | 61 +- .../dataflow/DataflowPipelineRegistrar.java | 11 +- .../dataflow/DataflowPipelineTranslator.java | 8 +- .../beam/runners/dataflow/DataflowRunner.java | 49 +- .../runners/dataflow/internal/IsmFormat.java | 5 +- .../dataflow/internal/ReadTranslator.java | 12 +- .../BlockingDataflowPipelineOptions.java | 28 - .../options/DataflowPipelineOptions.java | 8 + .../testing/TestDataflowPipelineOptions.java | 6 +- .../dataflow/testing/TestDataflowRunner.java | 7 +- .../dataflow/util/DataflowTemplateJob.java | 70 ++ .../dataflow/util/DataflowTransport.java | 9 +- .../beam/runners/dataflow/util/DoFnInfo.java | 53 +- .../runners/dataflow/util/MonitoringUtil.java | 1 - .../runners/dataflow/util/RandomAccessData.java | 2 +- .../dataflow/BlockingDataflowRunnerTest.java | 300 ------- .../dataflow/DataflowPipelineJobTest.java | 30 +- .../dataflow/DataflowPipelineRegistrarTest.java | 7 +- .../DataflowPipelineTranslatorTest.java | 28 + .../runners/dataflow/DataflowRunnerTest.java | 45 ++ runners/pom.xml | 16 + runners/spark/pom.xml | 87 +- .../runners/spark/SparkPipelineOptions.java | 14 +- .../apache/beam/runners/spark/SparkRunner.java | 19 - .../metrics/AggregatorMetricSource.java | 9 +- .../metrics/WithNamedAggregatorsSupport.java | 7 +- .../coders/BeamSparkRunnerRegistrator.java | 46 ++ .../runners/spark/io/EmptyCheckpointMark.java | 52 ++ .../apache/beam/runners/spark/io/KafkaIO.java | 131 --- .../beam/runners/spark/io/MicrobatchSource.java | 262 ++++++ .../beam/runners/spark/io/SourceDStream.java | 156 ++++ .../apache/beam/runners/spark/io/SourceRDD.java | 77 +- .../runners/spark/io/SparkUnboundedSource.java | 167 ++++ .../spark/stateful/StateSpecFunctions.java | 167 ++++ .../runners/spark/stateful/package-info.java | 22 + .../spark/translation/EvaluationContext.java | 6 +- .../translation/GroupCombineFunctions.java | 65 +- .../ReifyTimestampsAndWindowsFunction.java | 47 ++ .../spark/translation/SparkContextFactory.java | 5 +- .../spark/translation/SparkProcessContext.java | 46 +- .../spark/translation/SparkRuntimeContext.java | 2 +- .../spark/translation/TransformTranslator.java | 43 +- .../spark/translation/TranslationUtils.java | 17 + .../SparkRunnerStreamingContextFactory.java | 30 +- .../streaming/StreamingEvaluationContext.java | 6 +- .../streaming/StreamingTransformTranslator.java | 100 +-- .../spark/util/SparkSideInputReader.java | 2 +- .../runners/spark/ClearAggregatorsRule.java | 33 - .../runners/spark/InMemoryMetricsSinkRule.java | 32 - .../metrics/sink/ClearAggregatorsRule.java | 33 + .../metrics/sink/InMemoryMetrics.java | 15 +- .../metrics/sink/InMemoryMetricsSinkRule.java | 31 + .../metrics/sink/NamedAggregatorsTest.java | 92 +++ .../streaming/EmptyStreamAssertionTest.java | 3 +- .../streaming/FlattenStreamingTest.java | 6 +- .../streaming/KafkaStreamingTest.java | 152 +++- .../ResumeFromCheckpointStreamingTest.java | 37 +- .../streaming/SimpleStreamingWordCountTest.java | 3 +- .../streaming/utils/EmbeddedKafkaCluster.java | 51 +- .../utils/KafkaWriteOnBatchCompleted.java | 105 +++ .../utils/TestOptionsForStreaming.java | 12 +- sdks/java/build-tools/pom.xml | 4 +- .../src/main/resources/beam/checkstyle.xml | 9 +- sdks/java/core/pom.xml | 157 +--- .../org/apache/beam/sdk/PipelineResult.java | 10 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 147 +++- .../sdk/io/BoundedReadFromUnboundedSource.java | 4 +- .../org/apache/beam/sdk/io/BoundedSource.java | 11 +- .../apache/beam/sdk/io/CompressedSource.java | 4 +- .../org/apache/beam/sdk/io/FileBasedSink.java | 43 +- .../org/apache/beam/sdk/io/FileBasedSource.java | 59 +- .../beam/sdk/io/PubsubUnboundedSource.java | 123 ++- .../main/java/org/apache/beam/sdk/io/Sink.java | 4 +- .../java/org/apache/beam/sdk/io/TextIO.java | 43 +- .../java/org/apache/beam/sdk/io/XmlSource.java | 2 +- .../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +- .../org/apache/beam/sdk/options/GcpOptions.java | 136 +--- .../sdk/options/PipelineOptionsFactory.java | 248 ++++-- .../sdk/options/ProxyInvocationHandler.java | 5 +- .../apache/beam/sdk/options/ValueProvider.java | 38 +- .../beam/sdk/options/ValueProviderUtils.java | 60 ++ .../beam/sdk/testing/BigqueryMatcher.java | 15 +- .../apache/beam/sdk/transforms/Aggregator.java | 13 +- .../apache/beam/sdk/transforms/CombineFns.java | 4 +- .../sdk/transforms/DelegatingAggregator.java | 7 +- .../apache/beam/sdk/transforms/Distinct.java | 158 ++++ .../org/apache/beam/sdk/transforms/DoFn.java | 42 +- .../beam/sdk/transforms/DoFnAdapters.java | 22 + .../org/apache/beam/sdk/transforms/Flatten.java | 3 +- .../apache/beam/sdk/transforms/GroupByKey.java | 4 +- .../org/apache/beam/sdk/transforms/ParDo.java | 45 +- .../org/apache/beam/sdk/transforms/Regex.java | 454 +++++++++++ .../beam/sdk/transforms/RemoveDuplicates.java | 159 ---- .../org/apache/beam/sdk/transforms/View.java | 2 +- .../sdk/transforms/display/DisplayData.java | 5 +- .../beam/sdk/transforms/join/CoGroupByKey.java | 6 +- .../sdk/transforms/reflect/DoFnInvokers.java | 150 ++-- .../sdk/transforms/reflect/DoFnSignature.java | 113 ++- .../sdk/transforms/reflect/DoFnSignatures.java | 794 ++++++++++++------- .../splittabledofn/RestrictionTracker.java | 2 +- .../transforms/splittabledofn/package-info.java | 4 +- .../beam/sdk/transforms/windowing/Window.java | 130 +-- .../beam/sdk/transforms/windowing/WindowFn.java | 12 + .../beam/sdk/util/BaseExecutionContext.java | 2 +- .../BufferedElementCountingOutputStream.java | 4 +- .../apache/beam/sdk/util/CredentialFactory.java | 4 +- .../org/apache/beam/sdk/util/Credentials.java | 192 ----- .../apache/beam/sdk/util/ExecutableTrigger.java | 131 --- .../apache/beam/sdk/util/ExecutionContext.java | 8 +- .../sdk/util/ExposedByteArrayInputStream.java | 5 +- .../sdk/util/ExposedByteArrayOutputStream.java | 2 - .../beam/sdk/util/FileIOChannelFactory.java | 8 +- .../apache/beam/sdk/util/GatherAllPanes.java | 18 +- .../beam/sdk/util/GcpCredentialFactory.java | 41 +- .../beam/sdk/util/GcsIOChannelFactory.java | 8 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 2 - .../apache/beam/sdk/util/IOChannelFactory.java | 4 + .../apache/beam/sdk/util/MovingFunction.java | 4 +- .../beam/sdk/util/NoopCredentialFactory.java | 9 +- .../org/apache/beam/sdk/util/PubsubClient.java | 32 +- .../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +- .../apache/beam/sdk/util/PubsubJsonClient.java | 9 +- .../apache/beam/sdk/util/PubsubTestClient.java | 32 + .../sdk/util/ReifyTimestampAndWindowsDoFn.java | 41 - .../sdk/util/ReifyTimestampsAndWindows.java | 63 -- .../org/apache/beam/sdk/util/StringUtils.java | 8 +- .../apache/beam/sdk/util/TestCredential.java | 44 +- .../apache/beam/sdk/util/TimerInternals.java | 35 +- .../org/apache/beam/sdk/util/Transport.java | 9 +- .../apache/beam/sdk/util/WindowingStrategy.java | 10 +- .../beam/sdk/util/common/ReflectHelpers.java | 17 + .../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 13 +- .../sdk/util/state/InMemoryTimerInternals.java | 12 + .../beam/sdk/util/state/StateNamespace.java | 16 +- .../java/org/apache/beam/sdk/values/PDone.java | 2 +- .../apache/beam/sdk/values/TypeDescriptor.java | 48 +- .../apache/beam/sdk/values/TypeDescriptors.java | 1 - .../apache/beam/sdk/values/TypeParameter.java | 62 ++ .../java/org/apache/beam/sdk/io/AvroIOTest.java | 29 + .../apache/beam/sdk/io/CountingInputTest.java | 6 +- .../apache/beam/sdk/io/CountingSourceTest.java | 8 +- .../apache/beam/sdk/io/FileBasedSinkTest.java | 23 +- .../beam/sdk/io/PubsubUnboundedSourceTest.java | 79 ++ .../org/apache/beam/sdk/io/XmlSinkTest.java | 16 +- .../sdk/options/PipelineOptionsFactoryTest.java | 239 ++++++ .../sdk/options/ProxyInvocationHandlerTest.java | 29 +- .../beam/sdk/options/ValueProviderTest.java | 50 +- .../sdk/options/ValueProviderUtilsTest.java | 78 ++ .../apache/beam/sdk/testing/ExpectedLogs.java | 15 +- .../beam/sdk/testing/ExpectedLogsTest.java | 30 +- .../apache/beam/sdk/testing/PAssertTest.java | 27 + .../sdk/transforms/ApproximateUniqueTest.java | 2 +- .../beam/sdk/transforms/DistinctTest.java | 130 +++ .../apache/beam/sdk/transforms/ParDoTest.java | 61 ++ .../apache/beam/sdk/transforms/RegexTest.java | 248 ++++++ .../sdk/transforms/RemoveDuplicatesTest.java | 130 --- .../transforms/reflect/DoFnInvokersTest.java | 100 ++- .../DoFnSignaturesProcessElementTest.java | 18 +- .../DoFnSignaturesSplittableDoFnTest.java | 24 +- .../transforms/reflect/DoFnSignaturesTest.java | 43 +- .../reflect/DoFnSignaturesTestUtils.java | 13 +- .../sdk/transforms/windowing/WindowTest.java | 6 +- .../beam/sdk/util/ExecutableTriggerTest.java | 109 --- .../beam/sdk/util/PubsubGrpcClientTest.java | 9 +- .../sdk/util/common/ReflectHelpersTest.java | 25 + .../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 25 + .../beam/sdk/values/TypeDescriptorTest.java | 12 + sdks/java/extensions/join-library/pom.xml | 8 - sdks/java/extensions/pom.xml | 1 + sdks/java/extensions/sorter/README.md | 42 + sdks/java/extensions/sorter/pom.xml | 159 ++++ .../sorter/BufferedExternalSorter.java | 125 +++ .../sdk/extensions/sorter/ExternalSorter.java | 225 ++++++ .../sdk/extensions/sorter/InMemorySorter.java | 166 ++++ .../beam/sdk/extensions/sorter/SortValues.java | 213 +++++ .../beam/sdk/extensions/sorter/Sorter.java | 47 ++ .../sdk/extensions/sorter/package-info.java | 23 + .../sorter/BufferedExternalSorterTest.java | 177 +++++ .../extensions/sorter/ExternalSorterTest.java | 87 ++ .../extensions/sorter/InMemorySorterTest.java | 144 ++++ .../sdk/extensions/sorter/SortValuesTest.java | 128 +++ .../sdk/extensions/sorter/SorterTestUtils.java | 129 +++ sdks/java/io/google-cloud-platform/pom.xml | 17 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 +- .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 27 +- .../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 132 +++ .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 25 +- sdks/java/io/hdfs/pom.xml | 8 - sdks/java/io/jdbc/pom.xml | 14 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 39 + .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 33 + sdks/java/io/jms/pom.xml | 21 +- sdks/java/io/kafka/pom.xml | 28 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 6 +- sdks/java/io/kinesis/pom.xml | 23 +- .../beam/sdk/io/kinesis/CustomOptional.java | 21 +- .../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +- .../beam/sdk/io/kinesis/KinesisRecord.java | 3 +- .../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 + sdks/java/io/mongodb/pom.xml | 25 +- sdks/java/java8tests/pom.xml | 11 - .../beam/sdk/transforms/DistinctJava8Test.java | 104 +++ .../transforms/RemoveDuplicatesJava8Test.java | 104 --- .../main/resources/archetype-resources/pom.xml | 32 - sdks/java/microbenchmarks/pom.xml | 23 +- sdks/java/pom.xml | 9 +- sdks/pom.xml | 21 +- 310 files changed, 10045 insertions(+), 5852 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0c362282/runners/pom.xml ---------------------------------------------------------------------- diff --cc runners/pom.xml index 4be31d0,8084d0b..457bae5 --- a/runners/pom.xml +++ b/runners/pom.xml @@@ -42,14 -42,21 +42,30 @@@ <profiles> <profile> + <id>release</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + </plugin> + </plugins> + </build> + </profile> ++ <profile> + <id>java8</id> + <activation> + <jdk>[1.8,)</jdk> + </activation> + <modules> + <module>gearpump</module> + </modules> + </profile> <!-- A profile that adds an integration test phase if and only if the runnableOnServicePipelineOptions maven property has been set.