Merge branch master into gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2a0ba61e Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2a0ba61e Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2a0ba61e Branch: refs/heads/gearpump-runner Commit: 2a0ba61e8507e1539115b583749a78f14d577bd8 Parents: 02b2248 2a1055d Author: Kenneth Knowles <[email protected]> Authored: Thu Aug 25 11:36:45 2016 -0700 Committer: Kenneth Knowles <[email protected]> Committed: Thu Aug 25 11:36:45 2016 -0700 ---------------------------------------------------------------------- .travis.yml | 17 +- .travis/README.md | 23 + .travis/settings.xml | 33 + .travis/test_wordcount.sh | 125 + KEYS | 141 - NOTICE | 1 - README.md | 25 +- examples/java/pom.xml | 48 +- .../beam/examples/DebuggingWordCount.java | 2 +- .../apache/beam/examples/MinimalWordCount.java | 2 +- .../apache/beam/examples/WindowedWordCount.java | 6 +- .../org/apache/beam/examples/WordCount.java | 26 +- .../beam/examples/common/ExampleUtils.java | 4 +- .../examples/common/PubsubFileInjector.java | 10 +- .../beam/examples/complete/AutoComplete.java | 37 +- .../examples/complete/StreamingWordExtract.java | 12 +- .../apache/beam/examples/complete/TfIdf.java | 12 +- .../examples/complete/TopWikipediaSessions.java | 29 +- .../examples/complete/TrafficMaxLaneFlow.java | 8 +- .../beam/examples/complete/TrafficRoutes.java | 10 +- .../examples/cookbook/BigQueryTornadoes.java | 6 +- .../cookbook/CombinePerKeyExamples.java | 6 +- .../examples/cookbook/DatastoreWordCount.java | 9 +- .../beam/examples/cookbook/FilterExamples.java | 10 +- .../beam/examples/cookbook/JoinExamples.java | 10 +- .../examples/cookbook/MaxPerKeyExamples.java | 6 +- .../beam/examples/cookbook/TriggerExample.java | 21 +- .../org/apache/beam/examples/WordCountIT.java | 2 +- .../org/apache/beam/examples/WordCountTest.java | 3 +- .../examples/complete/AutoCompleteTest.java | 2 +- .../examples/cookbook/TriggerExampleTest.java | 2 +- examples/java/src/test/resources/LICENSE | 202 ++ examples/java8/pom.xml | 7 +- .../beam/examples/complete/game/GameStats.java | 16 +- .../beam/examples/complete/game/UserScore.java | 2 +- .../complete/game/utils/WriteToBigQuery.java | 18 +- .../game/utils/WriteWindowedToBigQuery.java | 14 +- .../examples/complete/game/UserScoreTest.java | 2 +- examples/pom.xml | 2 +- pom.xml | 72 +- runners/core-java/pom.xml | 2 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 28 +- .../core/UnboundedReadFromBoundedSource.java | 6 +- .../apache/beam/runners/core/package-info.java | 22 + .../apache/beam/sdk/util/AssignWindowsDoFn.java | 10 +- .../org/apache/beam/sdk/util/DoFnRunner.java | 21 +- .../apache/beam/sdk/util/DoFnRunnerBase.java | 82 +- .../org/apache/beam/sdk/util/DoFnRunners.java | 104 +- .../beam/sdk/util/GroupAlsoByWindowsDoFn.java | 16 +- .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 14 +- .../sdk/util/GroupByKeyViaGroupByKeyOnly.java | 85 +- .../sdk/util/LateDataDroppingDoFnRunner.java | 4 +- .../apache/beam/sdk/util/PaneInfoTracker.java | 1 - .../apache/beam/sdk/util/ReduceFnRunner.java | 4 +- .../apache/beam/sdk/util/SimpleDoFnRunner.java | 19 +- .../org/apache/beam/sdk/util/WatermarkHold.java | 1 - .../beam/sdk/util/common/package-info.java | 20 + .../org/apache/beam/sdk/util/package-info.java | 20 + .../UnboundedReadFromBoundedSourceTest.java | 2 +- .../sdk/util/GroupAlsoByWindowsProperties.java | 50 +- ...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 8 +- .../beam/sdk/util/ReduceFnRunnerTest.java | 1 + .../apache/beam/sdk/util/ReduceFnTester.java | 1 + .../beam/sdk/util/SimpleDoFnRunnerTest.java | 6 +- runners/direct-java/pom.xml | 2 +- .../runners/direct/AggregatorContainer.java | 183 ++ .../direct/BoundedReadEvaluatorFactory.java | 26 +- .../beam/runners/direct/CloningThreadLocal.java | 43 + .../beam/runners/direct/CommittedResult.java | 23 +- .../beam/runners/direct/CompletionCallback.java | 8 + .../beam/runners/direct/DirectRegistrar.java | 4 +- .../beam/runners/direct/DirectRunner.java | 57 +- .../beam/runners/direct/EvaluationContext.java | 57 +- .../beam/runners/direct/EvaluatorKey.java | 55 - .../direct/ExecutorServiceParallelExecutor.java | 186 +- .../GroupAlsoByWindowEvaluatorFactory.java | 33 +- .../ImmutabilityCheckingBundleFactory.java | 4 +- .../beam/runners/direct/ParDoEvaluator.java | 24 +- .../direct/ParDoMultiEvaluatorFactory.java | 53 +- .../direct/ParDoSingleEvaluatorFactory.java | 59 +- ...rializableCloningThreadLocalCacheLoader.java | 54 - .../beam/runners/direct/SideInputContainer.java | 2 +- .../runners/direct/StepTransformResult.java | 95 +- .../direct/TransformEvaluatorFactory.java | 10 +- .../beam/runners/direct/TransformExecutor.java | 3 +- .../beam/runners/direct/TransformResult.java | 16 +- .../direct/UnboundedReadEvaluatorFactory.java | 32 +- .../runners/direct/ViewEvaluatorFactory.java | 14 +- .../direct/WriteWithShardingFactory.java | 11 +- .../runners/direct/AggregatorContainerTest.java | 134 + .../runners/direct/CloningThreadLocalTest.java | 92 + .../runners/direct/CommittedResultTest.java | 26 +- .../ConsumerTrackingPipelineVisitorTest.java | 22 +- .../runners/direct/DirectRegistrarTest.java | 17 +- .../beam/runners/direct/DirectRunnerTest.java | 72 +- .../runners/direct/EvaluationContextTest.java | 32 +- .../ImmutabilityCheckingBundleFactoryTest.java | 6 +- .../ImmutabilityEnforcementFactoryTest.java | 6 +- .../direct/KeyedPValueTrackingVisitorTest.java | 6 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 18 +- .../direct/ParDoMultiEvaluatorFactoryTest.java | 41 +- .../direct/ParDoSingleEvaluatorFactoryTest.java | 41 +- ...izableCloningThreadLocalCacheLoaderTest.java | 99 - .../runners/direct/StepTransformResultTest.java | 91 + .../runners/direct/TransformExecutorTest.java | 33 +- .../runners/direct/WatermarkManagerTest.java | 18 +- runners/flink/README.md | 25 +- runners/flink/examples/pom.xml | 13 +- .../beam/runners/flink/examples/TFIDF.java | 12 +- .../beam/runners/flink/examples/WordCount.java | 7 +- .../flink/examples/streaming/AutoComplete.java | 25 +- .../flink/examples/streaming/JoinExamples.java | 6 +- .../examples/streaming/KafkaIOExamples.java | 3 +- .../KafkaWindowedWordCountExample.java | 4 +- .../examples/streaming/WindowedWordCount.java | 4 +- runners/flink/pom.xml | 2 +- runners/flink/runner/pom.xml | 116 +- .../beam/runners/flink/FlinkRunnerResult.java | 26 +- .../FlinkBatchTransformTranslators.java | 12 +- .../FlinkStreamingTransformTranslators.java | 9 +- .../functions/FlinkDoFnFunction.java | 10 +- .../FlinkMergingNonShuffleReduceFunction.java | 8 +- .../functions/FlinkMultiOutputDoFnFunction.java | 10 +- .../FlinkMultiOutputProcessContext.java | 6 +- .../functions/FlinkNoElementAssignContext.java | 8 +- .../functions/FlinkPartialReduceFunction.java | 8 +- .../functions/FlinkProcessContext.java | 18 +- .../functions/FlinkReduceFunction.java | 8 +- .../functions/SideInputInitializer.java | 2 +- .../streaming/FlinkAbstractParDoWrapper.java | 18 +- .../FlinkGroupAlsoByWindowWrapper.java | 41 +- .../streaming/FlinkParDoBoundMultiWrapper.java | 4 +- .../streaming/FlinkParDoBoundWrapper.java | 4 +- .../state/AbstractFlinkTimerInternals.java | 4 +- .../beam/runners/flink/PipelineOptionsTest.java | 6 +- .../beam/runners/flink/ReadSourceITCase.java | 2 +- .../flink/ReadSourceStreamingITCase.java | 4 +- .../flink/streaming/GroupByNullKeyTest.java | 8 +- .../streaming/TopWikipediaSessionsITCase.java | 6 +- runners/google-cloud-dataflow-java/pom.xml | 35 +- .../dataflow/BlockingDataflowRunner.java | 5 +- .../runners/dataflow/DataflowPipelineJob.java | 106 +- .../dataflow/DataflowPipelineTranslator.java | 6 +- .../beam/runners/dataflow/DataflowRunner.java | 198 +- .../dataflow/internal/AssignWindows.java | 6 +- .../DataflowUnboundedReadFromBoundedSource.java | 4 +- .../options/DataflowPipelineDebugOptions.java | 43 - .../options/DataflowPipelineOptions.java | 6 +- .../runners/dataflow/options/package-info.java | 22 + .../beam/runners/dataflow/package-info.java | 22 + .../dataflow/testing/TestDataflowRunner.java | 7 +- .../runners/dataflow/testing/package-info.java | 24 + .../dataflow/util/DataflowPathValidator.java | 98 - .../beam/runners/dataflow/util/DoFnInfo.java | 16 +- .../runners/dataflow/util/package-info.java | 20 + .../dataflow/BlockingDataflowRunnerTest.java | 9 +- .../dataflow/DataflowPipelineJobTest.java | 47 +- .../DataflowPipelineTranslatorTest.java | 12 +- .../runners/dataflow/DataflowRunnerTest.java | 39 +- .../options/DataflowPipelineOptionsTest.java | 3 + .../testing/TestDataflowRunnerTest.java | 34 +- .../util/DataflowPathValidatorTest.java | 94 - runners/pom.xml | 72 +- runners/spark/README.md | 3 +- runners/spark/pom.xml | 27 +- .../runners/spark/SparkPipelineOptions.java | 16 +- .../apache/beam/runners/spark/SparkRunner.java | 33 +- .../runners/spark/SparkRunnerRegistrar.java | 6 +- .../spark/SparkStreamingPipelineOptions.java | 41 - .../beam/runners/spark/TestSparkRunner.java | 2 - .../runners/spark/aggregators/package-info.java | 20 + .../beam/runners/spark/coders/package-info.java | 22 + .../beam/runners/spark/examples/WordCount.java | 137 + .../runners/spark/io/hadoop/package-info.java | 22 + .../beam/runners/spark/io/package-info.java | 22 + .../apache/beam/runners/spark/package-info.java | 22 + .../runners/spark/translation/DoFnFunction.java | 8 +- .../spark/translation/EvaluationContext.java | 25 +- .../spark/translation/MultiDoFnFunction.java | 8 +- .../spark/translation/SparkContextFactory.java | 8 +- .../spark/translation/SparkProcessContext.java | 20 +- .../spark/translation/SparkRuntimeContext.java | 2 +- .../spark/translation/TransformTranslator.java | 73 +- .../runners/spark/translation/package-info.java | 22 + .../streaming/StreamingTransformTranslator.java | 14 +- .../StreamingWindowPipelineDetector.java | 102 - .../translation/streaming/package-info.java | 22 + .../beam/runners/spark/util/package-info.java | 22 + .../apache/beam/runners/spark/DeDupTest.java | 2 +- .../beam/runners/spark/EmptyInputTest.java | 2 +- .../beam/runners/spark/SimpleWordCountTest.java | 80 +- .../runners/spark/SparkRunnerRegistrarTest.java | 2 +- .../apache/beam/runners/spark/TfIdfTest.java | 201 +- .../beam/runners/spark/io/AvroPipelineTest.java | 7 +- .../beam/runners/spark/io/NumShardsTest.java | 4 +- .../io/hadoop/HadoopFileFormatPipelineTest.java | 7 +- .../spark/translation/CombineGloballyTest.java | 2 +- .../spark/translation/CombinePerKeyTest.java | 9 +- .../spark/translation/DoFnOutputTest.java | 10 +- .../translation/MultiOutputWordCountTest.java | 15 +- .../spark/translation/SerializationTest.java | 10 +- .../spark/translation/SideEffectsTest.java | 10 +- .../translation/TransformTranslatorTest.java | 13 +- .../translation/WindowedWordCountTest.java | 12 +- .../streaming/FlattenStreamingTest.java | 11 +- .../streaming/KafkaStreamingTest.java | 19 +- .../streaming/SimpleStreamingWordCountTest.java | 18 +- .../streaming/utils/EmbeddedKafkaCluster.java | 4 +- sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/checkstyle.xml | 14 + .../src/main/resources/beam/findbugs-filter.xml | 13 +- .../src/main/resources/beam/suppressions.xml | 24 + sdks/java/core/pom.xml | 27 +- .../beam/sdk/AggregatorPipelineExtractor.java | 93 + .../beam/sdk/AggregatorRetrievalException.java | 33 + .../org/apache/beam/sdk/AggregatorValues.java | 52 + .../main/java/org/apache/beam/sdk/Pipeline.java | 12 +- .../org/apache/beam/sdk/PipelineResult.java | 40 +- .../org/apache/beam/sdk/coders/AvroCoder.java | 1 - .../java/org/apache/beam/sdk/coders/Coder.java | 2 +- .../apache/beam/sdk/coders/CoderRegistry.java | 38 +- .../apache/beam/sdk/coders/DurationCoder.java | 1 - .../apache/beam/sdk/coders/InstantCoder.java | 1 - .../org/apache/beam/sdk/coders/JAXBCoder.java | 2 +- .../apache/beam/sdk/coders/StandardCoder.java | 14 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 3 +- .../java/org/apache/beam/sdk/io/BigQueryIO.java | 2447 ---------------- .../org/apache/beam/sdk/io/FileBasedSink.java | 2 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 14 +- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 9 +- .../beam/sdk/io/PubsubUnboundedSource.java | 3 +- .../java/org/apache/beam/sdk/io/Source.java | 2 +- .../main/java/org/apache/beam/sdk/io/Write.java | 17 +- .../org/apache/beam/sdk/io/package-info.java | 3 +- .../org/apache/beam/sdk/options/GcpOptions.java | 11 +- .../org/apache/beam/sdk/options/GcsOptions.java | 44 + .../beam/sdk/options/PipelineOptions.java | 3 +- .../sdk/options/PipelineOptionsFactory.java | 3 +- .../sdk/options/PipelineOptionsReflector.java | 1 + .../runners/AggregatorPipelineExtractor.java | 93 - .../runners/AggregatorRetrievalException.java | 33 - .../beam/sdk/runners/AggregatorValues.java | 52 - .../beam/sdk/testing/CoderProperties.java | 82 +- .../org/apache/beam/sdk/testing/PAssert.java | 19 +- .../beam/sdk/testing/SerializableMatchers.java | 23 +- .../apache/beam/sdk/testing/TestPipeline.java | 1 - .../beam/sdk/testing/TestPipelineOptions.java | 1 + .../apache/beam/sdk/transforms/Aggregator.java | 32 +- .../sdk/transforms/AggregatorRetriever.java | 6 +- .../org/apache/beam/sdk/transforms/Combine.java | 16 +- .../apache/beam/sdk/transforms/CombineFns.java | 10 +- .../org/apache/beam/sdk/transforms/Count.java | 2 +- .../org/apache/beam/sdk/transforms/Create.java | 2 +- .../org/apache/beam/sdk/transforms/DoFn.java | 450 ++- .../beam/sdk/transforms/DoFnReflector.java | 234 +- .../apache/beam/sdk/transforms/DoFnTester.java | 153 +- .../beam/sdk/transforms/DoFnWithContext.java | 429 --- .../org/apache/beam/sdk/transforms/Filter.java | 7 +- .../beam/sdk/transforms/FlatMapElements.java | 126 +- .../org/apache/beam/sdk/transforms/Flatten.java | 2 +- .../apache/beam/sdk/transforms/GroupByKey.java | 2 +- .../transforms/IntraBundleParallelization.java | 40 +- .../org/apache/beam/sdk/transforms/Keys.java | 2 +- .../org/apache/beam/sdk/transforms/KvSwap.java | 2 +- .../apache/beam/sdk/transforms/MapElements.java | 60 +- .../org/apache/beam/sdk/transforms/OldDoFn.java | 567 ++++ .../apache/beam/sdk/transforms/PTransform.java | 2 +- .../org/apache/beam/sdk/transforms/ParDo.java | 227 +- .../apache/beam/sdk/transforms/Partition.java | 2 +- .../beam/sdk/transforms/RemoveDuplicates.java | 2 +- .../org/apache/beam/sdk/transforms/Sample.java | 2 +- .../beam/sdk/transforms/SimpleFunction.java | 56 +- .../org/apache/beam/sdk/transforms/Values.java | 2 +- .../org/apache/beam/sdk/transforms/ViewFn.java | 45 + .../apache/beam/sdk/transforms/WithKeys.java | 2 +- .../beam/sdk/transforms/WithTimestamps.java | 2 +- .../sdk/transforms/display/DisplayData.java | 1 - .../beam/sdk/transforms/join/CoGbkResult.java | 1 - .../beam/sdk/transforms/join/CoGroupByKey.java | 6 +- .../sdk/transforms/windowing/AfterEach.java | 1 + .../windowing/AfterProcessingTime.java | 6 +- .../transforms/windowing/IntervalWindow.java | 1 - .../beam/sdk/transforms/windowing/Never.java | 1 + .../beam/sdk/transforms/windowing/PaneInfo.java | 12 +- .../beam/sdk/transforms/windowing/Window.java | 25 +- .../org/apache/beam/sdk/util/AvroUtils.java | 207 -- .../beam/sdk/util/BaseExecutionContext.java | 4 +- .../apache/beam/sdk/util/BigQueryServices.java | 177 -- .../beam/sdk/util/BigQueryServicesImpl.java | 515 ---- .../beam/sdk/util/BigQueryTableInserter.java | 469 --- .../beam/sdk/util/BigQueryTableRowIterator.java | 472 --- .../apache/beam/sdk/util/BucketingFunction.java | 1 + .../org/apache/beam/sdk/util/CoderUtils.java | 4 +- .../beam/sdk/util/CombineContextFactory.java | 6 +- .../apache/beam/sdk/util/CounterAggregator.java | 35 +- .../beam/sdk/util/DirectSideInputReader.java | 4 +- .../apache/beam/sdk/util/ExecutionContext.java | 8 +- .../beam/sdk/util/FileIOChannelFactory.java | 31 +- .../apache/beam/sdk/util/GcsPathValidator.java | 97 + .../apache/beam/sdk/util/IOChannelUtils.java | 1 + .../beam/sdk/util/MapAggregatorValues.java | 50 - .../apache/beam/sdk/util/MovingFunction.java | 1 + .../apache/beam/sdk/util/PCollectionViews.java | 228 +- .../beam/sdk/util/PerKeyCombineFnRunner.java | 44 +- .../beam/sdk/util/PerKeyCombineFnRunners.java | 30 +- .../org/apache/beam/sdk/util/PubsubClient.java | 1 + .../apache/beam/sdk/util/PubsubTestClient.java | 1 + .../sdk/util/ReifyTimestampAndWindowsDoFn.java | 6 +- .../org/apache/beam/sdk/util/Reshuffle.java | 2 +- .../apache/beam/sdk/util/SerializableUtils.java | 2 +- .../org/apache/beam/sdk/util/StringUtils.java | 2 +- .../beam/sdk/util/SystemDoFnInternal.java | 6 +- .../org/apache/beam/sdk/util/TimeDomain.java | 2 +- .../apache/beam/sdk/util/TimerInternals.java | 16 +- .../apache/beam/sdk/util/ValueWithRecordId.java | 6 +- .../org/apache/beam/sdk/util/WindowedValue.java | 1 - .../beam/sdk/util/WindowingInternals.java | 4 +- .../apache/beam/sdk/util/WindowingStrategy.java | 6 +- .../util/common/ElementByteSizeObserver.java | 18 +- .../beam/sdk/util/common/ReflectHelpers.java | 1 + .../beam/sdk/util/common/package-info.java | 3 +- .../beam/sdk/util/gcsfs/package-info.java | 3 +- .../org/apache/beam/sdk/util/package-info.java | 3 +- .../CopyOnAccessInMemoryStateInternals.java | 4 +- .../sdk/util/state/StateInternalsFactory.java | 36 + .../beam/sdk/util/state/StateNamespaces.java | 2 +- .../apache/beam/sdk/util/state/StateTags.java | 6 +- .../beam/sdk/util/state/package-info.java | 22 + .../org/apache/beam/sdk/values/PCollection.java | 34 +- .../apache/beam/sdk/values/PCollectionList.java | 2 +- .../apache/beam/sdk/values/PCollectionView.java | 29 +- .../org/apache/beam/sdk/values/POutput.java | 2 +- .../beam/sdk/values/TimestampedValue.java | 1 - .../org/apache/beam/sdk/values/TupleTag.java | 2 +- .../apache/beam/sdk/values/TypeDescriptors.java | 13 + .../dataflow/util/GcsPathValidatorTest.java | 104 + .../sdk/AggregatorPipelineExtractorTest.java | 229 ++ .../java/org/apache/beam/sdk/PipelineTest.java | 2 +- .../apache/beam/sdk/coders/AvroCoderTest.java | 2 +- .../beam/sdk/coders/BigDecimalCoderTest.java | 15 +- .../beam/sdk/coders/BigIntegerCoderTest.java | 14 +- .../beam/sdk/coders/ByteArrayCoderTest.java | 5 +- .../beam/sdk/coders/CoderRegistryTest.java | 22 +- .../beam/sdk/coders/SerializableCoderTest.java | 4 +- .../beam/sdk/coders/StandardCoderTest.java | 25 + .../org/apache/beam/sdk/io/AvroSourceTest.java | 5 +- .../org/apache/beam/sdk/io/BigQueryIOTest.java | 1231 -------- .../io/BoundedReadFromUnboundedSourceTest.java | 1 + .../beam/sdk/io/CompressedSourceTest.java | 1 + .../apache/beam/sdk/io/CountingInputTest.java | 3 +- .../apache/beam/sdk/io/CountingSourceTest.java | 2 +- .../beam/sdk/io/OffsetBasedSourceTest.java | 1 + .../beam/sdk/io/PubsubUnboundedSinkTest.java | 2 +- .../java/org/apache/beam/sdk/io/ReadTest.java | 1 + .../java/org/apache/beam/sdk/io/TextIOTest.java | 1 + .../java/org/apache/beam/sdk/io/WriteTest.java | 17 +- .../org/apache/beam/sdk/io/XmlSinkTest.java | 1 + .../apache/beam/sdk/options/GcpOptionsTest.java | 3 + .../sdk/options/GoogleApiDebugOptionsTest.java | 1 - .../sdk/options/PipelineOptionsFactoryTest.java | 1 - .../beam/sdk/options/PipelineOptionsTest.java | 1 - .../sdk/options/ProxyInvocationHandlerTest.java | 32 +- .../AggregatorPipelineExtractorTest.java | 229 -- .../beam/sdk/testing/CoderPropertiesTest.java | 26 + .../sdk/testing/DataflowJUnitTestRunner.java | 130 - .../apache/beam/sdk/testing/PAssertTest.java | 1 - .../sdk/testing/PCollectionViewTesting.java | 35 +- .../beam/sdk/testing/TestPipelineTest.java | 1 - .../transforms/ApproximateQuantilesTest.java | 1 + .../sdk/transforms/ApproximateUniqueTest.java | 3 +- .../beam/sdk/transforms/CombineFnsTest.java | 4 +- .../apache/beam/sdk/transforms/CombineTest.java | 10 +- .../apache/beam/sdk/transforms/CreateTest.java | 2 +- .../beam/sdk/transforms/DoFnContextTest.java | 69 - .../DoFnDelegatingAggregatorTest.java | 16 +- .../beam/sdk/transforms/DoFnReflectorTest.java | 294 +- .../apache/beam/sdk/transforms/DoFnTest.java | 68 +- .../beam/sdk/transforms/DoFnTesterTest.java | 10 +- .../sdk/transforms/DoFnWithContextTest.java | 237 -- .../sdk/transforms/FlatMapElementsTest.java | 67 + .../apache/beam/sdk/transforms/FlattenTest.java | 4 +- .../beam/sdk/transforms/GroupByKeyTest.java | 6 +- .../IntraBundleParallelizationTest.java | 23 +- .../beam/sdk/transforms/MapElementsTest.java | 93 + .../org/apache/beam/sdk/transforms/MaxTest.java | 1 + .../org/apache/beam/sdk/transforms/MinTest.java | 2 + .../apache/beam/sdk/transforms/NoOpDoFn.java | 144 - .../apache/beam/sdk/transforms/NoOpOldDoFn.java | 144 + .../beam/sdk/transforms/OldDoFnContextTest.java | 69 + .../apache/beam/sdk/transforms/OldDoFnTest.java | 243 ++ .../apache/beam/sdk/transforms/ParDoTest.java | 108 +- .../beam/sdk/transforms/PartitionTest.java | 1 + .../apache/beam/sdk/transforms/SampleTest.java | 1 + .../org/apache/beam/sdk/transforms/TopTest.java | 1 + .../apache/beam/sdk/transforms/ViewTest.java | 396 +-- .../beam/sdk/transforms/WithTimestampsTest.java | 4 +- .../display/DisplayDataEvaluatorTest.java | 4 +- .../display/DisplayDataMatchersTest.java | 1 + .../sdk/transforms/display/DisplayDataTest.java | 4 +- .../dofnreflector/DoFnReflectorTestHelper.java | 116 + .../sdk/transforms/join/CoGroupByKeyTest.java | 26 +- .../sdk/transforms/windowing/NeverTest.java | 1 + .../sdk/transforms/windowing/WindowTest.java | 4 +- .../sdk/transforms/windowing/WindowingTest.java | 17 +- .../org/apache/beam/sdk/util/AvroUtilsTest.java | 78 - .../beam/sdk/util/BigQueryServicesImplTest.java | 303 -- .../sdk/util/BigQueryTableInserterTest.java | 311 -- .../sdk/util/BigQueryTableRowIteratorTest.java | 256 -- .../apache/beam/sdk/util/BigQueryUtilTest.java | 485 --- .../beam/sdk/util/BucketingFunctionTest.java | 4 +- .../beam/sdk/util/FileIOChannelFactoryTest.java | 26 +- .../beam/sdk/util/IOChannelUtilsTest.java | 8 + .../beam/sdk/util/MovingFunctionTest.java | 4 +- .../util/RetryHttpRequestInitializerTest.java | 20 +- .../beam/sdk/util/SerializableUtilsTest.java | 1 - .../apache/beam/sdk/util/SerializerTest.java | 1 - .../apache/beam/sdk/util/StringUtilsTest.java | 16 +- .../beam/sdk/util/TimerInternalsTest.java | 47 + .../org/apache/beam/sdk/util/TriggerTester.java | 1 + .../beam/sdk/util/common/CounterTest.java | 1 + .../beam/sdk/util/common/CounterTestUtils.java | 57 - .../beam/sdk/values/PCollectionTupleTest.java | 2 +- .../apache/beam/sdk/values/TypedPValueTest.java | 20 +- sdks/java/extensions/join-library/pom.xml | 2 +- .../beam/sdk/extensions/joinlibrary/Join.java | 6 +- .../extensions/joinlibrary/package-info.java | 23 + sdks/java/extensions/pom.xml | 2 +- sdks/java/io/google-cloud-platform/pom.xml | 56 +- .../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 236 ++ .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 2749 ++++++++++++++++++ .../sdk/io/gcp/bigquery/BigQueryServices.java | 191 ++ .../io/gcp/bigquery/BigQueryServicesImpl.java | 722 +++++ .../io/gcp/bigquery/BigQueryTableInserter.java | 221 ++ .../gcp/bigquery/BigQueryTableRowIterator.java | 474 +++ .../beam/sdk/io/gcp/bigquery/package-info.java | 24 + .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 21 +- .../beam/sdk/io/gcp/datastore/V1Beta3.java | 817 +++--- .../io/gcp/bigquery/BigQueryAvroUtilsTest.java | 143 + .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1438 +++++++++ .../gcp/bigquery/BigQueryServicesImplTest.java | 359 +++ .../gcp/bigquery/BigQueryTableInserterTest.java | 249 ++ .../bigquery/BigQueryTableRowIteratorTest.java | 256 ++ .../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 454 +++ .../sdk/io/gcp/bigtable/BigtableIOTest.java | 25 + .../sdk/io/gcp/bigtable/BigtableWriteIT.java | 2 +- .../beam/sdk/io/gcp/datastore/V1Beta3Test.java | 466 ++- .../sdk/io/gcp/datastore/V1Beta3TestUtil.java | 3 +- sdks/java/io/hdfs/pom.xml | 2 +- .../apache/beam/sdk/io/hdfs/package-info.java | 22 + .../sdk/io/hdfs/simpleauth/package-info.java | 22 + sdks/java/io/jms/pom.xml | 134 + .../beam/sdk/io/jms/JmsCheckpointMark.java | 82 + .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 516 ++++ .../org/apache/beam/sdk/io/jms/JmsRecord.java | 153 + .../apache/beam/sdk/io/jms/package-info.java | 22 + .../org/apache/beam/sdk/io/jms/JmsIOTest.java | 145 + sdks/java/io/kafka/pom.xml | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 13 +- .../apache/beam/sdk/io/kafka/package-info.java | 22 + .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 5 +- sdks/java/io/pom.xml | 3 +- sdks/java/java8tests/pom.xml | 2 +- .../transforms/RemoveDuplicatesJava8Test.java | 10 +- .../beam/sdk/transforms/WithKeysJava8Test.java | 11 +- .../sdk/transforms/WithTimestampsJava8Test.java | 4 +- sdks/java/maven-archetypes/examples/pom.xml | 2 +- .../main/resources/archetype-resources/pom.xml | 7 + .../src/main/java/DebuggingWordCount.java | 4 +- .../src/main/java/MinimalWordCount.java | 6 +- .../src/main/java/WindowedWordCount.java | 8 +- .../src/main/java/WordCount.java | 6 +- .../main/java/common/DataflowExampleUtils.java | 5 +- .../main/java/common/PubsubFileInjector.java | 4 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- .../src/main/java/StarterPipeline.java | 6 +- .../src/main/java/it/pkg/StarterPipeline.java | 6 +- sdks/java/microbenchmarks/pom.xml | 2 +- .../microbenchmarks/coders/package-info.java | 22 + .../transforms/DoFnReflectorBenchmark.java | 62 +- .../transforms/package-info.java | 22 + sdks/java/pom.xml | 2 +- sdks/pom.xml | 2 +- testing/travis/README.md | 23 - testing/travis/test_wordcount.sh | 125 - 485 files changed, 17493 insertions(+), 13132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2a0ba61e/runners/pom.xml ---------------------------------------------------------------------- diff --cc runners/pom.xml index 3f985c5,605c3b2..6b8434a --- a/runners/pom.xml +++ b/runners/pom.xml @@@ -40,44 -40,45 +40,54 @@@ </modules> <profiles> + <profile> + <id>java8</id> + <activation> + <jdk>[1.8,)</jdk> + </activation> + <modules> + <module>gearpump</module> + </modules> + </profile> + + <!-- A profile that adds an integration test phase if and only if + the runnableOnServicePipelineOptions maven property has been set. + It should be set to a valid PipelineOptions JSON string. --> + <profile> + <id>runnable-on-service-tests</id> + <activation> + <property><name>runnableOnServicePipelineOptions</name></property> + </activation> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <executions> + <execution> + <id>runnable-on-service-tests</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <groups>org.apache.beam.sdk.testing.RunnableOnService</groups> + <parallel>all</parallel> + <threadCount>4</threadCount> + <dependenciesToScan> + <dependency>org.apache.beam:beam-sdks-java-core</dependency> + </dependenciesToScan> + <systemPropertyVariables> + <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions> + </systemPropertyVariables> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> </profiles> - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <executions> - <execution> - <id>runnable-on-service-tests</id> - <phase>integration-test</phase> - <goals> - <goal>test</goal> - </goals> - <configuration> - <groups>org.apache.beam.sdk.testing.RunnableOnService</groups> - <parallel>all</parallel> - <threadCount>4</threadCount> - <dependenciesToScan> - <dependency>org.apache.beam:beam-sdks-java-core</dependency> - </dependenciesToScan> - <systemPropertyVariables> - <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions> - </systemPropertyVariables> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </pluginManagement> - </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2a0ba61e/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java ----------------------------------------------------------------------
