Merge remote-tracking branch 'upstream/master' into gearpump-runner # Conflicts: # .travis.yml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9a59ea32 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9a59ea32 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9a59ea32 Branch: refs/heads/master Commit: 9a59ea3207c1a5c606692400c837f1ca7c6c5cbb Parents: 4078c22 7d6f8bd Author: manuzhang <owenzhang1...@gmail.com> Authored: Wed May 3 16:48:21 2017 +0800 Committer: manuzhang <owenzhang1...@gmail.com> Committed: Wed May 3 16:48:21 2017 +0800 ---------------------------------------------------------------------- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .gitignore | 2 + .../jenkins/common_job_properties.groovy | 1 - .../job_beam_PerformanceTests_JDBC.groovy | 3 + .test-infra/jenkins/test_wordcount.sh | 125 + .test-infra/travis/README.md | 23 - .test-infra/travis/settings.xml | 33 - .test-infra/travis/test_wordcount.sh | 125 - .travis.yml | 94 - README.md | 3 +- examples/java/pom.xml | 34 +- .../beam/examples/DebuggingWordCount.java | 22 +- .../apache/beam/examples/MinimalWordCount.java | 7 +- .../apache/beam/examples/WindowedWordCount.java | 2 +- .../org/apache/beam/examples/WordCount.java | 20 +- .../common/ExampleBigQueryTableOptions.java | 2 +- ...xamplePubsubTopicAndSubscriptionOptions.java | 2 +- .../common/ExamplePubsubTopicOptions.java | 2 +- .../beam/examples/common/ExampleUtils.java | 68 +- .../examples/common/WriteOneFilePerWindow.java | 11 +- .../beam/examples/complete/AutoComplete.java | 2 +- .../examples/complete/StreamingWordExtract.java | 4 +- .../apache/beam/examples/complete/TfIdf.java | 6 +- .../examples/complete/TopWikipediaSessions.java | 4 +- .../examples/complete/TrafficMaxLaneFlow.java | 2 +- .../beam/examples/complete/TrafficRoutes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 13 +- .../beam/examples/cookbook/DistinctExample.java | 4 +- .../beam/examples/cookbook/JoinExamples.java | 2 +- .../beam/examples/cookbook/TriggerExample.java | 6 +- .../beam/examples/WindowedWordCountIT.java | 17 +- .../org/apache/beam/examples/WordCountIT.java | 14 +- .../examples/cookbook/BigQueryTornadoesIT.java | 4 +- examples/java8/pom.xml | 8 +- .../beam/examples/MinimalWordCountJava8.java | 7 +- .../beam/examples/complete/game/GameStats.java | 18 +- .../examples/complete/game/HourlyTeamScore.java | 2 +- .../examples/complete/game/LeaderBoard.java | 6 +- .../beam/examples/complete/game/UserScore.java | 10 +- .../complete/game/utils/WriteToBigQuery.java | 2 +- .../examples/MinimalWordCountJava8Test.java | 6 +- pom.xml | 83 +- runners/apex/README.md | 2 +- runners/apex/pom.xml | 6 - .../beam/runners/apex/ApexRunnerResult.java | 9 - .../operators/ApexGroupByKeyOperator.java | 191 +- .../operators/ApexParDoOperator.java | 155 +- .../operators/ApexTimerInternals.java | 266 ++ .../translation/utils/ApexStateInternals.java | 100 +- .../apex/translation/utils/ApexStreamTuple.java | 6 +- .../apex/translation/utils/NoOpStepContext.java | 2 +- .../translation/utils/StateInternalsProxy.java | 6 +- .../runners/apex/examples/WordCountTest.java | 13 +- .../translation/GroupByKeyTranslatorTest.java | 10 +- .../translation/ReadUnboundTranslatorTest.java | 6 +- .../operators/ApexTimerInternalsTest.java | 145 + .../utils/ApexStateInternalsTest.java | 39 +- runners/core-construction-java/pom.xml | 10 - .../core/construction/CoderTranslator.java | 44 + .../core/construction/CoderTranslators.java | 107 + .../beam/runners/core/construction/Coders.java | 109 +- .../core/construction/ForwardingPTransform.java | 5 +- .../core/construction/PTransformMatchers.java | 6 +- .../UnboundedReadFromBoundedSource.java | 19 +- .../core/construction/WindowingStrategies.java | 52 +- .../runners/core/construction/CodersTest.java | 37 +- .../EmptyFlattenAsCreateFactoryTest.java | 10 +- .../construction/ForwardingPTransformTest.java | 7 +- .../core/construction/PCollectionsTest.java | 6 +- .../construction/PTransformMatchersTest.java | 22 +- .../PTransformReplacementsTest.java | 4 +- .../core/construction/PTransformsTest.java | 17 +- .../core/construction/SdkComponentsTest.java | 16 +- .../UnboundedReadFromBoundedSourceTest.java | 14 +- .../construction/WindowingStrategiesTest.java | 6 +- runners/core-java/pom.xml | 15 - .../beam/runners/core/AggregatorFactory.java | 38 - .../beam/runners/core/BaseExecutionContext.java | 2 +- .../apache/beam/runners/core/DoFnAdapters.java | 18 - .../apache/beam/runners/core/DoFnRunners.java | 44 +- .../beam/runners/core/ExecutionContext.java | 2 +- .../runners/core/GlobalCombineFnRunner.java | 78 + .../runners/core/GlobalCombineFnRunners.java | 193 ++ .../GroupAlsoByWindowViaOutputBufferDoFn.java | 3 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 15 +- .../GroupAlsoByWindowViaWindowSetNewDoFn.java | 14 +- .../runners/core/GroupAlsoByWindowsDoFn.java | 7 - .../runners/core/InMemoryStateInternals.java | 112 +- .../beam/runners/core/KeyedWorkItemCoder.java | 22 +- .../core/LateDataDroppingDoFnRunner.java | 20 +- .../apache/beam/runners/core/LateDataUtils.java | 6 +- .../runners/core/MergingActiveWindowSet.java | 4 +- .../beam/runners/core/MergingStateAccessor.java | 2 +- .../apache/beam/runners/core/NonEmptyPanes.java | 2 +- .../org/apache/beam/runners/core/OldDoFn.java | 131 - ...eBoundedSplittableProcessElementInvoker.java | 8 - .../beam/runners/core/PaneInfoTracker.java | 2 +- .../runners/core/PerKeyCombineFnRunner.java | 79 - .../runners/core/PerKeyCombineFnRunners.java | 161 - .../runners/core/ReduceFnContextFactory.java | 37 +- .../beam/runners/core/ReduceFnRunner.java | 27 +- .../beam/runners/core/SideInputHandler.java | 14 +- .../beam/runners/core/SimpleDoFnRunner.java | 37 +- .../beam/runners/core/SimpleOldDoFnRunner.java | 24 +- .../beam/runners/core/SplittableParDo.java | 22 +- .../apache/beam/runners/core/StateAccessor.java | 2 +- .../beam/runners/core/StateInternals.java | 8 +- .../runners/core/StateInternalsFactory.java | 2 +- .../apache/beam/runners/core/StateMerging.java | 44 +- .../beam/runners/core/StateNamespaces.java | 13 +- .../apache/beam/runners/core/StateTable.java | 10 +- .../org/apache/beam/runners/core/StateTag.java | 51 +- .../org/apache/beam/runners/core/StateTags.java | 127 +- .../beam/runners/core/StatefulDoFnRunner.java | 18 +- .../beam/runners/core/SystemReduceFn.java | 21 +- .../core/TestInMemoryStateInternals.java | 8 +- .../beam/runners/core/TimerInternals.java | 18 +- .../apache/beam/runners/core/WatermarkHold.java | 45 +- .../beam/runners/core/WindowingInternals.java | 2 +- .../AfterDelayFromFirstElementStateMachine.java | 2 +- .../core/triggers/AfterPaneStateMachine.java | 2 +- .../TriggerStateMachineContextFactory.java | 12 +- .../triggers/TriggerStateMachineRunner.java | 4 +- .../core/DoFnDelegatingAggregatorTest.java | 144 - .../core/GroupAlsoByWindowsProperties.java | 37 +- .../core/InMemoryStateInternalsTest.java | 44 +- .../core/LateDataDroppingDoFnRunnerTest.java | 47 +- .../core/MergingActiveWindowSetTest.java | 2 +- .../apache/beam/runners/core/NoOpOldDoFn.java | 7 - .../beam/runners/core/OldDoFnContextTest.java | 72 - .../apache/beam/runners/core/OldDoFnTest.java | 141 - .../beam/runners/core/ReduceFnRunnerTest.java | 119 +- .../beam/runners/core/ReduceFnTester.java | 96 +- .../beam/runners/core/SimpleDoFnRunnerTest.java | 9 - .../runners/core/SimpleOldDoFnRunnerTest.java | 2 +- .../beam/runners/core/SplittableParDoTest.java | 2 +- .../apache/beam/runners/core/StateTagTest.java | 83 +- .../runners/core/StatefulDoFnRunnerTest.java | 59 +- runners/direct-java/pom.xml | 10 - .../runners/direct/AggregatorContainer.java | 200 -- .../CopyOnAccessInMemoryStateInternals.java | 198 +- .../runners/direct/DirectExecutionContext.java | 15 +- .../beam/runners/direct/DirectRunner.java | 45 +- .../beam/runners/direct/EvaluationContext.java | 31 +- .../GroupAlsoByWindowEvaluatorFactory.java | 30 +- .../beam/runners/direct/ParDoEvaluator.java | 16 +- .../direct/ParDoMultiOverrideFactory.java | 6 +- ...littableProcessElementsEvaluatorFactory.java | 4 +- .../direct/StatefulParDoEvaluatorFactory.java | 6 +- .../runners/direct/StepTransformResult.java | 12 +- .../beam/runners/direct/TransformResult.java | 8 +- .../direct/WriteWithShardingFactory.java | 15 +- .../runners/direct/AggregatorContainerTest.java | 137 - .../direct/CloningBundleFactoryTest.java | 10 +- .../CopyOnAccessInMemoryStateInternalsTest.java | 180 +- .../runners/direct/DirectGraphVisitorTest.java | 6 +- .../beam/runners/direct/DirectRunnerTest.java | 15 +- .../runners/direct/EvaluationContextTest.java | 49 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 5 - .../StatefulParDoEvaluatorFactoryTest.java | 10 +- .../UnboundedReadEvaluatorFactoryTest.java | 4 +- .../direct/WriteWithShardingFactoryTest.java | 20 +- runners/flink/pom.xml | 15 +- .../flink/FlinkBatchTransformTranslators.java | 94 +- .../flink/FlinkBatchTranslationContext.java | 4 + .../flink/FlinkDetachedRunnerResult.java | 11 - .../FlinkPipelineExecutionEnvironment.java | 8 + .../runners/flink/FlinkPipelineOptions.java | 18 + .../apache/beam/runners/flink/FlinkRunner.java | 3 +- .../beam/runners/flink/FlinkRunnerResult.java | 37 +- .../flink/FlinkStreamingPipelineTranslator.java | 3 + .../FlinkStreamingTransformTranslators.java | 70 +- .../flink/FlinkStreamingTranslationContext.java | 3 + .../metrics/DoFnRunnerWithMetricsUpdate.java | 91 + .../flink/metrics/FlinkMetricContainer.java | 315 ++ .../flink/metrics/FlinkMetricResults.java | 146 + .../flink/metrics/ReaderInvocationUtil.java | 71 + .../runners/flink/metrics/package-info.java | 22 + .../functions/AbstractFlinkCombineRunner.java | 182 ++ .../functions/FlinkAggregatorFactory.java | 53 - .../functions/FlinkDoFnFunction.java | 12 +- .../FlinkMergingNonShuffleReduceFunction.java | 173 +- .../FlinkMergingPartialReduceFunction.java | 201 -- .../functions/FlinkMergingReduceFunction.java | 199 -- .../functions/FlinkNoOpStepContext.java | 2 +- .../functions/FlinkPartialReduceFunction.java | 118 +- .../functions/FlinkReduceFunction.java | 118 +- .../functions/FlinkStatefulDoFnFunction.java | 14 +- .../functions/HashingFlinkCombineRunner.java | 180 ++ .../functions/SortingFlinkCombineRunner.java | 192 ++ .../SerializableFnAggregatorWrapper.java | 98 - .../translation/wrappers/SourceInputFormat.java | 23 +- .../wrappers/streaming/DoFnOperator.java | 87 +- .../streaming/SingletonKeyedWorkItemCoder.java | 22 +- .../streaming/SplittableDoFnOperator.java | 6 +- .../wrappers/streaming/WindowDoFnOperator.java | 6 +- .../streaming/io/BoundedSourceWrapper.java | 17 +- .../streaming/io/UnboundedSourceWrapper.java | 18 +- .../state/FlinkBroadcastStateInternals.java | 202 +- .../state/FlinkKeyGroupStateInternals.java | 130 +- .../state/FlinkSplitStateInternals.java | 130 +- .../streaming/state/FlinkStateInternals.java | 225 +- .../beam/runners/flink/FlinkTestPipeline.java | 9 +- .../beam/runners/flink/PipelineOptionsTest.java | 19 + .../beam/runners/flink/ReadSourceITCase.java | 6 +- .../flink/ReadSourceStreamingITCase.java | 6 +- .../beam/runners/flink/WriteSinkITCase.java | 192 -- .../flink/streaming/DoFnOperatorTest.java | 9 +- .../FlinkBroadcastStateInternalsTest.java | 6 +- .../FlinkKeyGroupStateInternalsTest.java | 2 +- .../streaming/FlinkSplitStateInternalsTest.java | 2 +- .../streaming/FlinkStateInternalsTest.java | 40 +- .../flink/streaming/GroupByNullKeyTest.java | 13 +- .../streaming/TopWikipediaSessionsITCase.java | 2 +- .../streaming/UnboundedSourceWrapperTest.java | 12 +- runners/google-cloud-dataflow-java/pom.xml | 31 +- .../beam/runners/dataflow/AssignWindows.java | 5 +- .../runners/dataflow/BatchViewOverrides.java | 21 +- .../dataflow/DataflowAggregatorTransforms.java | 79 - .../dataflow/DataflowMetricUpdateExtractor.java | 109 - .../beam/runners/dataflow/DataflowMetrics.java | 44 +- .../runners/dataflow/DataflowPipelineJob.java | 103 +- .../dataflow/DataflowPipelineTranslator.java | 13 +- .../beam/runners/dataflow/DataflowRunner.java | 164 +- .../runners/dataflow/internal/IsmFormat.java | 92 +- .../options/DataflowPipelineOptions.java | 8 +- .../dataflow/testing/TestDataflowRunner.java | 181 +- .../util/AvroCoderCloudObjectTranslator.java | 62 + .../runners/dataflow/util/CloudKnownType.java | 138 + .../beam/runners/dataflow/util/CloudObject.java | 185 ++ .../runners/dataflow/util/CloudObjectKinds.java | 33 + .../dataflow/util/CloudObjectTranslator.java | 48 + .../dataflow/util/CloudObjectTranslators.java | 575 ++++ .../runners/dataflow/util/CloudObjects.java | 92 + .../CoderCloudObjectTranslatorRegistrar.java | 47 + .../dataflow/util/DataflowTemplateJob.java | 2 +- .../dataflow/util/DataflowTransport.java | 2 +- ...aultCoderCloudObjectTranslatorRegistrar.java | 136 + .../runners/dataflow/util/MonitoringUtil.java | 4 +- .../runners/dataflow/util/RandomAccessData.java | 9 +- .../SerializableCoderCloudObjectTranslator.java | 65 + .../beam/runners/dataflow/util/Serializer.java | 262 ++ .../BatchStatefulParDoOverridesTest.java | 4 +- .../runners/dataflow/DataflowMetricsTest.java | 2 +- .../dataflow/DataflowPipelineJobTest.java | 490 +--- .../DataflowPipelineTranslatorTest.java | 120 +- .../runners/dataflow/DataflowRunnerTest.java | 43 +- .../testing/TestDataflowRunnerTest.java | 374 +-- .../runners/dataflow/util/CloudObjectsTest.java | 217 ++ .../dataflow/util/MonitoringUtilTest.java | 2 +- .../runners/dataflow/util/PackageUtilTest.java | 2 +- runners/spark/pom.xml | 16 +- .../spark/SparkNativePipelineVisitor.java | 2 +- .../beam/runners/spark/SparkPipelineResult.java | 12 - .../beam/runners/spark/TestSparkRunner.java | 35 +- .../spark/aggregators/SparkAggregators.java | 110 - .../beam/runners/spark/examples/WordCount.java | 17 +- .../beam/runners/spark/io/MicrobatchSource.java | 188 +- .../beam/runners/spark/io/SourceDStream.java | 57 +- .../apache/beam/runners/spark/io/SourceRDD.java | 11 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 58 +- .../spark/stateful/SparkStateInternals.java | 111 +- .../spark/stateful/StateSpecFunctions.java | 30 +- .../spark/translation/MultiDoFnFunction.java | 2 - .../translation/SparkAbstractCombineFn.java | 4 +- .../spark/translation/SparkGlobalCombineFn.java | 37 +- ...SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +- .../spark/translation/SparkKeyedCombineFn.java | 63 +- .../spark/translation/SparkProcessContext.java | 2 +- .../spark/translation/SparkRuntimeContext.java | 84 +- .../spark/translation/TransformTranslator.java | 44 +- .../spark/translation/TranslationUtils.java | 2 +- .../streaming/StreamingTransformTranslator.java | 100 +- .../runners/spark/SparkRunnerDebuggerTest.java | 21 +- .../spark/aggregators/ClearAggregatorsRule.java | 38 - .../metrics/sink/NamedAggregatorsTest.java | 101 - .../metrics/sink/SparkMetricsSinkTest.java | 86 + .../beam/runners/spark/io/AvroPipelineTest.java | 4 +- .../beam/runners/spark/io/NumShardsTest.java | 3 +- .../translation/streaming/CreateStreamTest.java | 50 + .../ResumeFromCheckpointStreamingTest.java | 94 +- .../streaming/StreamingSourceMetricsTest.java | 19 +- .../src/main/resources/beam/findbugs-filter.xml | 10 +- sdks/java/core/pom.xml | 88 +- .../beam/sdk/AggregatorPipelineExtractor.java | 92 - .../beam/sdk/AggregatorRetrievalException.java | 33 - .../org/apache/beam/sdk/AggregatorValues.java | 51 - .../main/java/org/apache/beam/sdk/Pipeline.java | 152 +- .../org/apache/beam/sdk/PipelineResult.java | 12 - .../beam/sdk/annotations/Experimental.java | 3 - .../apache/beam/sdk/annotations/Internal.java | 42 + .../org/apache/beam/sdk/coders/AtomicCoder.java | 52 - .../org/apache/beam/sdk/coders/AvroCoder.java | 152 +- .../apache/beam/sdk/coders/BigDecimalCoder.java | 26 +- .../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +- .../beam/sdk/coders/BigEndianLongCoder.java | 5 +- .../apache/beam/sdk/coders/BigIntegerCoder.java | 18 +- .../org/apache/beam/sdk/coders/BitSetCoder.java | 63 + .../apache/beam/sdk/coders/ByteArrayCoder.java | 18 +- .../org/apache/beam/sdk/coders/ByteCoder.java | 4 +- .../apache/beam/sdk/coders/ByteStringCoder.java | 113 - .../java/org/apache/beam/sdk/coders/Coder.java | 42 +- .../apache/beam/sdk/coders/CoderFactories.java | 22 +- .../apache/beam/sdk/coders/CoderRegistrar.java | 45 + .../apache/beam/sdk/coders/CoderRegistry.java | 128 +- .../org/apache/beam/sdk/coders/CustomCoder.java | 80 +- .../apache/beam/sdk/coders/DelegateCoder.java | 43 +- .../sdk/coders/DeterministicStandardCoder.java | 39 - .../org/apache/beam/sdk/coders/DoubleCoder.java | 4 +- .../apache/beam/sdk/coders/DurationCoder.java | 19 +- .../apache/beam/sdk/coders/InstantCoder.java | 20 +- .../apache/beam/sdk/coders/IterableCoder.java | 9 - .../beam/sdk/coders/IterableLikeCoder.java | 2 +- .../org/apache/beam/sdk/coders/JAXBCoder.java | 201 -- .../org/apache/beam/sdk/coders/KvCoder.java | 13 +- .../beam/sdk/coders/LengthPrefixCoder.java | 25 +- .../org/apache/beam/sdk/coders/MapCoder.java | 15 +- .../apache/beam/sdk/coders/NullableCoder.java | 29 +- .../beam/sdk/coders/SerializableCoder.java | 34 +- .../apache/beam/sdk/coders/StandardCoder.java | 265 -- .../beam/sdk/coders/StringDelegateCoder.java | 21 +- .../apache/beam/sdk/coders/StringUtf8Coder.java | 9 +- .../apache/beam/sdk/coders/StructuredCoder.java | 188 ++ .../beam/sdk/coders/TableRowJsonCoder.java | 88 - .../beam/sdk/coders/TextualIntegerCoder.java | 9 +- .../org/apache/beam/sdk/coders/VarIntCoder.java | 7 +- .../apache/beam/sdk/coders/VarLongCoder.java | 22 +- .../org/apache/beam/sdk/coders/VoidCoder.java | 5 +- .../beam/sdk/coders/protobuf/ProtoCoder.java | 405 --- .../beam/sdk/coders/protobuf/ProtobufUtil.java | 171 -- .../beam/sdk/coders/protobuf/package-info.java | 24 - .../java/org/apache/beam/sdk/io/AvroIO.java | 1272 ++------ .../java/org/apache/beam/sdk/io/AvroSink.java | 142 + .../java/org/apache/beam/sdk/io/AvroSource.java | 153 +- .../apache/beam/sdk/io/BlockBasedSource.java | 10 +- .../org/apache/beam/sdk/io/BoundedSource.java | 9 - .../apache/beam/sdk/io/CompressedSource.java | 26 +- .../org/apache/beam/sdk/io/CountingInput.java | 283 -- .../org/apache/beam/sdk/io/CountingSource.java | 53 +- .../org/apache/beam/sdk/io/FileBasedSink.java | 187 +- .../org/apache/beam/sdk/io/FileBasedSource.java | 359 +-- .../java/org/apache/beam/sdk/io/FileSystem.java | 19 + .../apache/beam/sdk/io/FileSystemRegistrar.java | 15 +- .../org/apache/beam/sdk/io/FileSystems.java | 194 +- .../apache/beam/sdk/io/GenerateSequence.java | 194 ++ .../org/apache/beam/sdk/io/LocalFileSystem.java | 39 +- .../beam/sdk/io/LocalFileSystemRegistrar.java | 15 +- .../org/apache/beam/sdk/io/LocalResourceId.java | 45 +- .../org/apache/beam/sdk/io/LocalResources.java | 56 + .../apache/beam/sdk/io/ShardNameTemplate.java | 2 +- .../main/java/org/apache/beam/sdk/io/Sink.java | 295 -- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 42 +- .../java/org/apache/beam/sdk/io/TextIO.java | 1218 ++------ .../java/org/apache/beam/sdk/io/TextSink.java | 139 + .../java/org/apache/beam/sdk/io/TextSource.java | 236 ++ .../main/java/org/apache/beam/sdk/io/Write.java | 603 ---- .../java/org/apache/beam/sdk/io/WriteFiles.java | 558 ++++ .../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 --- .../java/org/apache/beam/sdk/io/XmlSink.java | 153 - .../java/org/apache/beam/sdk/io/XmlSource.java | 402 --- .../org/apache/beam/sdk/io/fs/MatchResult.java | 66 +- .../org/apache/beam/sdk/io/fs/ResourceId.java | 36 +- .../org/apache/beam/sdk/io/package-info.java | 2 +- .../org/apache/beam/sdk/io/range/ByteKey.java | 21 +- .../apache/beam/sdk/metrics/CounterCell.java | 19 +- .../beam/sdk/metrics/DistributionCell.java | 10 +- .../org/apache/beam/sdk/metrics/GaugeCell.java | 8 +- .../org/apache/beam/sdk/metrics/Metric.java | 8 +- .../org/apache/beam/sdk/metrics/MetricCell.java | 8 +- .../apache/beam/sdk/metrics/MetricResults.java | 29 +- .../org/apache/beam/sdk/metrics/Metrics.java | 39 + .../beam/sdk/metrics/MetricsContainer.java | 8 +- .../apache/beam/sdk/metrics/MetricsFilter.java | 9 +- .../apache/beam/sdk/metrics/SinkMetrics.java | 49 + .../apache/beam/sdk/metrics/SourceMetrics.java | 116 + .../beam/sdk/options/PipelineOptions.java | 8 +- .../sdk/options/PipelineOptionsFactory.java | 4 +- .../sdk/options/ProxyInvocationHandler.java | 16 +- .../beam/sdk/testing/CoderProperties.java | 47 +- .../org/apache/beam/sdk/testing/PAssert.java | 164 +- .../beam/sdk/testing/SourceTestUtils.java | 5 +- .../beam/sdk/testing/SuccessOrFailure.java | 82 + .../apache/beam/sdk/testing/TestPipeline.java | 13 +- .../org/apache/beam/sdk/testing/TestStream.java | 116 - .../beam/sdk/testing/UsesCounterMetrics.java | 25 + .../sdk/testing/UsesDistributionMetrics.java | 26 + .../beam/sdk/testing/UsesGaugeMetrics.java | 25 + .../beam/sdk/testing/WindowFnTestUtils.java | 53 +- .../apache/beam/sdk/transforms/Aggregator.java | 38 +- .../sdk/transforms/AggregatorRetriever.java | 45 - .../sdk/transforms/ApproximateQuantiles.java | 8 +- .../beam/sdk/transforms/ApproximateUnique.java | 3 +- .../org/apache/beam/sdk/transforms/Combine.java | 686 ++--- .../beam/sdk/transforms/CombineFnBase.java | 136 - .../apache/beam/sdk/transforms/CombineFns.java | 463 +-- .../beam/sdk/transforms/CombineWithContext.java | 174 +- .../org/apache/beam/sdk/transforms/Count.java | 35 +- .../sdk/transforms/DelegatingAggregator.java | 126 - .../org/apache/beam/sdk/transforms/DoFn.java | 128 +- .../apache/beam/sdk/transforms/DoFnTester.java | 78 - .../org/apache/beam/sdk/transforms/Filter.java | 35 +- .../apache/beam/sdk/transforms/GroupByKey.java | 30 +- .../beam/sdk/transforms/GroupIntoBatches.java | 8 +- .../org/apache/beam/sdk/transforms/Latest.java | 5 +- .../org/apache/beam/sdk/transforms/Mean.java | 10 +- .../apache/beam/sdk/transforms/PTransform.java | 8 +- .../org/apache/beam/sdk/transforms/ParDo.java | 2 +- .../org/apache/beam/sdk/transforms/Top.java | 6 +- .../org/apache/beam/sdk/transforms/View.java | 27 +- .../beam/sdk/transforms/join/CoGbkResult.java | 33 +- .../sdk/transforms/join/CoGbkResultSchema.java | 21 +- .../beam/sdk/transforms/join/UnionCoder.java | 18 +- .../sdk/transforms/reflect/DoFnSignatures.java | 27 +- .../transforms/windowing/AfterWatermark.java | 3 - .../sdk/transforms/windowing/GlobalWindow.java | 25 +- .../transforms/windowing/IntervalWindow.java | 31 +- .../sdk/transforms/windowing/OutputTimeFn.java | 314 -- .../sdk/transforms/windowing/OutputTimeFns.java | 212 -- .../beam/sdk/transforms/windowing/PaneInfo.java | 9 +- .../transforms/windowing/TimestampCombiner.java | 186 ++ .../beam/sdk/transforms/windowing/Trigger.java | 12 +- .../beam/sdk/transforms/windowing/Window.java | 27 +- .../org/apache/beam/sdk/util/ApiSurface.java | 4 +- .../apache/beam/sdk/util/AppliedCombineFn.java | 35 +- .../org/apache/beam/sdk/util/AvroUtils.java | 137 - .../org/apache/beam/sdk/util/BitSetCoder.java | 15 +- .../org/apache/beam/sdk/util/ClassPath.java | 544 ++++ .../apache/beam/sdk/util/CloudKnownType.java | 9 +- .../org/apache/beam/sdk/util/CloudObject.java | 3 + .../org/apache/beam/sdk/util/CoderUtils.java | 15 +- .../org/apache/beam/sdk/util/CombineFnUtil.java | 123 +- .../beam/sdk/util/ExplicitShardedFile.java | 23 +- .../apache/beam/sdk/util/IOChannelUtils.java | 50 +- .../beam/sdk/util/NumberedShardedFile.java | 29 +- .../apache/beam/sdk/util/PCollectionViews.java | 4 +- .../org/apache/beam/sdk/util/PropertyNames.java | 4 +- .../org/apache/beam/sdk/util/Reshuffle.java | 7 +- .../sdk/util/RetryHttpRequestInitializer.java | 248 -- .../apache/beam/sdk/util/SerializableUtils.java | 26 +- .../org/apache/beam/sdk/util/Serializer.java | 3 + .../java/org/apache/beam/sdk/util/Structs.java | 74 +- .../apache/beam/sdk/util/ValueWithRecordId.java | 16 +- .../org/apache/beam/sdk/util/WindowedValue.java | 19 +- .../apache/beam/sdk/util/WindowingStrategy.java | 178 +- .../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 626 ---- .../beam/sdk/util/gcsfs/package-info.java | 20 - .../apache/beam/sdk/util/state/StateBinder.java | 53 +- .../apache/beam/sdk/util/state/StateSpec.java | 15 +- .../apache/beam/sdk/util/state/StateSpecs.java | 278 +- .../beam/sdk/util/state/WatermarkHoldState.java | 19 +- .../org/apache/beam/sdk/values/PCollection.java | 9 +- .../java/org/apache/beam/sdk/values/PDone.java | 33 +- .../beam/sdk/values/POutputValueBase.java | 74 - .../java/org/apache/beam/sdk/values/PValue.java | 2 + .../org/apache/beam/sdk/values/PValueBase.java | 29 +- .../beam/sdk/values/TimestampedValue.java | 22 +- .../beam/sdk/values/ValueInSingleWindow.java | 23 +- sdks/java/core/src/main/proto/README.md | 45 - .../main/proto/proto2_coder_test_messages.proto | 53 - .../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 - .../sdk/AggregatorPipelineExtractorTest.java | 226 -- .../java/org/apache/beam/sdk/PipelineTest.java | 97 +- .../apache/beam/sdk/coders/AvroCoderTest.java | 86 +- .../beam/sdk/coders/AvroCoderTestPojo.java | 51 + .../beam/sdk/coders/BigDecimalCoderTest.java | 18 - .../sdk/coders/BigEndianIntegerCoderTest.java | 8 - .../beam/sdk/coders/BigEndianLongCoderTest.java | 8 - .../beam/sdk/coders/BigIntegerCoderTest.java | 18 - .../apache/beam/sdk/coders/BitSetCoderTest.java | 103 + .../beam/sdk/coders/ByteArrayCoderTest.java | 8 - .../apache/beam/sdk/coders/ByteCoderTest.java | 8 - .../beam/sdk/coders/ByteStringCoderTest.java | 128 - .../beam/sdk/coders/CoderRegistryTest.java | 118 +- .../beam/sdk/coders/CollectionCoderTest.java | 8 - .../apache/beam/sdk/coders/CustomCoderTest.java | 46 - .../beam/sdk/coders/DefaultCoderTest.java | 8 +- .../beam/sdk/coders/DelegateCoderTest.java | 38 +- .../apache/beam/sdk/coders/DoubleCoderTest.java | 8 - .../beam/sdk/coders/InstantCoderTest.java | 8 - .../beam/sdk/coders/IterableCoderTest.java | 18 - .../apache/beam/sdk/coders/JAXBCoderTest.java | 223 -- .../org/apache/beam/sdk/coders/KvCoderTest.java | 20 - .../beam/sdk/coders/LengthPrefixCoderTest.java | 18 +- .../apache/beam/sdk/coders/ListCoderTest.java | 8 - .../apache/beam/sdk/coders/MapCoderTest.java | 8 - .../beam/sdk/coders/NullableCoderTest.java | 13 +- .../beam/sdk/coders/SerializableCoderTest.java | 19 +- .../apache/beam/sdk/coders/SetCoderTest.java | 8 - .../beam/sdk/coders/StandardCoderTest.java | 238 -- .../sdk/coders/StringDelegateCoderTest.java | 6 - .../beam/sdk/coders/StructuredCoderTest.java | 238 ++ .../beam/sdk/coders/TableRowJsonCoderTest.java | 94 - .../sdk/coders/TextualIntegerCoderTest.java | 8 - .../apache/beam/sdk/coders/VarIntCoderTest.java | 8 - .../beam/sdk/coders/VarLongCoderTest.java | 8 - .../sdk/coders/protobuf/ProtoCoderTest.java | 182 -- .../sdk/coders/protobuf/ProtobufUtilTest.java | 192 -- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 156 +- .../apache/beam/sdk/io/AvroIOTransformTest.java | 30 +- .../org/apache/beam/sdk/io/AvroSourceTest.java | 56 +- .../beam/sdk/io/CompressedSourceTest.java | 12 +- .../apache/beam/sdk/io/CountingInputTest.java | 221 -- .../apache/beam/sdk/io/CountingSourceTest.java | 4 +- .../apache/beam/sdk/io/FileBasedSinkTest.java | 102 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 192 +- .../org/apache/beam/sdk/io/FileSystemsTest.java | 20 +- .../beam/sdk/io/GenerateSequenceTest.java | 194 ++ .../sdk/io/LocalFileSystemRegistrarTest.java | 6 + .../apache/beam/sdk/io/LocalFileSystemTest.java | 91 +- .../apache/beam/sdk/io/LocalResourceIdTest.java | 45 + .../java/org/apache/beam/sdk/io/SimpleSink.java | 96 + .../org/apache/beam/sdk/io/TFRecordIOTest.java | 6 +- .../java/org/apache/beam/sdk/io/TextIOTest.java | 106 +- .../org/apache/beam/sdk/io/WriteFilesTest.java | 457 +++ .../java/org/apache/beam/sdk/io/WriteTest.java | 705 ----- .../org/apache/beam/sdk/io/XmlSinkTest.java | 253 -- .../org/apache/beam/sdk/io/XmlSourceTest.java | 892 ------ .../range/ByteKeyRangeInterpolateKeyTest.java | 1 - .../beam/sdk/io/range/ByteKeyRangeTest.java | 16 +- .../apache/beam/sdk/io/range/ByteKeyTest.java | 4 +- .../apache/beam/sdk/metrics/MetricMatchers.java | 144 +- .../apache/beam/sdk/metrics/MetricsTest.java | 183 +- .../sdk/options/PipelineOptionsFactoryTest.java | 71 +- .../beam/sdk/options/PipelineOptionsTest.java | 14 + .../sdk/options/ProxyInvocationHandlerTest.java | 2 +- .../sdk/runners/TransformHierarchyTest.java | 7 +- .../beam/sdk/runners/TransformTreeTest.java | 12 +- .../beam/sdk/testing/GatherAllPanesTest.java | 8 +- .../apache/beam/sdk/testing/PAssertTest.java | 71 +- .../sdk/testing/SerializableMatchersTest.java | 4 +- .../beam/sdk/testing/SystemNanoTimeSleeper.java | 1 - .../apache/beam/sdk/testing/TestStreamTest.java | 24 - .../beam/sdk/testing/WindowSupplierTest.java | 4 +- .../beam/sdk/transforms/CombineFnsTest.java | 118 +- .../apache/beam/sdk/transforms/CombineTest.java | 218 +- .../apache/beam/sdk/transforms/CreateTest.java | 3 +- .../apache/beam/sdk/transforms/DoFnTest.java | 162 -- .../beam/sdk/transforms/DoFnTesterTest.java | 47 +- .../apache/beam/sdk/transforms/FilterTest.java | 13 + .../apache/beam/sdk/transforms/FlattenTest.java | 8 +- .../beam/sdk/transforms/GroupByKeyTest.java | 19 +- .../beam/sdk/transforms/LatestFnTest.java | 42 +- .../apache/beam/sdk/transforms/ParDoTest.java | 189 +- .../org/apache/beam/sdk/transforms/SumTest.java | 2 +- .../apache/beam/sdk/transforms/ViewTest.java | 2 +- .../display/DisplayDataEvaluatorTest.java | 5 +- .../sdk/transforms/join/CoGroupByKeyTest.java | 6 +- .../sdk/transforms/join/UnionCoderTest.java | 11 + .../transforms/reflect/DoFnInvokersTest.java | 6 +- .../transforms/reflect/DoFnSignaturesTest.java | 26 +- .../transforms/windowing/GlobalWindowTest.java | 7 - .../transforms/windowing/OutputTimeFnsTest.java | 51 - .../sdk/transforms/windowing/SessionsTest.java | 6 +- .../sdk/transforms/windowing/WindowTest.java | 27 +- .../sdk/transforms/windowing/WindowingTest.java | 4 +- .../org/apache/beam/sdk/util/AvroUtilsTest.java | 147 - .../apache/beam/sdk/util/CoderUtilsTest.java | 108 +- .../apache/beam/sdk/util/CombineFnUtilTest.java | 18 +- .../beam/sdk/util/IOChannelUtilsTest.java | 30 - .../org/apache/beam/sdk/util/NameUtilsTest.java | 2 +- .../beam/sdk/util/NumberedShardedFileTest.java | 41 +- .../beam/sdk/util/SerializableUtilsTest.java | 56 +- .../apache/beam/sdk/util/WindowedValueTest.java | 10 - .../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 --- .../beam/sdk/values/PCollectionListTest.java | 25 +- .../beam/sdk/values/PCollectionTupleTest.java | 6 +- .../org/apache/beam/sdk/values/PDoneTest.java | 2 +- sdks/java/extensions/gcp-core/pom.xml | 222 -- .../beam/sdk/options/BigQueryOptions.java | 32 - .../options/CloudResourceManagerOptions.java | 40 - .../org/apache/beam/sdk/options/GcpOptions.java | 227 -- .../options/GcpPipelineOptionsRegistrar.java | 39 - .../org/apache/beam/sdk/options/GcsOptions.java | 154 - .../beam/sdk/options/GoogleApiDebugOptions.java | 87 - .../apache/beam/sdk/options/PubsubOptions.java | 36 - .../apache/beam/sdk/options/package-info.java | 22 - .../beam/sdk/testing/BigqueryMatcher.java | 256 -- .../apache/beam/sdk/testing/package-info.java | 21 - .../apache/beam/sdk/util/CredentialFactory.java | 29 - .../org/apache/beam/sdk/util/DefaultBucket.java | 105 - .../beam/sdk/util/GcpCredentialFactory.java | 67 - .../apache/beam/sdk/util/GcpProjectUtil.java | 106 - .../beam/sdk/util/GcsIOChannelFactory.java | 111 - .../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 - .../apache/beam/sdk/util/GcsPathValidator.java | 95 - .../java/org/apache/beam/sdk/util/GcsUtil.java | 798 ----- .../beam/sdk/util/NoopCredentialFactory.java | 68 - .../sdk/util/NullCredentialInitializer.java | 62 - .../apache/beam/sdk/util/TestCredential.java | 59 - .../org/apache/beam/sdk/util/Transport.java | 178 -- .../org/apache/beam/sdk/util/package-info.java | 20 - .../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 - .../apache/beam/sdk/options/GcpOptionsTest.java | 171 -- .../sdk/options/GoogleApiDebugOptionsTest.java | 145 - .../beam/sdk/testing/BigqueryMatcherTest.java | 176 -- .../apache/beam/sdk/util/DefaultBucketTest.java | 112 - .../beam/sdk/util/GcpProjectUtilTest.java | 76 - .../util/GcsIOChannelFactoryRegistrarTest.java | 44 - .../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 - .../beam/sdk/util/GcsPathValidatorTest.java | 87 - .../org/apache/beam/sdk/util/GcsUtilTest.java | 798 ----- .../util/RetryHttpRequestInitializerTest.java | 290 -- .../google-cloud-platform-core/pom.xml | 185 ++ .../extensions/gcp/auth/CredentialFactory.java | 29 + .../gcp/auth/GcpCredentialFactory.java | 67 + .../gcp/auth/NoopCredentialFactory.java | 68 + .../gcp/auth/NullCredentialInitializer.java | 62 + .../sdk/extensions/gcp/auth/package-info.java | 22 + .../options/CloudResourceManagerOptions.java | 32 + .../sdk/extensions/gcp/options/GcpOptions.java | 389 +++ .../options/GcpPipelineOptionsRegistrar.java | 39 + .../sdk/extensions/gcp/options/GcsOptions.java | 160 + .../gcp/options/GoogleApiDebugOptions.java | 89 + .../extensions/gcp/options/package-info.java | 22 + .../extensions/gcp/storage/GcsFileSystem.java | 266 ++ .../gcp/storage/GcsFileSystemRegistrar.java | 43 + .../extensions/gcp/storage/GcsResourceId.java | 128 + .../extensions/gcp/storage/package-info.java | 21 + .../beam/sdk/util/GcsIOChannelFactory.java | 111 + .../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 + .../apache/beam/sdk/util/GcsPathValidator.java | 98 + .../java/org/apache/beam/sdk/util/GcsUtil.java | 796 +++++ .../sdk/util/RetryHttpRequestInitializer.java | 192 ++ .../org/apache/beam/sdk/util/Transport.java | 122 + .../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 627 ++++ .../beam/sdk/util/gcsfs/package-info.java | 20 + .../org/apache/beam/sdk/util/package-info.java | 20 + .../extensions/gcp/GcpCoreApiSurfaceTest.java | 58 + .../sdk/extensions/gcp/auth/TestCredential.java | 59 + .../extensions/gcp/options/GcpOptionsTest.java | 273 ++ .../gcp/options/GoogleApiDebugOptionsTest.java | 147 + .../gcp/storage/GcsFileSystemRegistrarTest.java | 52 + .../gcp/storage/GcsFileSystemTest.java | 274 ++ .../gcp/storage/GcsResourceIdTest.java | 169 ++ .../util/GcsIOChannelFactoryRegistrarTest.java | 44 + .../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 + .../beam/sdk/util/GcsPathValidatorTest.java | 106 + .../org/apache/beam/sdk/util/GcsUtilTest.java | 799 +++++ .../util/RetryHttpRequestInitializerTest.java | 281 ++ .../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 +++ sdks/java/extensions/jackson/pom.xml | 99 +- sdks/java/extensions/join-library/pom.xml | 22 - sdks/java/extensions/pom.xml | 3 +- sdks/java/extensions/protobuf/pom.xml | 121 + .../extensions/protobuf/ByteStringCoder.java | 118 + .../sdk/extensions/protobuf/ProtoCoder.java | 332 +++ .../protobuf/ProtobufCoderRegistrar.java | 39 + .../sdk/extensions/protobuf/ProtobufUtil.java | 171 ++ .../sdk/extensions/protobuf/package-info.java | 24 + .../protobuf/ByteStringCoderTest.java | 131 + .../sdk/extensions/protobuf/ProtoCoderTest.java | 169 ++ .../extensions/protobuf/ProtobufUtilTest.java | 191 ++ .../test/proto/proto2_coder_test_messages.proto | 53 + sdks/java/extensions/sorter/pom.xml | 24 +- sdks/java/harness/pom.xml | 12 +- .../org/apache/beam/fn/harness/FnHarness.java | 2 +- .../harness/control/ProcessBundleHandler.java | 2 - .../fn/harness/fake/FakeAggregatorFactory.java | 52 - .../beam/fn/harness/fake/FakeStepContext.java | 2 +- .../fn/harness/logging/BeamFnLoggingClient.java | 2 +- .../harness/stream/StreamObserverFactory.java | 2 +- .../beam/runners/core/BeamFnDataReadRunner.java | 14 +- .../runners/core/BeamFnDataWriteRunner.java | 15 +- .../apache/beam/fn/harness/FnHarnessTest.java | 2 +- .../control/ProcessBundleHandlerTest.java | 28 +- .../runners/core/BeamFnDataReadRunnerTest.java | 12 +- .../runners/core/BeamFnDataWriteRunnerTest.java | 12 +- sdks/java/io/elasticsearch/pom.xml | 1 - .../sdk/io/elasticsearch/ElasticsearchIO.java | 4 +- .../elasticsearch/ElasticSearchIOTestUtils.java | 2 +- sdks/java/io/google-cloud-platform/pom.xml | 19 +- .../beam/sdk/io/gcp/bigquery/BatchLoads.java | 28 +- .../io/gcp/bigquery/BigQueryCoderRegistrar.java | 39 + .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 45 +- .../sdk/io/gcp/bigquery/BigQueryOptions.java | 39 + .../io/gcp/bigquery/BigQueryQuerySource.java | 1 - .../sdk/io/gcp/bigquery/BigQueryServices.java | 1 - .../io/gcp/bigquery/BigQueryServicesImpl.java | 43 +- .../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 4 +- .../io/gcp/bigquery/BigQueryTableSource.java | 1 - .../beam/sdk/io/gcp/bigquery/CreateTables.java | 1 - .../io/gcp/bigquery/PassThroughThenCleanup.java | 4 +- .../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 1 - .../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 17 +- .../sdk/io/gcp/bigquery/StreamingWriteFn.java | 5 +- .../io/gcp/bigquery/StreamingWriteTables.java | 1 - .../io/gcp/bigquery/TableDestinationCoder.java | 6 +- .../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 7 +- .../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 89 + .../sdk/io/gcp/bigquery/TableRowWriter.java | 28 +- .../io/gcp/bigquery/WriteBundlesToFiles.java | 9 +- .../sdk/io/gcp/bigquery/WritePartition.java | 2 +- .../beam/sdk/io/gcp/bigquery/WriteRename.java | 1 - .../beam/sdk/io/gcp/bigquery/WriteResult.java | 37 +- .../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 - .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 28 +- .../io/gcp/bigtable/BigtableServiceImpl.java | 4 +- .../common/GcpIoPipelineOptionsRegistrar.java | 39 + .../beam/sdk/io/gcp/common/package-info.java | 20 + .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 6 +- .../beam/sdk/io/gcp/pubsub/PubsubClient.java | 43 +- .../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 39 +- .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 644 ++-- .../sdk/io/gcp/pubsub/PubsubJsonClient.java | 37 +- .../pubsub/PubsubMessagePayloadOnlyCoder.java | 48 + .../PubsubMessageWithAttributesCoder.java | 57 + .../beam/sdk/io/gcp/pubsub/PubsubOptions.java | 44 + .../sdk/io/gcp/pubsub/PubsubTestClient.java | 7 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 166 +- .../io/gcp/pubsub/PubsubUnboundedSource.java | 211 +- .../beam/sdk/io/gcp/storage/GcsFileSystem.java | 242 -- .../io/gcp/storage/GcsFileSystemRegistrar.java | 49 - .../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 - .../beam/sdk/io/gcp/storage/package-info.java | 21 - .../beam/sdk/io/gcp/testing/package-info.java | 21 + .../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 + .../bigquery/BigQueryCoderRegistrarTest.java | 40 + .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 20 +- .../io/gcp/bigquery/FakeBigQueryServices.java | 2 - .../sdk/io/gcp/bigquery/FakeJobService.java | 1 - .../io/gcp/bigquery/TableRowJsonCoderTest.java | 87 + .../sdk/io/gcp/bigtable/BigtableIOTest.java | 32 +- .../sdk/io/gcp/bigtable/BigtableWriteIT.java | 6 +- .../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +- .../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +- .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 4 +- .../beam/sdk/io/gcp/datastore/V1WriteIT.java | 6 +- .../sdk/io/gcp/pubsub/PubsubClientTest.java | 50 +- .../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 18 +- .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 88 +- .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 14 +- .../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 58 +- .../gcp/pubsub/PubsubUnboundedSourceTest.java | 112 +- .../gcp/storage/GcsFileSystemRegistrarTest.java | 51 - .../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 -- .../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 - .../sdk/io/gcp/testing/BigqueryMatcher.java | 257 ++ .../sdk/io/gcp/testing/BigqueryMatcherTest.java | 177 ++ sdks/java/io/hadoop-common/pom.xml | 5 - .../beam/sdk/io/hadoop/WritableCoder.java | 29 +- sdks/java/io/hadoop-file-system/README.md | 43 + sdks/java/io/hadoop-file-system/pom.xml | 195 ++ .../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 +++ .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ++++ .../beam/sdk/io/hdfs/HadoopFileSystem.java | 240 ++ .../sdk/io/hdfs/HadoopFileSystemModule.java | 84 + .../sdk/io/hdfs/HadoopFileSystemOptions.java | 49 + .../hdfs/HadoopFileSystemOptionsRegistrar.java | 35 + .../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 62 + .../beam/sdk/io/hdfs/HadoopResourceId.java | 81 + .../java/org/apache/beam/sdk/io/hdfs/Sink.java | 195 ++ .../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 + .../java/org/apache/beam/sdk/io/hdfs/Write.java | 585 ++++ .../apache/beam/sdk/io/hdfs/package-info.java | 22 + .../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 172 ++ .../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 ++ .../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 65 + .../HadoopFileSystemOptionsRegistrarTest.java | 49 + .../io/hdfs/HadoopFileSystemOptionsTest.java | 48 + .../io/hdfs/HadoopFileSystemRegistrarTest.java | 81 + .../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 247 ++ sdks/java/io/hadoop/input-format/pom.xml | 15 - .../hadoop/inputformat/HadoopInputFormatIO.java | 10 +- .../inputformat/HadoopInputFormatIOTest.java | 29 +- sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 10 - sdks/java/io/hadoop/pom.xml | 18 + sdks/java/io/hbase/pom.xml | 7 +- .../org/apache/beam/sdk/io/hbase/HBaseIO.java | 28 +- .../beam/sdk/io/hbase/HBaseMutationCoder.java | 5 +- .../beam/sdk/io/hbase/HBaseResultCoder.java | 5 +- .../apache/beam/sdk/io/hbase/HBaseIOTest.java | 17 +- sdks/java/io/hdfs/README.md | 43 - sdks/java/io/hdfs/pom.xml | 141 - .../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 --- .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ---- .../beam/sdk/io/hdfs/HadoopFileSystem.java | 71 - .../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 42 - .../beam/sdk/io/hdfs/HadoopResourceId.java | 42 - .../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 - .../apache/beam/sdk/io/hdfs/package-info.java | 22 - .../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 -- .../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 -- .../io/hdfs/HadoopFileSystemRegistrarTest.java | 52 - .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 5 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +- sdks/java/io/kafka/pom.xml | 5 - .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 470 ++- .../beam/sdk/io/kafka/KafkaRecordCoder.java | 18 +- .../CoderBasedKafkaDeserializer.java | 71 + .../CoderBasedKafkaSerializer.java | 73 + .../serialization/InstantDeserializer.java | 45 + .../kafka/serialization/InstantSerializer.java | 45 + .../io/kafka/serialization/package-info.java | 22 + .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 399 ++- sdks/java/io/kinesis/pom.xml | 6 + .../apache/beam/sdk/io/kinesis/KinesisIO.java | 148 +- .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +- .../sdk/io/kinesis/KinesisMockReadTest.java | 14 +- .../beam/sdk/io/kinesis/KinesisReaderIT.java | 10 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +- .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +- sdks/java/io/pom.xml | 3 +- sdks/java/io/xml/pom.xml | 113 + .../org/apache/beam/sdk/io/xml/JAXBCoder.java | 165 ++ .../java/org/apache/beam/sdk/io/xml/XmlIO.java | 543 ++++ .../org/apache/beam/sdk/io/xml/XmlSink.java | 161 + .../org/apache/beam/sdk/io/xml/XmlSource.java | 399 +++ .../apache/beam/sdk/io/xml/package-info.java | 22 + .../apache/beam/sdk/io/xml/JAXBCoderTest.java | 221 ++ .../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 276 ++ .../apache/beam/sdk/io/xml/XmlSourceTest.java | 987 +++++++ sdks/java/java8tests/pom.xml | 19 - sdks/java/javadoc/pom.xml | 9 +- .../main/resources/archetype-resources/pom.xml | 9 +- .../main/resources/archetype-resources/pom.xml | 9 +- sdks/java/maven-archetypes/pom.xml | 35 + .../main/resources/archetype-resources/pom.xml | 2 +- sdks/python/.pylintrc | 1 + sdks/python/MANIFEST.in | 2 + sdks/python/apache_beam/coders/coders.py | 16 - sdks/python/apache_beam/coders/coders_test.py | 1 + .../apache_beam/coders/coders_test_common.py | 7 +- .../apache_beam/coders/fast_coders_test.py | 1 + .../apache_beam/coders/slow_coders_test.py | 1 + .../examples/complete/autocomplete_test.py | 1 + .../examples/complete/game/hourly_team_score.py | 1 + .../complete/game/hourly_team_score_test.py | 1 + .../examples/complete/game/user_score.py | 1 + .../examples/complete/game/user_score_test.py | 1 + .../complete/juliaset/juliaset/juliaset_test.py | 1 + .../examples/cookbook/bigquery_side_input.py | 2 +- .../cookbook/bigquery_tornadoes_it_test.py | 1 + .../apache_beam/examples/cookbook/bigshuffle.py | 94 - .../examples/cookbook/bigshuffle_test.py | 63 - .../apache_beam/examples/cookbook/filters.py | 2 +- .../examples/cookbook/multiple_output_pardo.py | 6 +- .../apache_beam/examples/snippets/snippets.py | 122 +- .../examples/snippets/snippets_test.py | 41 +- sdks/python/apache_beam/examples/wordcount.py | 1 + .../examples/wordcount_debugging_test.py | 1 + .../apache_beam/internal/gcp/json_value_test.py | 11 + sdks/python/apache_beam/internal/module_test.py | 1 + sdks/python/apache_beam/internal/pickler.py | 3 + .../python/apache_beam/internal/pickler_test.py | 1 + sdks/python/apache_beam/io/concat_source.py | 74 +- .../python/apache_beam/io/concat_source_test.py | 3 +- sdks/python/apache_beam/io/filebasedsource.py | 20 +- .../apache_beam/io/filebasedsource_test.py | 19 +- sdks/python/apache_beam/io/fileio.py | 45 +- sdks/python/apache_beam/io/fileio_test.py | 56 + sdks/python/apache_beam/io/filesystem.py | 35 +- sdks/python/apache_beam/io/filesystems.py | 204 ++ sdks/python/apache_beam/io/filesystems_test.py | 224 ++ sdks/python/apache_beam/io/filesystems_util.py | 3 +- sdks/python/apache_beam/io/gcp/bigquery.py | 23 +- .../io/gcp/datastore/v1/datastoreio.py | 10 +- .../io/gcp/datastore/v1/datastoreio_test.py | 1 + sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 48 +- .../apache_beam/io/gcp/gcsfilesystem_test.py | 29 + sdks/python/apache_beam/io/gcp/pubsub_test.py | 1 + sdks/python/apache_beam/io/iobase.py | 5 +- sdks/python/apache_beam/io/localfilesystem.py | 27 +- .../apache_beam/io/localfilesystem_test.py | 68 +- sdks/python/apache_beam/io/range_trackers.py | 11 +- sdks/python/apache_beam/io/source_test_utils.py | 8 +- .../apache_beam/io/source_test_utils_test.py | 1 + sdks/python/apache_beam/io/sources_test.py | 2 +- sdks/python/apache_beam/io/textio.py | 13 +- sdks/python/apache_beam/io/tfrecordio.py | 2 + sdks/python/apache_beam/io/tfrecordio_test.py | 9 +- sdks/python/apache_beam/pipeline.py | 52 +- sdks/python/apache_beam/pipeline_test.py | 14 +- sdks/python/apache_beam/pvalue.py | 6 +- sdks/python/apache_beam/runners/api/__init__.py | 14 + .../apache_beam/runners/api/beam_fn_api_pb2.py | 2740 ++++++++++++++++++ .../runners/api/beam_fn_api_pb2_grpc.py | 203 ++ .../runners/api/beam_runner_api_pb2.py | 298 +- sdks/python/apache_beam/runners/common.pxd | 62 +- sdks/python/apache_beam/runners/common.py | 499 +++- .../runners/dataflow/dataflow_metrics.py | 3 + .../runners/dataflow/dataflow_metrics_test.py | 1 + .../runners/dataflow/dataflow_runner.py | 20 +- .../runners/dataflow/dataflow_runner_test.py | 4 +- .../runners/dataflow/internal/apiclient.py | 17 +- .../clients/dataflow/dataflow_v1b3_messages.py | 564 +++- .../runners/dataflow/internal/dependency.py | 29 +- .../dataflow/internal/dependency_test.py | 6 +- .../runners/dataflow/native_io/iobase.py | 1 + .../runners/dataflow/template_runner_test.py | 2 +- .../runners/dataflow/test_dataflow_runner.py | 4 +- .../consumer_tracking_pipeline_visitor_test.py | 2 +- .../apache_beam/runners/direct/direct_runner.py | 12 +- .../apache_beam/runners/portability/__init__.py | 16 + .../runners/portability/fn_api_runner.py | 471 +++ .../runners/portability/fn_api_runner_test.py | 40 + .../portability/maptask_executor_runner.py | 468 +++ .../portability/maptask_executor_runner_test.py | 204 ++ sdks/python/apache_beam/runners/runner.py | 34 - .../apache_beam/runners/worker/__init__.py | 16 + .../apache_beam/runners/worker/data_plane.py | 288 ++ .../runners/worker/data_plane_test.py | 139 + .../apache_beam/runners/worker/log_handler.py | 100 + .../runners/worker/log_handler_test.py | 105 + .../apache_beam/runners/worker/logger.pxd | 25 + .../python/apache_beam/runners/worker/logger.py | 173 ++ .../apache_beam/runners/worker/logger_test.py | 182 ++ .../apache_beam/runners/worker/opcounters.pxd | 45 + .../apache_beam/runners/worker/opcounters.py | 162 ++ .../runners/worker/opcounters_test.py | 149 + .../runners/worker/operation_specs.py | 368 +++ .../apache_beam/runners/worker/operations.pxd | 89 + .../apache_beam/runners/worker/operations.py | 651 +++++ .../apache_beam/runners/worker/sdk_worker.py | 451 +++ .../runners/worker/sdk_worker_main.py | 62 + .../runners/worker/sdk_worker_test.py | 168 ++ .../apache_beam/runners/worker/sideinputs.py | 166 ++ .../runners/worker/sideinputs_test.py | 150 + .../apache_beam/runners/worker/statesampler.pyx | 237 ++ .../runners/worker/statesampler_fake.py | 34 + .../runners/worker/statesampler_test.py | 102 + .../apache_beam/tests/pipeline_verifiers.py | 7 +- sdks/python/apache_beam/transforms/core.py | 90 +- .../python/apache_beam/transforms/ptransform.py | 42 +- .../apache_beam/transforms/ptransform_test.py | 195 +- sdks/python/apache_beam/transforms/trigger.py | 10 +- .../apache_beam/transforms/trigger_test.py | 10 +- sdks/python/apache_beam/transforms/window.py | 114 +- .../apache_beam/transforms/window_test.py | 16 + sdks/python/apache_beam/typehints/decorators.py | 1 + sdks/python/apache_beam/typehints/opcodes.py | 5 +- sdks/python/apache_beam/typehints/typecheck.py | 4 +- .../apache_beam/typehints/typehints_test.py | 1 + sdks/python/apache_beam/utils/__init__.py | 4 - sdks/python/apache_beam/utils/annotations.py | 14 +- .../apache_beam/utils/annotations_test.py | 29 + sdks/python/apache_beam/utils/path.py | 46 - sdks/python/apache_beam/utils/path_test.py | 70 - .../apache_beam/utils/pipeline_options_test.py | 2 +- .../utils/pipeline_options_validator_test.py | 1 + .../apache_beam/utils/test_stream_test.py | 1 + sdks/python/apache_beam/utils/timestamp.py | 4 - sdks/python/apache_beam/utils/urns.py | 93 + sdks/python/apache_beam/utils/value_provider.py | 2 +- .../apache_beam/utils/value_provider_test.py | 4 +- sdks/python/generate_pydoc.sh | 2 + sdks/python/pom.xml | 2 +- sdks/python/run_postcommit.sh | 4 +- sdks/python/run_pylint.sh | 9 +- sdks/python/setup.py | 11 +- sdks/python/tox.ini | 4 +- 950 files changed, 43325 insertions(+), 34844 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/9a59ea32/examples/java/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/9a59ea32/pom.xml ----------------------------------------------------------------------