This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch jstorm-runner in repository https://gitbox.apache.org/repos/asf/beam.git
commit ae7357a77e6d75fcabff4ccf7c9373adf12bd28b Merge: cd9c548 9022ef6 Author: Kenn Knowles <[email protected]> AuthorDate: Tue Dec 5 00:28:09 2017 -0800 Merge pull request #4107: Merge branch 'master' into jstorm-runner at commit 727253e .gitattributes | 2 + .github/PULL_REQUEST_TEMPLATE.md | 16 +- .gitignore | 10 +- .test-infra/jenkins/PreCommit_Pipeline.groovy | 129 + .test-infra/jenkins/common_job_properties.groovy | 216 +- .test-infra/jenkins/job_00_seed.groovy | 114 + .test-infra/jenkins/job_beam_Java_Build.groovy | 74 + .../jenkins/job_beam_Java_CodeHealth.groovy | 39 + .../jenkins/job_beam_Java_IntegrationTest.groovy | 63 + .test-infra/jenkins/job_beam_Java_UnitTest.groovy | 40 + .../job_beam_PerformanceTests_Python.groovy | 58 + ...job_beam_PostCommit_Java_JDKVersionsTest.groovy | 62 + .../job_beam_PostCommit_Java_MavenInstall.groovy | 21 +- ...eam_PostCommit_Java_MavenInstall_Windows.groovy | 5 +- ...PostCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +- ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 54 + .../job_beam_PreCommit_Go_MavenInstall.groovy | 56 + .../job_beam_PreCommit_Java_MavenInstall.groovy | 21 +- .../jenkins/job_beam_PreCommit_Pipeline.groovy | 84 + .../job_beam_PreCommit_Python_MavenInstall.groovy | 57 + .../job_beam_PreCommit_Website_Merge.groovy | 62 + .../job_beam_PreCommit_Website_Stage.groovy | 3 + .../jenkins/job_beam_PreCommit_Website_Test.groovy | 3 + .../jenkins/job_beam_Python_UnitTest.groovy | 40 + .../job_beam_Release_NightlySnapshot.groovy | 22 +- .test-infra/jenkins/job_seed.groovy | 53 - .test-infra/jenkins/job_seed_standalone.groovy | 114 + .../kubernetes/cassandra/LargeITCluster/setup.sh | 21 + .../cassandra/LargeITCluster/start-up.sh | 22 - .../cassandra/LargeITCluster/teardown.sh | 1 - .../kubernetes/cassandra/SmallITCluster/setup.sh | 22 + .../cassandra/SmallITCluster/start-up.sh | 23 - .../cassandra/SmallITCluster/teardown.sh | 1 - .../elasticsearch/LargeProductionCluster/setup.sh | 21 + .../LargeProductionCluster/start-up.sh | 22 - .../LargeProductionCluster/teardown.sh | 1 - .../elasticsearch/SmallITCluster/setup.sh | 22 + .../elasticsearch/SmallITCluster/start-up.sh | 23 - .../elasticsearch/SmallITCluster/teardown.sh | 1 - .../kubernetes/postgres/pkb-config-local.yml | 34 + .test-infra/kubernetes/postgres/pkb-config.yml | 32 + README.md | 4 +- examples/java/pom.xml | 52 +- .../apache/beam/examples/WindowedWordCount.java | 1 - .../java/org/apache/beam/examples/WordCount.java | 4 + .../examples/common/WriteOneFilePerWindow.java | 59 +- .../org/apache/beam/examples/complete/TfIdf.java | 17 +- .../examples/complete/TopWikipediaSessions.java | 24 +- .../beam/examples/complete/TrafficRoutes.java | 19 + .../beam/examples/cookbook/BigQueryTornadoes.java | 2 +- .../examples/cookbook/CombinePerKeyExamples.java | 2 +- .../beam/examples/cookbook/FilterExamples.java | 2 +- .../beam/examples/cookbook/JoinExamples.java | 6 +- .../beam/examples/cookbook/MaxPerKeyExamples.java | 2 +- .../beam/examples/cookbook/TriggerExample.java | 6 +- .../beam/examples/DebuggingWordCountTest.java | 11 +- .../apache/beam/examples/WindowedWordCountIT.java | 4 +- examples/java8/pom.xml | 55 +- .../examples/complete/game/injector/Injector.java | 2 +- .../complete/game/injector/InjectorUtils.java | 2 +- .../examples/complete/game/utils/WriteToText.java | 49 +- .../beam/examples/website_snippets/Snippets.java | 87 + .../examples/complete/game/LeaderBoardTest.java | 2 + .../examples/website_snippets/SnippetsTest.java | 114 + examples/pom.xml | 2 +- model/fn-execution/pom.xml | 114 + .../fn-execution/src/main/proto/beam_fn_api.proto | 729 ++ .../src/main/proto/beam_provision_api.proto | 99 + .../model/fnexecution}/v1/standard_coders.yaml | 0 model/job-management/pom.xml | 114 + .../src/main/proto/beam_artifact_api.proto | 134 + .../src/main/proto/beam_job_api.proto | 174 + model/pipeline/pom.xml | 89 + .../pipeline/src/main/proto/beam_runner_api.proto | 843 ++ model/pipeline/src/main/proto/endpoints.proto | 47 + .../src/main/proto/standard_window_fns.proto | 54 + model/pom.xml | 40 + pom.xml | 601 +- runners/apex/pom.xml | 38 +- .../beam/runners/apex/ApexPipelineOptions.java | 2 +- .../org/apache/beam/runners/apex/ApexRunner.java | 171 +- .../apache/beam/runners/apex/ApexRunnerResult.java | 4 +- .../apex/translation/ApexPipelineTranslator.java | 31 +- .../translation/FlattenPCollectionTranslator.java | 15 +- .../runners/apex/translation/ParDoTranslator.java | 42 +- .../apex/translation/TranslationContext.java | 9 +- .../operators/ApexGroupByKeyOperator.java | 10 +- .../translation/operators/ApexParDoOperator.java | 59 +- .../operators/ApexReadUnboundedInputOperator.java | 6 +- .../apex/translation/utils/ApexStateInternals.java | 17 +- .../apex/translation/utils/NoOpStepContext.java | 37 +- .../utils/SerializablePipelineOptions.java | 78 - .../apex/translation/utils/ValuesSource.java | 6 +- .../beam/runners/apex/ApexYarnLauncherTest.java | 9 +- .../runners/apex/examples/UnboundedTextSource.java | 6 +- .../beam/runners/apex/examples/WordCountTest.java | 8 +- .../translation/ApexGroupByKeyOperatorTest.java | 6 +- .../FlattenPCollectionTranslatorTest.java | 1 - .../apex/translation/GroupByKeyTranslatorTest.java | 6 +- .../apex/translation/ParDoTranslatorTest.java | 4 +- .../translation/utils/ApexStateInternalsTest.java | 411 +- .../apex/translation/utils/CollectionSource.java | 6 +- .../translation/utils/PipelineOptionsTest.java | 150 - runners/core-construction-java/pom.xml | 46 +- .../core/construction/ArtifactServiceStager.java | 244 + .../core/construction/CoderTranslation.java | 183 + .../beam/runners/core/construction/Coders.java | 193 - .../core/construction/CombineTranslation.java | 339 + .../CreatePCollectionViewTranslation.java | 131 + .../core/construction/DisplayDataTranslation.java | 39 + .../core/construction/FlattenTranslator.java | 69 + .../core/construction/ForwardingPTransform.java | 18 +- .../core/construction/GroupByKeyTranslation.java | 65 + .../core/construction/PCollectionTranslation.java | 96 + .../runners/core/construction/PCollections.java | 97 - .../core/construction/PTransformMatchers.java | 109 +- .../core/construction/PTransformReplacements.java | 6 + .../core/construction/PTransformTranslation.java | 443 + .../runners/core/construction/PTransforms.java | 110 - .../core/construction/ParDoTranslation.java | 768 ++ .../construction/PipelineOptionsTranslation.java | 51 + .../core/construction/PipelineTranslation.java | 195 + .../runners/core/construction/PrimitiveCreate.java | 14 +- .../runners/core/construction/ReadTranslation.java | 225 + .../core/construction/RehydratedComponents.java | 160 + .../core/construction/RunnerPCollectionView.java | 113 + .../runners/core/construction/SdkComponents.java | 79 +- .../construction/SerializablePipelineOptions.java | 74 + .../runners/core/construction/SplittableParDo.java | 375 + .../core/construction/TestStreamTranslation.java | 316 + .../runners/core/construction/TransformInputs.java | 50 + .../TransformPayloadTranslatorRegistrar.java | 31 + .../core/construction/TriggerTranslation.java | 336 + .../beam/runners/core/construction/Triggers.java | 336 - .../core/construction/WindowIntoTranslation.java | 149 + .../core/construction/WindowingStrategies.java | 275 - .../construction/WindowingStrategyTranslation.java | 405 + .../core/construction/WriteFilesTranslation.java | 335 + .../core/construction/metrics/MetricFiltering.java | 102 + .../core/construction/metrics/MetricKey.java | 43 + .../core/construction/metrics/package-info.java | 22 + .../beam/runners/core/metrics/MetricFiltering.java | 102 - .../beam/runners/core/metrics/MetricKey.java | 43 - .../beam/runners/core/metrics/package-info.java | 22 - .../construction/ArtifactServiceStagerTest.java | 141 + .../core/construction/CoderTranslationTest.java | 167 + .../beam/runners/core/construction/CodersTest.java | 164 - .../core/construction/CombineTranslationTest.java | 224 + .../CreatePCollectionViewTranslationTest.java | 134 + .../construction/ForwardingPTransformTest.java | 17 +- .../construction/GroupByKeyTranslationTest.java | 44 + .../InMemoryArtifactStagerService.java | 152 + .../construction/PCollectionTranslationTest.java | 227 + .../core/construction/PCollectionsTest.java | 189 - .../core/construction/PTransformMatchersTest.java | 167 +- .../construction/PTransformTranslationTest.java | 220 + .../runners/core/construction/PTransformsTest.java | 188 - .../core/construction/ParDoTranslationTest.java | 323 + .../PipelineOptionsTranslationTest.java | 143 + .../core/construction/PipelineTranslationTest.java | 199 + .../core/construction/ReadTranslationTest.java | 173 + .../construction/RehydratedComponentsTest.java | 96 + .../core/construction/ReplacementOutputsTest.java | 14 +- .../core/construction/SdkComponentsTest.java | 102 +- .../SerializablePipelineOptionsTest.java | 89 + .../core/construction/SplittableParDoTest.java | 158 + .../construction/TestStreamTranslationTest.java | 130 + .../core/construction/TransformInputsTest.java | 166 + .../core/construction/TriggerTranslationTest.java | 112 + .../runners/core/construction/TriggersTest.java | 111 - .../UnboundedReadFromBoundedSourceTest.java | 2 +- .../construction/WindowIntoTranslationTest.java | 127 + .../core/construction/WindowingStrategiesTest.java | 110 - .../WindowingStrategyTranslationTest.java | 124 + .../construction/WriteFilesTranslationTest.java | 195 + .../construction/metrics/MetricFilteringTest.java | 148 + .../runners/core/metrics/MetricFilteringTest.java | 148 - runners/core-java/pom.xml | 31 +- .../beam/runners/core/BaseExecutionContext.java | 164 - .../org/apache/beam/runners/core/DoFnRunners.java | 3 +- .../beam/runners/core/ElementAndRestriction.java | 42 - .../runners/core/ElementAndRestrictionCoder.java | 88 - .../apache/beam/runners/core/ExecutionContext.java | 99 - .../core/GroupAlsoByWindowViaWindowSetNewDoFn.java | 4 +- .../runners/core/GroupByKeyViaGroupByKeyOnly.java | 15 +- .../beam/runners/core/InMemoryStateInternals.java | 39 +- .../beam/runners/core/InMemoryTimerInternals.java | 37 +- .../runners/core/LateDataDroppingDoFnRunner.java | 33 +- ...TimeBoundedSplittableProcessElementInvoker.java | 78 +- .../apache/beam/runners/core/ProcessFnRunner.java | 21 +- .../apache/beam/runners/core/ReduceFnRunner.java | 135 +- .../apache/beam/runners/core/SideInputHandler.java | 2 +- .../apache/beam/runners/core/SimpleDoFnRunner.java | 414 +- .../apache/beam/runners/core/SplittableParDo.java | 602 -- .../core/SplittableParDoViaKeyedWorkItems.java | 435 + .../core/SplittableProcessElementInvoker.java | 25 +- .../org/apache/beam/runners/core/StateTable.java | 40 +- .../org/apache/beam/runners/core/StateTags.java | 16 + .../org/apache/beam/runners/core/StepContext.java | 33 + .../apache/beam/runners/core/SystemReduceFn.java | 6 + .../runners/core/TestInMemoryStateInternals.java | 6 +- .../apache/beam/runners/core/WatermarkHold.java | 6 +- .../beam/runners/core/fn/FnApiControlClient.java | 152 + .../core/fn/FnApiControlClientPoolService.java | 72 + .../beam/runners/core/fn/FnDataReceiver.java | 37 + .../apache/beam/runners/core/fn/FnDataService.java | 85 + .../beam/runners/core/fn/SdkHarnessClient.java | 176 + .../beam/runners/core/fn/SdkHarnessDoFnRunner.java | 108 + .../apache/beam/runners/core/fn/package-info.java | 22 + .../beam/runners/core/metrics/CounterCell.java | 10 +- .../beam/runners/core/metrics/DirtyState.java | 4 +- .../runners/core/metrics/DistributionCell.java | 10 +- .../beam/runners/core/metrics/GaugeCell.java | 11 +- .../beam/runners/core/metrics/MetricUpdates.java | 1 + .../runners/core/metrics/MetricsContainerImpl.java | 45 +- .../core/metrics/MetricsContainerStepMap.java | 2 + .../core/triggers/AfterAllStateMachine.java | 25 +- .../AfterDelayFromFirstElementStateMachine.java | 6 +- .../core/triggers/AfterFirstStateMachine.java | 20 +- .../core/triggers/AfterPaneStateMachine.java | 6 +- .../core/triggers/AfterWatermarkStateMachine.java | 15 +- .../triggers/ExecutableTriggerStateMachine.java | 23 +- .../runners/core/triggers/NeverStateMachine.java | 5 +- .../runners/core/triggers/TriggerStateMachine.java | 27 - .../core/triggers/TriggerStateMachineRunner.java | 3 +- .../core/triggers/TriggerStateMachines.java | 2 +- .../core/ElementAndRestrictionCoderTest.java | 127 - .../runners/core/InMemoryStateInternalsTest.java | 569 +- ...BoundedSplittableProcessElementInvokerTest.java | 47 +- .../beam/runners/core/ReduceFnRunnerTest.java | 457 +- .../apache/beam/runners/core/ReduceFnTester.java | 80 +- .../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +- .../runners/core/SplittableParDoProcessFnTest.java | 594 ++ .../beam/runners/core/SplittableParDoTest.java | 606 -- .../beam/runners/core/StateInternalsTest.java | 672 ++ .../beam/runners/core/StatefulDoFnRunnerTest.java | 4 +- .../apache/beam/runners/core/WindowMatchers.java | 15 + .../core/fn/FnApiControlClientPoolServiceTest.java | 65 + .../runners/core/fn/FnApiControlClientTest.java | 139 + .../beam/runners/core/fn/SdkHarnessClientTest.java | 96 + .../runners/core/fn/SdkHarnessDoFnRunnerTest.java | 73 + .../core/metrics/MetricsContainerImplTest.java | 10 + .../core/triggers/AfterFirstStateMachineTest.java | 5 +- .../triggers/AfterWatermarkStateMachineTest.java | 36 +- .../core/triggers/StubTriggerStateMachine.java | 7 +- .../core/triggers/TriggerStateMachineTester.java | 13 +- .../core/triggers/TriggerStateMachinesTest.java | 2 +- runners/direct-java/pom.xml | 190 +- .../direct/BoundedReadEvaluatorFactory.java | 14 +- .../beam/runners/direct/CommittedResult.java | 12 +- .../direct/CopyOnAccessInMemoryStateInternals.java | 10 +- .../runners/direct/DirectExecutionContext.java | 35 +- ...DirectGBKIntoKeyedWorkItemsOverrideFactory.java | 9 +- .../apache/beam/runners/direct/DirectGraph.java | 62 +- .../beam/runners/direct/DirectGraphVisitor.java | 64 +- .../beam/runners/direct/DirectGroupByKey.java | 50 +- .../direct/DirectGroupByKeyOverrideFactory.java | 14 +- .../apache/beam/runners/direct/DirectMetrics.java | 4 +- .../beam/runners/direct/DirectRegistrar.java | 2 +- .../apache/beam/runners/direct/DirectRunner.java | 119 +- .../beam/runners/direct/DirectTestOptions.java | 42 + .../beam/runners/direct/DirectTimerInternals.java | 9 + .../beam/runners/direct/EmptyInputProvider.java | 8 +- .../beam/runners/direct/EvaluationContext.java | 72 +- .../direct/ExecutorServiceParallelExecutor.java | 27 +- .../direct/GroupAlsoByWindowEvaluatorFactory.java | 28 +- .../runners/direct/KeyedPValueTrackingVisitor.java | 11 +- .../beam/runners/direct/MultiStepCombine.java | 439 + .../apache/beam/runners/direct/ParDoEvaluator.java | 18 +- .../beam/runners/direct/ParDoEvaluatorFactory.java | 55 +- .../runners/direct/ParDoMultiOverrideFactory.java | 129 +- .../beam/runners/direct/ReadEvaluatorFactory.java | 97 + .../beam/runners/direct/RootInputProvider.java | 7 +- .../beam/runners/direct/RootProviderRegistry.java | 28 +- .../apache/beam/runners/direct/SourceShard.java | 33 + .../SplittableProcessElementsEvaluatorFactory.java | 55 +- .../direct/StatefulParDoEvaluatorFactory.java | 29 +- .../runners/direct/TestStreamEvaluatorFactory.java | 46 +- .../runners/direct/TransformEvaluatorRegistry.java | 135 +- .../direct/UnboundedReadEvaluatorFactory.java | 31 +- .../beam/runners/direct/ViewEvaluatorFactory.java | 8 +- .../beam/runners/direct/ViewOverrideFactory.java | 71 +- .../beam/runners/direct/WatermarkManager.java | 37 +- .../runners/direct/WindowEvaluatorFactory.java | 5 +- .../runners/direct/WriteWithShardingFactory.java | 51 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 6 +- .../beam/runners/direct/CommittedResultTest.java | 43 +- .../CopyOnAccessInMemoryStateInternalsTest.java | 74 +- .../runners/direct/DirectGraphVisitorTest.java | 13 +- .../apache/beam/runners/direct/DirectGraphs.java | 17 +- .../beam/runners/direct/DirectMetricsTest.java | 2 +- .../beam/runners/direct/DirectRegistrarTest.java | 2 +- .../beam/runners/direct/DirectRunnerTest.java | 4 +- .../beam/runners/direct/EvaluationContextTest.java | 31 +- .../direct/ImmutabilityEnforcementFactoryTest.java | 4 +- .../beam/runners/direct/MultiStepCombineTest.java | 228 + .../beam/runners/direct/ParDoEvaluatorTest.java | 11 +- .../direct/StatefulParDoEvaluatorFactoryTest.java | 65 +- .../beam/runners/direct/TransformExecutorTest.java | 12 +- .../direct/UnboundedReadEvaluatorFactoryTest.java | 5 +- .../runners/direct/ViewEvaluatorFactoryTest.java | 8 +- .../runners/direct/ViewOverrideFactoryTest.java | 70 +- .../direct/WatermarkCallbackExecutorTest.java | 1 + .../beam/runners/direct/WatermarkManagerTest.java | 16 +- .../runners/direct/WindowEvaluatorFactoryTest.java | 10 + .../direct/WriteWithShardingFactoryTest.java | 75 +- runners/flink/pom.xml | 28 +- .../runners/flink/CreateStreamingFlinkView.java | 156 + .../flink/FlinkBatchPipelineTranslator.java | 5 +- .../flink/FlinkBatchTranslationContext.java | 3 +- .../flink/FlinkPipelineExecutionEnvironment.java | 6 +- .../beam/runners/flink/FlinkPipelineOptions.java | 22 + .../org/apache/beam/runners/flink/FlinkRunner.java | 10 +- .../flink/FlinkStreamingPipelineTranslator.java | 87 +- .../flink/FlinkStreamingTransformTranslators.java | 412 +- .../flink/FlinkStreamingTranslationContext.java | 3 +- .../runners/flink/FlinkStreamingViewOverrides.java | 372 - .../runners/flink/FlinkTransformOverrides.java | 53 + .../translation/functions/FlinkAssignContext.java | 17 +- .../translation/functions/FlinkDoFnFunction.java | 24 +- .../FlinkMergingNonShuffleReduceFunction.java | 8 +- .../functions/FlinkNoOpStepContext.java | 36 +- .../functions/FlinkPartialReduceFunction.java | 8 +- .../translation/functions/FlinkReduceFunction.java | 8 +- .../functions/FlinkStatefulDoFnFunction.java | 20 +- .../translation/types/CoderTypeSerializer.java | 78 + .../translation/types/EncodedValueSerializer.java | 18 +- .../utils/SerializedPipelineOptions.java | 77 - .../translation/wrappers/SourceInputFormat.java | 10 +- .../wrappers/streaming/DoFnOperator.java | 570 +- .../wrappers/streaming/SplittableDoFnOperator.java | 38 +- .../wrappers/streaming/WindowDoFnOperator.java | 8 +- .../streaming/io/BoundedSourceWrapper.java | 10 +- .../streaming/io/UnboundedSocketSource.java | 2 +- .../streaming/io/UnboundedSourceWrapper.java | 19 +- .../state/FlinkBroadcastStateInternals.java | 29 +- .../streaming/state/FlinkSplitStateInternals.java | 8 +- .../streaming/state/FlinkStateInternals.java | 503 +- .../beam/runners/flink/PipelineOptionsTest.java | 191 +- .../runners/flink/streaming/DoFnOperatorTest.java | 332 +- .../FlinkBroadcastStateInternalsTest.java | 242 +- .../streaming/FlinkKeyGroupStateInternalsTest.java | 359 +- .../streaming/FlinkSplitStateInternalsTest.java | 132 +- .../flink/streaming/FlinkStateInternalsTest.java | 343 +- .../flink/streaming/TestCountingSource.java | 53 +- .../streaming/UnboundedSourceWrapperTest.java | 330 +- .../translation/types/CoderTypeSerializerTest.java | 79 + runners/gcp/gcemd/Dockerfile | 30 + runners/gcp/gcemd/main.go | 85 + runners/gcp/gcemd/pom.xml | 154 + runners/gcp/gcsproxy/Dockerfile | 30 + runners/gcp/gcsproxy/main.go | 91 + runners/gcp/gcsproxy/pom.xml | 154 + runners/gcp/pom.xml | 38 + runners/gearpump/README.md | 61 + runners/gearpump/pom.xml | 277 + .../runners/gearpump/GearpumpPipelineOptions.java | 67 + .../runners/gearpump/GearpumpPipelineResult.java | 110 + .../beam/runners/gearpump/GearpumpRunner.java | 119 + .../runners/gearpump/GearpumpRunnerRegistrar.java | 64 + .../beam/runners/gearpump/TestGearpumpRunner.java | 63 + .../apache/beam/runners/gearpump/package-info.java | 22 + .../CreateGearpumpPCollectionViewTranslator.java | 45 + .../translators/CreateStreamingGearpumpView.java | 156 + .../translators/FlattenPCollectionsTranslator.java | 83 + .../translators/GearpumpPipelineTranslator.java | 143 + .../gearpump/translators/GroupByKeyTranslator.java | 258 + .../translators/ParDoMultiOutputTranslator.java | 103 + .../translators/ReadBoundedTranslator.java | 46 + .../translators/ReadUnboundedTranslator.java | 48 + .../gearpump/translators/TransformTranslator.java | 30 + .../gearpump/translators/TranslationContext.java | 105 + .../translators/WindowAssignTranslator.java | 103 + .../translators/functions/DoFnFunction.java | 193 + .../translators/functions/package-info.java | 22 + .../translators/io/BoundedSourceWrapper.java | 45 + .../gearpump/translators/io/GearpumpSource.java | 114 + .../translators/io/UnboundedSourceWrapper.java | 46 + .../gearpump/translators/io/ValuesSource.java | 165 + .../gearpump/translators/io/package-info.java | 22 + .../runners/gearpump/translators/package-info.java | 22 + .../translators/utils/DoFnRunnerFactory.java | 83 + .../translators/utils/NoOpStepContext.java | 41 + .../translators/utils/TranslatorUtils.java | 198 + .../gearpump/translators/utils/package-info.java | 22 + .../gearpump/GearpumpRunnerRegistrarTest.java | 55 + .../beam/runners/gearpump/PipelineOptionsTest.java | 73 + ...reateGearpumpPCollectionViewTranslatorTest.java | 56 + .../FlattenPCollectionsTranslatorTest.java | 155 + .../translators/GroupByKeyTranslatorTest.java | 152 + .../translators/ReadBoundedTranslatorTest.java | 70 + .../translators/ReadUnboundedTranslatorTest.java | 70 + .../translators/WindowAssignTranslatorTest.java | 110 + .../translators/io/GearpumpSourceTest.java | 99 + .../gearpump/translators/io/ValueSoureTest.java | 79 + .../translators/utils/TranslatorUtilsTest.java | 73 + runners/google-cloud-dataflow-java/pom.xml | 174 +- .../beam/runners/dataflow/AssignWindows.java | 90 - .../dataflow/BatchStatefulParDoOverrides.java | 75 +- .../beam/runners/dataflow/BatchViewOverrides.java | 201 +- .../beam/runners/dataflow/CreateDataflowView.java | 7 +- .../beam/runners/dataflow/DataflowMetrics.java | 306 +- .../beam/runners/dataflow/DataflowPipelineJob.java | 18 +- .../dataflow/DataflowPipelineTranslator.java | 143 +- .../beam/runners/dataflow/DataflowRunner.java | 489 +- .../beam/runners/dataflow/DataflowRunnerInfo.java | 91 +- .../dataflow/PrimitiveParDoSingleFactory.java | 14 +- .../beam/runners/dataflow/ReadTranslator.java | 7 +- .../runners/dataflow/SplittableParDoOverrides.java | 76 + .../runners/dataflow/StreamingViewOverrides.java | 10 +- .../beam/runners/dataflow/TransformTranslator.java | 30 +- .../dataflow/options/CloudDebuggerOptions.java | 3 +- .../options/DataflowPipelineDebugOptions.java | 23 +- .../dataflow/options/DataflowPipelineOptions.java | 8 + .../options/DataflowPipelineWorkerPoolOptions.java | 3 + .../dataflow/options/DataflowProfilingOptions.java | 3 +- .../options/DataflowWorkerLoggingOptions.java | 6 + .../runners/dataflow/util/CloudObjectKinds.java | 1 + .../dataflow/util/CloudObjectTranslators.java | 5 +- ...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 + .../beam/runners/dataflow/util/DoFnInfo.java | 129 - .../beam/runners/dataflow/util/GcsStager.java | 53 +- .../beam/runners/dataflow/util/MonitoringUtil.java | 18 +- .../beam/runners/dataflow/util/PackageUtil.java | 581 +- .../beam/runners/dataflow/util/PropertyNames.java | 47 +- .../runners/dataflow/util/RandomAccessData.java | 2 +- .../apache/beam/runners/dataflow/util/Stager.java | 32 +- .../beam/runners/dataflow/util/TimeUtil.java | 24 +- .../dataflow/BatchStatefulParDoOverridesTest.java | 46 +- .../beam/runners/dataflow/DataflowMetricsTest.java | 215 +- .../dataflow/DataflowPipelineTranslatorTest.java | 214 +- .../runners/dataflow/DataflowRunnerInfoTest.java | 9 + .../beam/runners/dataflow/DataflowRunnerTest.java | 334 +- .../transforms/DataflowGroupByKeyTest.java | 12 +- .../dataflow/transforms/DataflowViewTest.java | 14 +- .../runners/dataflow/util/MonitoringUtilTest.java | 4 +- .../runners/dataflow/util/PackageUtilTest.java | 124 +- .../beam/runners/dataflow/util/TimeUtilTest.java | 6 + runners/java-fn-execution/pom.xml | 135 + .../beam/runners/fnexecution/ServerFactory.java | 104 + .../fnexecution/control/FnApiControlClient.java | 148 + .../control/FnApiControlClientPoolService.java | 66 + .../fnexecution/control/SdkHarnessClient.java | 173 + .../runners/fnexecution/control/package-info.java | 23 + .../runners/fnexecution/data/FnDataReceiver.java | 27 + .../runners/fnexecution/data/package-info.java | 23 + .../beam/runners/fnexecution/package-info.java | 23 + .../runners/fnexecution/ServerFactoryTest.java | 153 + .../control/FnApiControlClientPoolServiceTest.java | 65 + .../control/FnApiControlClientTest.java | 139 + .../fnexecution/control/SdkHarnessClientTest.java | 96 + runners/jstorm/pom.xml | 2 +- .../jstorm/translation/DefaultStepContext.java | 37 +- .../jstorm/translation/JStormMetricResults.java | 4 +- .../translation/JStormPipelineTranslator.java | 4 +- .../jstorm/translation/JStormTimerInternals.java | 3 + .../jstorm/translation/MetricsReporter.java | 2 +- runners/local-artifact-service-java/pom.xml | 116 + .../LocalFileSystemArtifactStagerService.java | 279 + .../apache/beam/artifact/local/package-info.java | 22 + .../LocalFileSystemArtifactStagerServiceTest.java | 301 + runners/pom.xml | 15 +- runners/reference/job-server/pom.xml | 82 + .../reference/job/ReferenceRunnerJobServer.java | 77 + .../reference/job/ReferenceRunnerJobService.java | 79 + .../beam/runners/reference/job/package-info.java | 23 + .../job/ReferenceRunnerJobServiceTest.java | 34 + runners/reference/pom.xml | 39 + runners/spark/pom.xml | 94 +- .../runners/spark/SparkNativePipelineVisitor.java | 3 +- .../org/apache/beam/runners/spark/SparkRunner.java | 14 +- .../apache/beam/runners/spark/TestSparkRunner.java | 2 +- .../spark/aggregators/NamedAggregators.java | 93 - .../apache/beam/runners/spark/io/CreateStream.java | 115 +- .../beam/runners/spark/io/MicrobatchSource.java | 4 +- .../beam/runners/spark/io/SourceDStream.java | 20 +- .../apache/beam/runners/spark/io/SourceRDD.java | 22 +- .../runners/spark/io/SparkUnboundedSource.java | 8 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 821 +- .../spark/stateful/SparkTimerInternals.java | 39 +- .../runners/spark/stateful/StateSpecFunctions.java | 10 +- .../runners/spark/translation/BoundedDataset.java | 17 +- .../beam/runners/spark/translation/Dataset.java | 3 +- .../spark/translation/EvaluationContext.java | 38 +- .../spark/translation/MultiDoFnFunction.java | 126 +- .../spark/translation/SparkAbstractCombineFn.java | 9 +- .../spark/translation/SparkContextFactory.java | 2 - .../spark/translation/SparkGlobalCombineFn.java | 5 +- .../SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +- .../spark/translation/SparkKeyedCombineFn.java | 5 +- .../spark/translation/SparkProcessContext.java | 50 +- .../spark/translation/SparkRuntimeContext.java | 98 - .../spark/translation/StorageLevelPTransform.java | 43 - .../spark/translation/TransformTranslator.java | 209 +- .../spark/translation/TranslationUtils.java | 78 + .../streaming/StreamingTransformTranslator.java | 110 +- .../translation/streaming/UnboundedDataset.java | 27 +- .../streaming/WatermarkSyncedDStream.java | 149 + .../runners/spark/util/GlobalWatermarkHolder.java | 307 +- .../util/SinglePrimitiveOutputPTransform.java | 51 - .../runners/spark/GlobalWatermarkHolderTest.java | 18 +- .../beam/runners/spark/SparkPipelineStateTest.java | 4 +- .../runners/spark/SparkRunnerDebuggerTest.java | 31 +- .../spark/stateful/SparkStateInternalsTest.java | 66 + .../spark/translation/SparkRuntimeContextTest.java | 122 - .../spark/translation/StorageLevelTest.java | 75 - .../translation/streaming/CreateStreamTest.java | 33 +- .../streaming/TrackStreamingSourcesTest.java | 14 +- runners/spark/src/test/resources/log4j.properties | 11 +- sdks/CONTAINERS.md | 162 + sdks/common/fn-api/pom.xml | 104 - .../common/fn-api/src/main/proto/beam_fn_api.proto | 761 -- sdks/common/pom.xml | 39 - sdks/common/runner-api/pom.xml | 86 - .../src/main/proto/beam_runner_api.proto | 711 -- sdks/go/BUILD.md | 63 + sdks/go/cmd/beamctl/cmd/artifact.go | 98 + sdks/go/cmd/beamctl/cmd/root.go | 56 + sdks/go/cmd/beamctl/main.go | 31 + sdks/go/descriptor.xml | 29 + sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go | 155 + sdks/go/pkg/beam/artifact/gcsproxy/staging.go | 200 + sdks/go/pkg/beam/artifact/materialize.go | 240 + sdks/go/pkg/beam/artifact/materialize_test.go | 238 + sdks/go/pkg/beam/artifact/server_test.go | 213 + sdks/go/pkg/beam/artifact/stage.go | 238 + sdks/go/pkg/beam/artifact/stage_test.go | 98 + .../beam/model/fnexecution_v1/beam_fn_api.pb.go | 2729 +++++ .../model/fnexecution_v1/beam_provision_api.pb.go | 306 + sdks/go/pkg/beam/model/gen.go | 22 + .../model/jobmanagement_v1/beam_artifact_api.pb.go | 690 ++ .../beam/model/jobmanagement_v1/beam_job_api.pb.go | 903 ++ .../beam/model/pipeline_v1/beam_runner_api.pb.go | 3491 +++++++ sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 160 + .../model/pipeline_v1/standard_window_fns.pb.go | 120 + sdks/go/pkg/beam/provision/provision_test.go | 54 + sdks/go/pkg/beam/provision/provison.go | 80 + sdks/go/pkg/beam/util/errorx/guarded.go | 47 + sdks/go/pkg/beam/util/execx/exec.go | 33 + sdks/go/pkg/beam/util/gcsx/gcs.go | 88 + sdks/go/pkg/beam/util/grpcx/dial.go | 37 + sdks/go/pkg/beam/util/grpcx/metadata.go | 55 + sdks/go/pkg/beam/util/syscallx/syscall.go | 27 + sdks/go/pkg/beam/util/syscallx/syscall_default.go | 28 + sdks/go/pkg/beam/util/syscallx/syscall_linux.go | 38 + sdks/go/pom.xml | 163 + sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/checkstyle.xml | 8 + .../src/main/resources/beam/findbugs-filter.xml | 74 +- .../main/resources/docker/file/openjdk7/Dockerfile | 49 + .../docker/file/openjdk7/docker-entrypoint.sh | 24 + .../main/resources/docker/file/openjdk8/Dockerfile | 49 + .../docker/file/openjdk8/docker-entrypoint.sh | 24 + .../main/resources/docker/git/openjdk8/Dockerfile | 53 + .../docker/git/openjdk8/docker-entrypoint.sh | 22 + .../resources/docker/release/python2/Dockerfile | 21 + sdks/java/container/Dockerfile | 28 + sdks/java/container/boot.go | 134 + sdks/java/container/pom.xml | 184 + sdks/java/core/pom.xml | 186 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 73 +- .../java/org/apache/beam/sdk/PipelineResult.java | 14 +- .../apache/beam/sdk/annotations/Experimental.java | 13 +- .../apache/beam/sdk/annotations/package-info.java | 4 + .../apache/beam/sdk/coders/BeamRecordCoder.java | 111 + .../org/apache/beam/sdk/coders/BooleanCoder.java | 59 + .../java/org/apache/beam/sdk/coders/Coder.java | 12 +- .../org/apache/beam/sdk/coders/CoderRegistry.java | 29 + .../org/apache/beam/sdk/coders/DefaultCoder.java | 19 +- .../org/apache/beam/sdk/coders/InstantCoder.java | 81 +- .../apache/beam/sdk/coders/LengthPrefixCoder.java | 3 +- .../apache/beam/sdk/coders/SerializableCoder.java | 7 +- .../apache/beam/sdk/coders/ShardedKeyCoder.java | 66 + .../apache/beam/sdk/coders/StructuredCoder.java | 42 +- .../java/org/apache/beam/sdk/coders/VoidCoder.java | 12 +- .../org/apache/beam/sdk/coders/package-info.java | 4 + .../main/java/org/apache/beam/sdk/io/AvroIO.java | 1216 ++- .../main/java/org/apache/beam/sdk/io/AvroSink.java | 95 +- .../java/org/apache/beam/sdk/io/AvroSource.java | 393 +- .../java/org/apache/beam/sdk/io/AvroUtils.java | 40 + .../org/apache/beam/sdk/io/BlockBasedSource.java | 31 +- .../sdk/io/BoundedReadFromUnboundedSource.java | 12 +- .../org/apache/beam/sdk/io/CompressedSource.java | 310 +- .../java/org/apache/beam/sdk/io/Compression.java | 228 + .../beam/sdk/io/ConstantAvroDestination.java | 130 + .../org/apache/beam/sdk/io/CountingSource.java | 49 +- .../apache/beam/sdk/io/DefaultFilenamePolicy.java | 378 +- .../beam/sdk/io/DynamicAvroDestinations.java | 46 + .../beam/sdk/io/DynamicFileDestinations.java | 150 + .../java/org/apache/beam/sdk/io/FileBasedSink.java | 833 +- .../org/apache/beam/sdk/io/FileBasedSource.java | 81 +- .../main/java/org/apache/beam/sdk/io/FileIO.java | 450 + .../java/org/apache/beam/sdk/io/FileSystems.java | 69 +- .../org/apache/beam/sdk/io/GenerateSequence.java | 3 +- .../org/apache/beam/sdk/io/LocalFileSystem.java | 45 +- .../org/apache/beam/sdk/io/OffsetBasedSource.java | 22 +- .../src/main/java/org/apache/beam/sdk/io/Read.java | 27 +- .../beam/sdk/io/ReadAllViaFileBasedSource.java | 114 + .../org/apache/beam/sdk/io/ReadableFileCoder.java | 50 + .../main/java/org/apache/beam/sdk/io/Source.java | 28 +- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 228 +- .../main/java/org/apache/beam/sdk/io/TextIO.java | 1054 +- .../main/java/org/apache/beam/sdk/io/TextSink.java | 22 +- .../java/org/apache/beam/sdk/io/TextSource.java | 120 +- .../java/org/apache/beam/sdk/io/WriteFiles.java | 782 +- .../org/apache/beam/sdk/io/WriteFilesResult.java | 81 + .../apache/beam/sdk/io/fs/EmptyMatchTreatment.java | 46 + .../org/apache/beam/sdk/io/fs/MatchResult.java | 5 +- .../org/apache/beam/sdk/io/fs/MetadataCoder.java | 63 + .../org/apache/beam/sdk/io/fs/ResourceIdCoder.java | 56 + .../java/org/apache/beam/sdk/io/package-info.java | 2 +- .../beam/sdk/io/range/ByteKeyRangeTracker.java | 27 +- .../org/apache/beam/sdk/io/range/OffsetRange.java | 101 + .../beam/sdk/io/range/OffsetRangeTracker.java | 25 +- .../org/apache/beam/sdk/metrics/MetricName.java | 7 + .../apache/beam/sdk/metrics/MetricsContainer.java | 3 +- .../org/apache/beam/sdk/metrics/package-info.java | 4 + .../options/DefaultPipelineOptionsRegistrar.java | 2 + .../beam/sdk/options/ExperimentalOptions.java | 38 + .../apache/beam/sdk/options/PipelineOptions.java | 43 +- .../beam/sdk/options/PipelineOptionsFactory.java | 63 +- .../beam/sdk/options/PipelineOptionsValidator.java | 34 +- .../beam/sdk/options/ProxyInvocationHandler.java | 19 +- .../apache/beam/sdk/options/SdkHarnessOptions.java | 173 + .../org/apache/beam/sdk/options/ValueProvider.java | 34 +- .../apache/beam/sdk/options/ValueProviders.java | 25 +- .../java/org/apache/beam/sdk/package-info.java | 4 + .../beam/sdk/runners/TransformHierarchy.java | 247 +- .../org/apache/beam/sdk/runners/package-info.java | 4 + .../java/org/apache/beam/sdk/state/BagState.java | 6 + .../org/apache/beam/sdk/state/CombiningState.java | 5 + .../org/apache/beam/sdk/state/GroupingState.java | 12 +- .../java/org/apache/beam/sdk/state/MapState.java | 20 +- .../org/apache/beam/sdk/state/ReadableState.java | 6 + .../org/apache/beam/sdk/state/ReadableStates.java | 4 +- .../java/org/apache/beam/sdk/state/SetState.java | 10 +- .../java/org/apache/beam/sdk/state/StateSpec.java | 53 + .../java/org/apache/beam/sdk/state/StateSpecs.java | 57 +- .../org/apache/beam/sdk/state/package-info.java | 4 + .../apache/beam/sdk/testing/CombineFnTester.java | 147 + .../beam/sdk/testing/FileChecksumMatcher.java | 41 +- .../org/apache/beam/sdk/testing/LargeKeys.java | 48 + .../java/org/apache/beam/sdk/testing/PAssert.java | 79 +- .../apache/beam/sdk/testing/PaneExtractors.java | 25 +- .../apache/beam/sdk/testing/SourceTestUtils.java | 23 +- .../org/apache/beam/sdk/testing/StaticWindows.java | 17 + .../org/apache/beam/sdk/testing/StreamingIT.java | 4 + .../apache/beam/sdk/testing/SuccessOrFailure.java | 24 +- .../org/apache/beam/sdk/testing/TestPipeline.java | 121 +- .../org/apache/beam/sdk/testing/TestStream.java | 77 +- .../beam/sdk/testing/UsesCustomWindowMerging.java | 23 + .../apache/beam/sdk/testing/WindowFnTestUtils.java | 141 +- .../apache/beam/sdk/testing/WindowSupplier.java | 4 +- .../org/apache/beam/sdk/testing/package-info.java | 5 + .../beam/sdk/transforms/ApproximateQuantiles.java | 14 +- .../beam/sdk/transforms/ApproximateUnique.java | 4 +- .../org/apache/beam/sdk/transforms/Combine.java | 63 +- .../org/apache/beam/sdk/transforms/CombineFns.java | 2 + .../org/apache/beam/sdk/transforms/Contextful.java | 127 + .../java/org/apache/beam/sdk/transforms/Count.java | 10 + .../org/apache/beam/sdk/transforms/Create.java | 119 +- .../org/apache/beam/sdk/transforms/Distinct.java | 82 +- .../java/org/apache/beam/sdk/transforms/DoFn.java | 66 +- .../org/apache/beam/sdk/transforms/DoFnTester.java | 48 +- .../org/apache/beam/sdk/transforms/Filter.java | 26 +- .../beam/sdk/transforms/FlatMapElements.java | 150 +- .../org/apache/beam/sdk/transforms/Flatten.java | 22 +- .../org/apache/beam/sdk/transforms/GroupByKey.java | 12 +- .../beam/sdk/transforms/GroupIntoBatches.java | 12 +- .../org/apache/beam/sdk/transforms/Latest.java | 2 +- .../apache/beam/sdk/transforms/MapElements.java | 77 +- .../java/org/apache/beam/sdk/transforms/Max.java | 9 +- .../java/org/apache/beam/sdk/transforms/Min.java | 12 +- .../org/apache/beam/sdk/transforms/PTransform.java | 28 +- .../java/org/apache/beam/sdk/transforms/ParDo.java | 110 +- .../apache/beam/sdk/transforms/Requirements.java | 61 + .../org/apache/beam/sdk/transforms/Reshuffle.java | 47 + .../beam/sdk/transforms/SerializableFunctions.java | 50 + .../java/org/apache/beam/sdk/transforms/Sum.java | 30 + .../java/org/apache/beam/sdk/transforms/Top.java | 41 +- .../java/org/apache/beam/sdk/transforms/View.java | 55 +- .../java/org/apache/beam/sdk/transforms/Watch.java | 1066 ++ .../org/apache/beam/sdk/transforms/WithKeys.java | 12 +- .../beam/sdk/transforms/display/DisplayData.java | 25 +- .../beam/sdk/transforms/display/package-info.java | 4 + .../beam/sdk/transforms/join/CoGbkResult.java | 15 +- .../sdk/transforms/join/KeyedPCollectionTuple.java | 10 +- .../beam/sdk/transforms/join/RawUnionValue.java | 8 +- .../beam/sdk/transforms/join/package-info.java | 4 + .../apache/beam/sdk/transforms/package-info.java | 4 + .../reflect/ByteBuddyDoFnInvokerFactory.java | 30 +- .../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 +- .../beam/sdk/transforms/reflect/DoFnInvoker.java | 65 +- .../beam/sdk/transforms/reflect/DoFnInvokers.java | 9 - .../beam/sdk/transforms/reflect/DoFnSignature.java | 33 +- .../sdk/transforms/reflect/DoFnSignatures.java | 45 +- .../transforms/reflect/OnTimerMethodSpecifier.java | 37 + .../beam/sdk/transforms/reflect/package-info.java | 3 + .../sdk/transforms/splittabledofn/OffsetRange.java | 77 - .../splittabledofn/OffsetRangeTracker.java | 16 +- .../splittabledofn/RestrictionTracker.java | 11 +- .../transforms/splittabledofn/package-info.java | 4 + .../sdk/transforms/windowing/BoundedWindow.java | 20 +- .../sdk/transforms/windowing/CalendarWindows.java | 36 + .../sdk/transforms/windowing/FixedWindows.java | 11 + .../sdk/transforms/windowing/GlobalWindows.java | 22 +- .../windowing/IncompatibleWindowException.java | 38 + .../sdk/transforms/windowing/InvalidWindows.java | 11 + .../windowing/MergeOverlappingIntervalWindows.java | 4 +- .../transforms/windowing/PartitioningWindowFn.java | 5 + .../beam/sdk/transforms/windowing/Sessions.java | 11 + .../sdk/transforms/windowing/SlidingWindows.java | 16 + .../beam/sdk/transforms/windowing/Trigger.java | 20 +- .../beam/sdk/transforms/windowing/Window.java | 47 +- .../beam/sdk/transforms/windowing/WindowFn.java | 32 + .../sdk/transforms/windowing/package-info.java | 4 + .../java/org/apache/beam/sdk/util/ApiSurface.java | 2 + .../java/org/apache/beam/sdk/util/ClassPath.java | 8 +- .../java/org/apache/beam/sdk/util/CoderUtils.java | 2 +- .../java/org/apache/beam/sdk/util/DoFnInfo.java | 104 + .../org/apache/beam/sdk/util/IdentityWindowFn.java | 17 +- .../apache/beam/sdk/util/MutationDetectors.java | 79 +- .../beam/sdk/util/SerializableThrowable.java | 49 + .../apache/beam/sdk/util/SerializableUtils.java | 69 +- .../org/apache/beam/sdk/values/BeamRecord.java | 319 + .../org/apache/beam/sdk/values/BeamRecordType.java | 96 + .../org/apache/beam/sdk/values/PCollection.java | 21 +- .../apache/beam/sdk/values/PCollectionTuple.java | 10 +- .../apache/beam/sdk/values/PCollectionViews.java | 37 + .../java/org/apache/beam/sdk/values/PValue.java | 4 +- .../org/apache/beam/sdk/values/PValueBase.java | 12 - .../org/apache/beam/sdk/values/ShardedKey.java | 65 + .../org/apache/beam/sdk/values/TypeDescriptor.java | 64 +- .../apache/beam/sdk/values/TypeDescriptors.java | 127 +- .../apache/beam/sdk/values/WindowingStrategy.java | 46 +- .../java/org/apache/beam/sdk/PipelineTest.java | 51 +- .../test/java/org/apache/beam/sdk/TestUtils.java | 88 - .../org/apache/beam/sdk/coders/AvroCoderTest.java | 39 +- .../apache/beam/sdk/coders/CoderRegistryTest.java | 69 + .../apache/beam/sdk/coders/CommonCoderTest.java | 4 +- .../apache/beam/sdk/coders/DefaultCoderTest.java | 3 +- .../org/apache/beam/sdk/coders/VoidCoderTest.java | 11 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 917 +- .../apache/beam/sdk/io/AvroIOTransformTest.java | 324 - .../org/apache/beam/sdk/io/AvroSourceTest.java | 59 +- .../apache/beam/sdk/io/CompressedSourceTest.java | 138 +- .../beam/sdk/io/DefaultFilenamePolicyTest.java | 108 +- .../sdk/io/DrunkWritableByteChannelFactory.java | 2 +- .../org/apache/beam/sdk/io/FileBasedSinkTest.java | 202 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 56 +- .../java/org/apache/beam/sdk/io/FileIOTest.java | 313 + .../org/apache/beam/sdk/io/FileSystemsTest.java | 15 +- .../apache/beam/sdk/io/LocalFileSystemTest.java | 49 + .../apache/beam/sdk/io/OffsetBasedSourceTest.java | 5 +- .../test/java/org/apache/beam/sdk/io/ReadTest.java | 10 +- .../java/org/apache/beam/sdk/io/SimpleSink.java | 77 +- .../org/apache/beam/sdk/io/TFRecordIOTest.java | 35 +- .../org/apache/beam/sdk/io/TextIOReadTest.java | 860 ++ .../java/org/apache/beam/sdk/io/TextIOTest.java | 1104 +- .../org/apache/beam/sdk/io/TextIOWriteTest.java | 643 ++ .../org/apache/beam/sdk/io/WriteFilesTest.java | 479 +- .../beam/sdk/io/range/ByteKeyRangeTrackerTest.java | 23 + .../beam/sdk/metrics/MetricResultsMatchers.java | 2 +- .../org/apache/beam/sdk/metrics/MetricsTest.java | 28 + .../sdk/options/PipelineOptionsFactoryTest.java | 39 + .../beam/sdk/options/PipelineOptionsTest.java | 11 + .../sdk/options/PipelineOptionsValidatorTest.java | 44 + .../sdk/options/ProxyInvocationHandlerTest.java | 23 +- .../beam/sdk/options/SdkHarnessOptionsTest.java | 76 + .../apache/beam/sdk/options/ValueProviderTest.java | 38 +- .../beam/sdk/runners/TransformHierarchyTest.java | 216 +- .../apache/beam/sdk/runners/TransformTreeTest.java | 19 +- .../sdk/runners/dataflow/TestCountingSource.java | 5 +- .../beam/sdk/testing/CombineFnTesterTest.java | 276 + .../sdk/testing/InterceptingUrlClassLoader.java | 57 + .../org/apache/beam/sdk/testing/PAssertTest.java | 41 +- .../beam/sdk/testing/PCollectionViewTesting.java | 8 + .../beam/sdk/testing/PaneExtractorsTest.java | 7 +- .../apache/beam/sdk/testing/StaticWindowsTest.java | 12 + .../apache/beam/sdk/testing/TestPipelineTest.java | 75 +- .../sdk/transforms/ApproximateQuantilesTest.java | 530 +- .../apache/beam/sdk/transforms/CombineTest.java | 377 +- .../org/apache/beam/sdk/transforms/CreateTest.java | 70 +- .../apache/beam/sdk/transforms/DistinctTest.java | 130 +- .../apache/beam/sdk/transforms/DoFnTesterTest.java | 32 + .../beam/sdk/transforms/FlatMapElementsTest.java | 35 +- .../apache/beam/sdk/transforms/FlattenTest.java | 37 +- .../apache/beam/sdk/transforms/GroupByKeyTest.java | 280 +- .../beam/sdk/transforms/GroupIntoBatchesTest.java | 4 +- .../beam/sdk/transforms/MapElementsTest.java | 42 +- .../org/apache/beam/sdk/transforms/MaxTest.java | 10 +- .../org/apache/beam/sdk/transforms/MeanTest.java | 4 +- .../org/apache/beam/sdk/transforms/MinTest.java | 10 +- .../org/apache/beam/sdk/transforms/ParDoTest.java | 322 +- .../beam/sdk/transforms/SplittableDoFnTest.java | 155 +- .../org/apache/beam/sdk/transforms/SumTest.java | 8 +- .../org/apache/beam/sdk/transforms/TopTest.java | 10 +- .../org/apache/beam/sdk/transforms/ViewTest.java | 11 +- .../org/apache/beam/sdk/transforms/WatchTest.java | 797 ++ .../apache/beam/sdk/transforms/WithKeysTest.java | 30 +- .../sdk/transforms/reflect/DoFnInvokersTest.java | 103 +- .../reflect/DoFnSignaturesProcessElementTest.java | 2 +- .../reflect/DoFnSignaturesSplittableDoFnTest.java | 83 +- .../sdk/transforms/reflect/DoFnSignaturesTest.java | 14 + .../splittabledofn/OffsetRangeTrackerTest.java | 1 + .../transforms/windowing/CalendarWindowsTest.java | 24 + .../sdk/transforms/windowing/FixedWindowsTest.java | 7 + .../sdk/transforms/windowing/SessionsTest.java | 14 + .../transforms/windowing/SlidingWindowsTest.java | 41 +- .../beam/sdk/transforms/windowing/WindowTest.java | 225 + .../beam/sdk/util/IdentitySideInputWindowFn.java | 4 + .../beam/sdk/util/MutationDetectorsTest.java | 56 + .../beam/sdk/util/SerializableUtilsTest.java | 60 + .../beam/sdk/values/PCollectionTupleTest.java | 7 +- .../beam/sdk/values/TypeDescriptorsTest.java | 60 + .../extensions/google-cloud-platform-core/pom.xml | 2 +- .../sdk/extensions/gcp/storage/GcsFileSystem.java | 5 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 16 +- .../beam/sdk/util/RetryHttpRequestInitializer.java | 147 +- .../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +- .../java/org/apache/beam/sdk/util/GcsUtilTest.java | 61 +- .../sdk/util/RetryHttpRequestInitializerTest.java | 31 +- sdks/java/extensions/jackson/pom.xml | 2 +- sdks/java/extensions/join-library/pom.xml | 2 +- .../beam/sdk/extensions/joinlibrary/Join.java | 65 +- .../extensions/joinlibrary/OuterFullJoinTest.java | 179 + sdks/java/extensions/pom.xml | 4 +- sdks/java/extensions/protobuf/pom.xml | 2 +- sdks/java/extensions/sketching/pom.xml | 104 + .../extensions/sketching/ApproximateDistinct.java | 573 ++ .../sdk/extensions/sketching/package-info.java | 22 + .../sketching/ApproximateDistinctTest.java | 209 + sdks/java/extensions/sorter/pom.xml | 8 +- .../beam/sdk/extensions/sorter/SortValues.java | 20 +- sdks/java/extensions/sql/NOTICE | 45 + sdks/java/extensions/sql/pom.xml | 425 + .../extensions/sql/src/main/codegen/config.fmpp | 23 + .../sql/src/main/codegen/data/Parser.tdd | 75 + .../sql/src/main/codegen/includes/license.ftl | 17 + .../sql/src/main/codegen/includes/parserImpls.ftl | 89 + .../beam/sdk/extensions/sql/BeamRecordSqlType.java | 186 + .../apache/beam/sdk/extensions/sql/BeamSql.java | 250 + .../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 115 + .../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 + .../beam/sdk/extensions/sql/BeamSqlTable.java | 54 + .../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 43 + .../sdk/extensions/sql/example/BeamSqlExample.java | 104 + .../sdk/extensions/sql/example/package-info.java | 23 + .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 136 + .../interpreter/BeamSqlExpressionExecutor.java | 44 + .../sql/impl/interpreter/BeamSqlFnExecutor.java | 463 + .../operator/BeamSqlCaseExpression.java | 64 + .../operator/BeamSqlCastExpression.java | 132 + .../interpreter/operator/BeamSqlExpression.java | 79 + .../operator/BeamSqlInputRefExpression.java | 48 + .../interpreter/operator/BeamSqlPrimitive.java | 157 + .../interpreter/operator/BeamSqlUdfExpression.java | 92 + .../operator/BeamSqlWindowEndExpression.java | 48 + .../operator/BeamSqlWindowExpression.java | 51 + .../operator/BeamSqlWindowStartExpression.java | 49 + .../sql/impl/interpreter/operator/UdafImpl.java | 87 + .../arithmetic/BeamSqlArithmeticExpression.java | 124 + .../arithmetic/BeamSqlDivideExpression.java | 37 + .../arithmetic/BeamSqlMinusExpression.java | 36 + .../operator/arithmetic/BeamSqlModExpression.java | 36 + .../arithmetic/BeamSqlMultiplyExpression.java | 36 + .../operator/arithmetic/BeamSqlPlusExpression.java | 36 + .../operator/arithmetic/package-info.java | 22 + .../comparison/BeamSqlCompareExpression.java | 97 + .../comparison/BeamSqlEqualsExpression.java | 49 + .../comparison/BeamSqlGreaterThanExpression.java | 49 + .../BeamSqlGreaterThanOrEqualsExpression.java | 49 + .../comparison/BeamSqlIsNotNullExpression.java | 54 + .../comparison/BeamSqlIsNullExpression.java | 54 + .../comparison/BeamSqlLessThanExpression.java | 49 + .../BeamSqlLessThanOrEqualsExpression.java | 49 + .../comparison/BeamSqlNotEqualsExpression.java | 49 + .../operator/comparison/package-info.java | 22 + .../date/BeamSqlCurrentDateExpression.java | 45 + .../date/BeamSqlCurrentTimeExpression.java | 53 + .../date/BeamSqlCurrentTimestampExpression.java | 49 + .../operator/date/BeamSqlDateCeilExpression.java | 55 + .../operator/date/BeamSqlDateFloorExpression.java | 55 + .../date/BeamSqlDatetimeMinusExpression.java | 107 + .../date/BeamSqlDatetimePlusExpression.java | 129 + .../operator/date/BeamSqlExtractExpression.java | 102 + .../date/BeamSqlIntervalMultiplyExpression.java | 103 + .../BeamSqlTimestampMinusIntervalExpression.java | 83 + .../BeamSqlTimestampMinusTimestampExpression.java | 97 + .../interpreter/operator/date/TimeUnitUtils.java | 54 + .../interpreter/operator/date/package-info.java | 22 + .../operator/logical/BeamSqlAndExpression.java | 48 + .../operator/logical/BeamSqlLogicalExpression.java | 46 + .../operator/logical/BeamSqlNotExpression.java | 54 + .../operator/logical/BeamSqlOrExpression.java | 48 + .../interpreter/operator/logical/package-info.java | 22 + .../operator/math/BeamSqlAbsExpression.java | 74 + .../operator/math/BeamSqlAcosExpression.java | 40 + .../operator/math/BeamSqlAsinExpression.java | 40 + .../operator/math/BeamSqlAtan2Expression.java | 42 + .../operator/math/BeamSqlAtanExpression.java | 40 + .../operator/math/BeamSqlCeilExpression.java | 45 + .../operator/math/BeamSqlCosExpression.java | 40 + .../operator/math/BeamSqlCotExpression.java | 40 + .../operator/math/BeamSqlDegreesExpression.java | 40 + .../operator/math/BeamSqlExpExpression.java | 40 + .../operator/math/BeamSqlFloorExpression.java | 45 + .../operator/math/BeamSqlLnExpression.java | 40 + .../operator/math/BeamSqlLogExpression.java | 40 + .../operator/math/BeamSqlMathBinaryExpression.java | 65 + .../operator/math/BeamSqlMathUnaryExpression.java | 60 + .../operator/math/BeamSqlPiExpression.java | 43 + .../operator/math/BeamSqlPowerExpression.java | 44 + .../operator/math/BeamSqlRadiansExpression.java | 40 + .../operator/math/BeamSqlRandExpression.java | 55 + .../math/BeamSqlRandIntegerExpression.java | 59 + .../operator/math/BeamSqlRoundExpression.java | 107 + .../operator/math/BeamSqlSignExpression.java | 72 + .../operator/math/BeamSqlSinExpression.java | 40 + .../operator/math/BeamSqlTanExpression.java | 40 + .../operator/math/BeamSqlTruncateExpression.java | 75 + .../interpreter/operator/math/package-info.java | 22 + .../impl/interpreter/operator/package-info.java | 22 + .../reinterpret/BeamSqlReinterpretExpression.java | 58 + .../DatetimeReinterpretConversions.java | 59 + .../reinterpret/IntegerReinterpretConversions.java | 44 + .../reinterpret/ReinterpretConversion.java | 114 + .../operator/reinterpret/Reinterpreter.java | 101 + .../operator/reinterpret/package-info.java | 22 + .../string/BeamSqlCharLengthExpression.java | 40 + .../operator/string/BeamSqlConcatExpression.java | 63 + .../operator/string/BeamSqlInitCapExpression.java | 56 + .../operator/string/BeamSqlLowerExpression.java | 40 + .../operator/string/BeamSqlOverlayExpression.java | 77 + .../operator/string/BeamSqlPositionExpression.java | 73 + .../string/BeamSqlStringUnaryExpression.java | 44 + .../string/BeamSqlSubstringExpression.java | 83 + .../operator/string/BeamSqlTrimExpression.java | 102 + .../operator/string/BeamSqlUpperExpression.java | 40 + .../interpreter/operator/string/package-info.java | 22 + .../sql/impl/interpreter/package-info.java | 22 + .../beam/sdk/extensions/sql/impl/package-info.java | 22 + .../extensions/sql/impl/parser/BeamSqlParser.java | 50 + .../sql/impl/parser/ColumnConstraint.java | 42 + .../sql/impl/parser/ColumnDefinition.java | 56 + .../extensions/sql/impl/parser/ParserUtils.java | 64 + .../extensions/sql/impl/parser/SqlCreateTable.java | 141 + .../extensions/sql/impl/parser/SqlDDLKeywords.java | 27 + .../extensions/sql/impl/parser/UnparseUtil.java | 59 + .../extensions/sql/impl/parser/package-info.java | 22 + .../sql/impl/planner/BeamQueryPlanner.java | 168 + .../sql/impl/planner/BeamRelDataTypeSystem.java | 40 + .../extensions/sql/impl/planner/BeamRuleSets.java | 75 + .../extensions/sql/impl/planner/package-info.java | 24 + .../sql/impl/rel/BeamAggregationRel.java | 182 + .../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 69 + .../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 75 + .../extensions/sql/impl/rel/BeamIOSourceRel.java | 62 + .../extensions/sql/impl/rel/BeamIntersectRel.java | 58 + .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 298 + .../sql/impl/rel/BeamLogicalConvention.java | 72 + .../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 56 + .../extensions/sql/impl/rel/BeamProjectRel.java | 80 + .../sdk/extensions/sql/impl/rel/BeamRelNode.java | 39 + .../sql/impl/rel/BeamSetOperatorRelBase.java | 98 + .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 235 + .../extensions/sql/impl/rel/BeamSqlRelUtils.java | 72 + .../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 88 + .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 79 + .../sdk/extensions/sql/impl/rel/package-info.java | 23 + .../sql/impl/rule/BeamAggregationRule.java | 162 + .../extensions/sql/impl/rule/BeamFilterRule.java | 49 + .../extensions/sql/impl/rule/BeamIOSinkRule.java | 81 + .../extensions/sql/impl/rule/BeamIOSourceRule.java | 49 + .../sql/impl/rule/BeamIntersectRule.java | 50 + .../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 53 + .../extensions/sql/impl/rule/BeamMinusRule.java | 50 + .../extensions/sql/impl/rule/BeamProjectRule.java | 50 + .../sdk/extensions/sql/impl/rule/BeamSortRule.java | 51 + .../extensions/sql/impl/rule/BeamUnionRule.java | 50 + .../extensions/sql/impl/rule/BeamValuesRule.java | 48 + .../sdk/extensions/sql/impl/rule/package-info.java | 23 + .../extensions/sql/impl/schema/BaseBeamTable.java | 36 + .../sdk/extensions/sql/impl/schema/BeamIOType.java | 28 + .../sql/impl/schema/BeamPCollectionTable.java | 63 + .../extensions/sql/impl/schema/BeamTableUtils.java | 118 + .../extensions/sql/impl/schema/package-info.java | 22 + .../impl/transform/BeamAggregationTransforms.java | 311 + .../impl/transform/BeamBuiltinAggregations.java | 557 ++ .../sql/impl/transform/BeamJoinTransforms.java | 161 + .../impl/transform/BeamSetOperatorsTransforms.java | 111 + .../sql/impl/transform/BeamSqlFilterFn.java | 63 + .../impl/transform/BeamSqlOutputToConsoleFn.java | 41 + .../sql/impl/transform/BeamSqlProjectFn.java | 72 + .../sql/impl/transform/package-info.java | 22 + .../extensions/sql/impl/utils/CalciteUtils.java | 113 + .../extensions/sql/impl/utils/SqlTypeUtils.java | 63 + .../extensions/sql/impl/utils/package-info.java | 22 + .../beam/sdk/extensions/sql/meta/Column.java | 51 + .../apache/beam/sdk/extensions/sql/meta/Table.java | 69 + .../beam/sdk/extensions/sql/meta/package-info.java | 22 + .../extensions/sql/meta/provider/MetaUtils.java | 40 + .../sql/meta/provider/TableProvider.java | 61 + .../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 111 + .../sql/meta/provider/kafka/BeamKafkaTable.java | 115 + .../meta/provider/kafka/KafkaTableProvider.java | 82 + .../sql/meta/provider/kafka/package-info.java | 22 + .../extensions/sql/meta/provider/package-info.java | 22 + .../sql/meta/provider/text/BeamTextCSVTable.java | 80 + .../provider/text/BeamTextCSVTableIOReader.java | 59 + .../provider/text/BeamTextCSVTableIOWriter.java | 59 + .../sql/meta/provider/text/BeamTextTable.java | 41 + .../sql/meta/provider/text/TextTableProvider.java | 83 + .../sql/meta/provider/text/package-info.java | 22 + .../sql/meta/store/InMemoryMetaStore.java | 113 + .../sdk/extensions/sql/meta/store/MetaStore.java | 56 + .../extensions/sql/meta/store/package-info.java | 22 + .../beam/sdk/extensions/sql/package-info.java | 22 + .../sql/src/main/resources/log4j.properties | 23 + ...ckaged.org.codehaus.commons.compiler.properties | 18 + .../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 62 + .../beam/sdk/extensions/sql/BeamSqlCliTest.java | 75 + .../extensions/sql/BeamSqlDslAggregationTest.java | 400 + .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 136 + .../sdk/extensions/sql/BeamSqlDslFilterTest.java | 155 + .../sdk/extensions/sql/BeamSqlDslJoinTest.java | 188 + .../sdk/extensions/sql/BeamSqlDslProjectTest.java | 227 + .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 139 + .../apache/beam/sdk/extensions/sql/TestUtils.java | 190 + .../impl/interpreter/BeamSqlFnExecutorTest.java | 460 + .../interpreter/BeamSqlFnExecutorTestBase.java | 88 + .../operator/BeamNullExperssionTest.java | 55 + .../operator/BeamSqlAndOrExpressionTest.java | 61 + .../operator/BeamSqlCaseExpressionTest.java | 93 + .../operator/BeamSqlCastExpressionTest.java | 129 + .../operator/BeamSqlCompareExpressionTest.java | 115 + .../operator/BeamSqlInputRefExpressionTest.java | 57 + .../interpreter/operator/BeamSqlPrimitiveTest.java | 59 + .../operator/BeamSqlReinterpretExpressionTest.java | 133 + .../operator/BeamSqlUdfExpressionTest.java | 51 + .../BeamSqlArithmeticExpressionTest.java | 237 + .../date/BeamSqlCurrentDateExpressionTest.java | 38 + .../date/BeamSqlCurrentTimeExpressionTest.java | 39 + .../BeamSqlCurrentTimestampExpressionTest.java | 39 + .../date/BeamSqlDateCeilExpressionTest.java | 50 + .../date/BeamSqlDateExpressionTestBase.java | 52 + .../date/BeamSqlDateFloorExpressionTest.java | 49 + .../date/BeamSqlDatetimeMinusExpressionTest.java | 133 + .../date/BeamSqlDatetimePlusExpressionTest.java | 155 + .../date/BeamSqlExtractExpressionTest.java | 103 + .../BeamSqlIntervalMultiplyExpressionTest.java | 107 + ...eamSqlTimestampMinusIntervalExpressionTest.java | 163 + ...amSqlTimestampMinusTimestampExpressionTest.java | 233 + .../operator/date/TimeUnitUtilsTest.java | 54 + .../operator/logical/BeamSqlNotExpressionTest.java | 47 + .../math/BeamSqlMathBinaryExpressionTest.java | 215 + .../math/BeamSqlMathUnaryExpressionTest.java | 312 + .../DatetimeReinterpretConversionsTest.java | 73 + .../IntegerReinterpretConversionsTest.java | 81 + .../reinterpret/ReinterpretConversionTest.java | 116 + .../operator/reinterpret/ReinterpreterTest.java | 155 + .../string/BeamSqlCharLengthExpressionTest.java | 44 + .../string/BeamSqlConcatExpressionTest.java | 66 + .../string/BeamSqlInitCapExpressionTest.java | 54 + .../string/BeamSqlLowerExpressionTest.java | 44 + .../string/BeamSqlOverlayExpressionTest.java | 87 + .../string/BeamSqlPositionExpressionTest.java | 84 + .../string/BeamSqlStringUnaryExpressionTest.java | 52 + .../string/BeamSqlSubstringExpressionTest.java | 101 + .../operator/string/BeamSqlTrimExpressionTest.java | 103 + .../string/BeamSqlUpperExpressionTest.java | 44 + .../sql/impl/parser/BeamSqlParserTest.java | 167 + .../sdk/extensions/sql/impl/rel/BaseRelTest.java | 34 + .../sql/impl/rel/BeamIntersectRelTest.java | 118 + .../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 203 + .../rel/BeamJoinRelUnboundedVsBoundedTest.java | 240 + .../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 218 + .../extensions/sql/impl/rel/BeamMinusRelTest.java | 117 + .../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 105 + .../extensions/sql/impl/rel/BeamSortRelTest.java | 257 + .../extensions/sql/impl/rel/BeamUnionRelTest.java | 103 + .../extensions/sql/impl/rel/BeamValuesRelTest.java | 104 + .../sdk/extensions/sql/impl/rel/CheckSize.java | 41 + .../sql/impl/schema/BeamSqlRowCoderTest.java | 77 + .../transform/BeamAggregationTransformTest.java | 453 + .../schema/transform/BeamTransformBaseTest.java | 97 + .../sql/impl/utils/SqlTypeUtilsTest.java | 76 + .../BeamSqlArithmeticOperatorsIntegrationTest.java | 165 + ...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 168 + .../BeamSqlComparisonOperatorsIntegrationTest.java | 329 + ...BeamSqlConditionalFunctionsIntegrationTest.java | 60 + .../BeamSqlDateFunctionsIntegrationTest.java | 195 + .../BeamSqlLogicalFunctionsIntegrationTest.java | 43 + .../BeamSqlMathFunctionsIntegrationTest.java | 351 + .../BeamSqlStringFunctionsIntegrationTest.java | 51 + .../meta/provider/kafka/BeamKafkaCSVTableTest.java | 107 + .../provider/kafka/KafkaTableProviderTest.java | 76 + .../meta/provider/text/BeamTextCSVTableTest.java | 176 + .../meta/provider/text/TextTableProviderTest.java | 87 + .../sql/meta/store/InMemoryMetaStoreTest.java | 185 + .../extensions/sql/mock/MockedBoundedTable.java | 134 + .../beam/sdk/extensions/sql/mock/MockedTable.java | 42 + .../extensions/sql/mock/MockedUnboundedTable.java | 110 + sdks/java/fn-execution/pom.xml | 115 + .../harness/channel/ManagedChannelFactory.java | 82 + .../beam/harness/channel/SocketAddressFactory.java | 64 + .../apache/beam/harness/channel/package-info.java | 22 + .../harness/channel/ManagedChannelFactoryTest.java | 71 + .../harness/channel/SocketAddressFactoryTest.java | 56 + .../org/apache/beam/harness/test/Consumer.java | 26 + .../org/apache/beam/harness/test/Supplier.java | 26 + .../apache/beam/harness/test/TestExecutors.java | 93 + .../beam/harness/test/TestExecutorsTest.java | 175 + .../org/apache/beam/harness/test/TestStreams.java | 185 + .../apache/beam/harness/test/TestStreamsTest.java | 109 + sdks/java/harness/pom.xml | 133 +- .../beam/fn/harness/BeamFnDataReadRunner.java | 171 + .../beam/fn/harness/BeamFnDataWriteRunner.java | 156 + .../beam/fn/harness/BoundedSourceRunner.java | 165 + .../apache/beam/fn/harness/FnApiDoFnRunner.java | 918 ++ .../java/org/apache/beam/fn/harness/FnHarness.java | 64 +- .../org/apache/beam/fn/harness/IdGenerator.java | 33 + .../beam/fn/harness/PTransformRunnerFactory.java | 84 + .../fn/harness/channel/ManagedChannelFactory.java | 80 - .../fn/harness/channel/SocketAddressFactory.java | 64 - .../beam/fn/harness/channel/package-info.java | 22 - .../fn/harness/control/BeamFnControlClient.java | 18 +- .../fn/harness/control/ProcessBundleHandler.java | 455 +- .../beam/fn/harness/control/RegisterHandler.java | 18 +- .../data/BeamFnDataBufferingOutboundObserver.java | 8 +- .../beam/fn/harness/data/BeamFnDataClient.java | 7 +- .../beam/fn/harness/data/BeamFnDataGrpcClient.java | 24 +- .../fn/harness/data/BeamFnDataGrpcMultiplexer.java | 33 +- .../fn/harness/data/BeamFnDataInboundObserver.java | 4 +- .../beam/fn/harness/fake/FakeStepContext.java | 70 - .../apache/beam/fn/harness/fake/package-info.java | 22 - .../beam/fn/harness/fn/ThrowingBiConsumer.java | 33 + .../fn/harness/logging/BeamFnLoggingClient.java | 238 +- .../apache/beam/fn/harness/state/BagUserState.java | 121 + .../beam/fn/harness/state/BeamFnStateClient.java | 39 + .../harness/state/BeamFnStateGrpcClientCache.java | 173 + .../state/LazyCachingIteratorToIterable.java | 72 + .../fn/harness/state/StateFetchingIterators.java | 126 + .../apache/beam/fn/harness/state/package-info.java | 22 + .../fn/harness/stream/BufferingStreamObserver.java | 16 +- .../apache/beam/fn/harness/stream/DataStreams.java | 229 + .../fn/harness/stream/StreamObserverFactory.java | 4 +- .../beam/runners/core/BeamFnDataReadRunner.java | 113 - .../beam/runners/core/BeamFnDataWriteRunner.java | 96 - .../beam/runners/core/BoundedSourceRunner.java | 105 - .../org/apache/beam/runners/core/package-info.java | 22 - .../beam/fn/harness/BeamFnDataReadRunnerTest.java | 280 + .../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 264 + .../beam/fn/harness/BoundedSourceRunnerTest.java | 184 + .../beam/fn/harness/FnApiDoFnRunnerTest.java | 408 + .../org/apache/beam/fn/harness/FnHarnessTest.java | 25 +- .../apache/beam/fn/harness/IdGeneratorTest.java | 40 + .../harness/channel/ManagedChannelFactoryTest.java | 74 - .../harness/channel/SocketAddressFactoryTest.java | 56 - .../harness/control/BeamFnControlClientTest.java | 13 +- .../harness/control/ProcessBundleHandlerTest.java | 777 +- .../fn/harness/control/RegisterHandlerTest.java | 34 +- .../BeamFnDataBufferingOutboundObserverTest.java | 4 +- .../fn/harness/data/BeamFnDataGrpcClientTest.java | 30 +- .../data/BeamFnDataGrpcMultiplexerTest.java | 9 +- .../data/BeamFnDataInboundObserverTest.java | 2 +- .../harness/logging/BeamFnLoggingClientTest.java | 136 +- .../beam/fn/harness/state/BagUserStateTest.java | 106 + .../state/BeamFnStateGrpcClientCacheTest.java | 234 + .../fn/harness/state/FakeBeamFnStateClient.java | 110 + .../state/LazyCachingIteratorToIterableTest.java | 76 + .../harness/state/StateFetchingIteratorsTest.java | 99 + .../stream/BufferingStreamObserverTest.java | 12 +- .../beam/fn/harness/stream/DataStreamsTest.java | 167 + .../harness/stream/DirectStreamObserverTest.java | 8 +- .../apache/beam/fn/harness/test/TestExecutors.java | 85 - .../beam/fn/harness/test/TestExecutorsTest.java | 160 - .../apache/beam/fn/harness/test/TestStreams.java | 162 - .../beam/fn/harness/test/TestStreamsTest.java | 84 - .../runners/core/BeamFnDataReadRunnerTest.java | 189 - .../runners/core/BeamFnDataWriteRunnerTest.java | 157 - .../beam/runners/core/BoundedSourceRunnerTest.java | 113 - sdks/java/io/amqp/pom.xml | 123 + .../java/org/apache/beam/sdk/io/amqp/AmqpIO.java | 380 + .../apache/beam/sdk/io/amqp/AmqpMessageCoder.java | 79 + .../io/amqp/AmqpMessageCoderProviderRegistrar.java | 44 + .../org/apache/beam/sdk/io/amqp/package-info.java | 22 + .../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 124 + .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 + sdks/java/io/cassandra/pom.xml | 113 + .../apache/beam/sdk/io/cassandra/CassandraIO.java | 486 + .../beam/sdk/io/cassandra/CassandraService.java | 66 + .../sdk/io/cassandra/CassandraServiceImpl.java | 398 + .../apache/beam/sdk/io/cassandra/package-info.java | 22 + .../beam/sdk/io/cassandra/CassandraIOIT.java | 254 + .../beam/sdk/io/cassandra/CassandraIOTest.java | 279 + .../sdk/io/cassandra/CassandraServiceImplTest.java | 138 + .../sdk/io/cassandra/CassandraTestDataSet.java | 153 + sdks/java/io/common/pom.xml | 12 +- .../beam/sdk/io/common/IOTestPipelineOptions.java | 20 +- .../org/apache/beam/sdk/io/common/TestRow.java | 114 + .../elasticsearch-tests-2/pom.xml | 60 + .../src/test/contrib/create_elk_container.sh | 0 .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 123 + .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 + .../elasticsearch-tests-5/pom.xml | 124 + .../src/test/contrib/create_elk_container.sh | 0 .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 122 + .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 + .../java/org/elasticsearch/bootstrap/JarHell.java | 39 + .../elasticsearch-tests-common/pom.xml | 77 + .../io/elasticsearch/ElasticSearchIOTestUtils.java | 141 + .../io/elasticsearch/ElasticsearchIOITCommon.java | 92 + .../elasticsearch/ElasticsearchIOTestCommon.java | 306 + sdks/java/io/elasticsearch-tests/pom.xml | 144 + sdks/java/io/elasticsearch/pom.xml | 224 +- .../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 479 +- .../beam/sdk/io/elasticsearch/package-info.java | 1 - .../io/elasticsearch/ElasticSearchIOTestUtils.java | 129 - .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 155 - .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 353 - .../io/elasticsearch/ElasticsearchTestDataSet.java | 111 - sdks/java/io/google-cloud-platform/pom.xml | 188 +- .../beam/sdk/io/gcp/bigquery/BatchLoads.java | 480 +- .../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 28 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 903 +- .../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 49 +- .../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 61 +- .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 121 +- .../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 153 +- .../io/gcp/bigquery/BigQueryTableRowIterator.java | 501 - .../sdk/io/gcp/bigquery/BigQueryTableSource.java | 40 +- .../beam/sdk/io/gcp/bigquery/CalculateSchemas.java | 78 - .../beam/sdk/io/gcp/bigquery/CreateTables.java | 23 +- .../sdk/io/gcp/bigquery/DynamicDestinations.java | 34 +- .../gcp/bigquery/DynamicDestinationsHelpers.java | 27 +- .../sdk/io/gcp/bigquery/GenerateShardedTable.java | 1 + .../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 86 + .../io/gcp/bigquery/PassThroughThenCleanup.java | 48 +- .../beam/sdk/io/gcp/bigquery/ReifyAsIterable.java | 51 + .../beam/sdk/io/gcp/bigquery/SchemaAndRecord.java | 43 + .../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 - .../beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java | 74 - .../beam/sdk/io/gcp/bigquery/StreamingInserts.java | 39 +- .../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 51 +- .../sdk/io/gcp/bigquery/StreamingWriteTables.java | 38 +- .../beam/sdk/io/gcp/bigquery/TableDestination.java | 43 +- .../sdk/io/gcp/bigquery/TableDestinationCoder.java | 2 + .../io/gcp/bigquery/TableDestinationCoderV2.java | 59 + .../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 1 + .../sdk/io/gcp/bigquery/TransformingSource.java | 136 - .../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 57 +- .../gcp/bigquery/WriteGroupedRecordsToFiles.java | 8 +- .../beam/sdk/io/gcp/bigquery/WritePartition.java | 21 +- .../beam/sdk/io/gcp/bigquery/WriteRename.java | 112 +- .../beam/sdk/io/gcp/bigquery/WriteResult.java | 27 +- .../beam/sdk/io/gcp/bigquery/WriteTables.java | 190 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 171 +- .../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 25 +- .../sdk/io/gcp/datastore/AdaptiveThrottler.java | 103 + .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 339 +- .../beam/sdk/io/gcp/datastore/MovingAverage.java | 50 + .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 53 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 6 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 43 +- .../sdk/io/gcp/spanner/CreateTransactionFn.java | 61 + .../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 + .../sdk/io/gcp/spanner/MutationGroupEncoder.java | 660 ++ .../sdk/io/gcp/spanner/MutationSizeEstimator.java | 155 + .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 + .../beam/sdk/io/gcp/spanner/OrderedCode.java | 764 ++ .../beam/sdk/io/gcp/spanner/ReadOperation.java | 96 + .../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 94 + .../sdk/io/gcp/spanner/SerializedMutation.java | 35 + .../io/gcp/spanner/SerializedMutationCoder.java | 60 + .../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 43 + .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 161 + .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 991 ++ .../beam/sdk/io/gcp/spanner/SpannerSchema.java | 144 + .../beam/sdk/io/gcp/spanner/Transaction.java | 33 + .../beam/sdk/io/gcp/spanner/package-info.java | 23 + .../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 18 +- .../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 1023 +- .../io/gcp/bigquery/BigQueryServicesImplTest.java | 88 +- .../gcp/bigquery/BigQueryTableRowIteratorTest.java | 358 - .../beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java | 198 +- .../sdk/io/gcp/bigquery/FakeBigQueryServices.java | 78 - .../sdk/io/gcp/bigquery/FakeDatasetService.java | 101 +- .../beam/sdk/io/gcp/bigquery/FakeJobService.java | 46 +- .../sdk/io/gcp/bigquery/InsertRetryPolicyTest.java | 79 + .../beam/sdk/io/gcp/bigquery/TableContainer.java | 2 + .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 30 +- .../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +- .../sdk/io/gcp/bigtable/BigtableTestOptions.java | 5 - .../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +- .../io/gcp/datastore/AdaptiveThrottlerTest.java | 111 + .../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 326 +- .../beam/sdk/io/gcp/datastore/SplitQueryFnIT.java | 5 +- .../apache/beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +- .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 17 +- .../beam/sdk/io/gcp/datastore/V1WriteIT.java | 36 +- .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 55 + .../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 + .../io/gcp/spanner/MutationGroupEncoderTest.java | 636 ++ .../io/gcp/spanner/MutationSizeEstimatorTest.java | 150 + .../beam/sdk/io/gcp/spanner/OrderedCodeTest.java | 890 ++ .../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 + .../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 134 + .../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 263 + .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 497 + .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 + .../beam/sdk/io/gcp/spanner/SpannerSchemaTest.java | 61 + .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 170 + sdks/java/io/hadoop-common/pom.xml | 2 +- .../sdk/io/hadoop/SerializableConfiguration.java | 18 +- sdks/java/io/hadoop-file-system/pom.xml | 47 +- .../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 27 +- .../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 95 +- sdks/java/io/hadoop/input-format/pom.xml | 7 +- .../io/hadoop/inputformat/HadoopInputFormatIO.java | 105 +- .../inputformat/HadoopInputFormatIOTest.java | 94 +- sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 5 +- .../hadoop/inputformat/HIFIOWithElasticTest.java | 21 +- sdks/java/io/hadoop/pom.xml | 2 +- sdks/java/io/hbase/pom.xml | 26 +- .../sdk/io/hbase/HBaseCoderProviderRegistrar.java | 38 + .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 1020 +- .../beam/sdk/io/hbase/HBaseMutationCoder.java | 41 + .../apache/beam/sdk/io/hbase/HBaseResultCoder.java | 6 +- .../apache/beam/sdk/io/hbase/SerializableScan.java | 37 +- .../io/hbase/HBaseCoderProviderRegistrarTest.java | 43 + .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 740 +- .../beam/sdk/io/hbase/HBaseMutationCoderTest.java | 4 +- .../beam/sdk/io/hbase/HBaseResultCoderTest.java | 4 +- .../beam/sdk/io/hbase/SerializableScanTest.java | 6 +- sdks/java/io/hcatalog/pom.xml | 179 + .../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 479 + .../apache/beam/sdk/io/hcatalog/package-info.java | 22 + .../sdk/io/hcatalog/EmbeddedMetastoreService.java | 87 + .../beam/sdk/io/hcatalog/HCatalogIOTest.java | 277 + .../beam/sdk/io/hcatalog/HCatalogIOTestUtils.java | 108 + .../io/hcatalog/src/test/resources/hive-site.xml | 301 + sdks/java/io/jdbc/pom.xml | 196 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 322 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 203 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 121 +- .../apache/beam/sdk/io/jdbc/JdbcTestDataSet.java | 130 - .../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 81 + sdks/java/io/jms/pom.xml | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 80 +- .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 78 + sdks/java/io/kafka/README.md | 36 + sdks/java/io/kafka/pom.xml | 31 +- .../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 56 + .../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 956 +- .../org/apache/beam/sdk/io/kafka/ProducerSpEL.java | 135 + .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 385 +- sdks/java/io/kinesis/pom.xml | 18 +- .../beam/sdk/io/kinesis/AWSClientsProvider.java | 36 + .../beam/sdk/io/kinesis/CheckpointGenerator.java | 6 +- .../apache/beam/sdk/io/kinesis/CustomOptional.java | 111 +- .../sdk/io/kinesis/DynamicCheckpointGenerator.java | 52 +- .../sdk/io/kinesis/GetKinesisRecordsResult.java | 55 +- .../beam/sdk/io/kinesis/KinesisClientProvider.java | 31 - .../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 348 +- .../apache/beam/sdk/io/kinesis/KinesisReader.java | 274 +- .../sdk/io/kinesis/KinesisReaderCheckpoint.java | 83 +- .../apache/beam/sdk/io/kinesis/KinesisRecord.java | 177 +- .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 68 +- .../apache/beam/sdk/io/kinesis/KinesisSource.java | 161 +- .../apache/beam/sdk/io/kinesis/RecordFilter.java | 18 +- .../org/apache/beam/sdk/io/kinesis/RoundRobin.java | 53 - .../beam/sdk/io/kinesis/ShardCheckpoint.java | 235 +- .../beam/sdk/io/kinesis/ShardReadersPool.java | 162 + .../beam/sdk/io/kinesis/ShardRecordsIterator.java | 120 +- .../sdk/io/kinesis/SimplifiedKinesisClient.java | 291 +- .../apache/beam/sdk/io/kinesis/StartingPoint.java | 84 +- .../sdk/io/kinesis/StaticCheckpointGenerator.java | 27 +- .../sdk/io/kinesis/TransientKinesisException.java | 9 +- .../beam/sdk/io/kinesis/AmazonKinesisMock.java | 532 +- .../beam/sdk/io/kinesis/CustomOptionalTest.java | 27 +- .../io/kinesis/DynamicCheckpointGeneratorTest.java | 33 +- .../beam/sdk/io/kinesis/KinesisMockReadTest.java | 98 +- .../io/kinesis/KinesisReaderCheckpointTest.java | 52 +- .../beam/sdk/io/kinesis/KinesisReaderIT.java | 128 +- .../beam/sdk/io/kinesis/KinesisReaderTest.java | 254 +- .../sdk/io/kinesis/KinesisRecordCoderTest.java | 34 +- .../beam/sdk/io/kinesis/KinesisTestOptions.java | 43 +- .../beam/sdk/io/kinesis/KinesisUploader.java | 70 +- .../beam/sdk/io/kinesis/RecordFilterTest.java | 52 +- .../apache/beam/sdk/io/kinesis/RoundRobinTest.java | 57 - .../beam/sdk/io/kinesis/ShardCheckpointTest.java | 203 +- .../beam/sdk/io/kinesis/ShardReadersPoolTest.java | 185 + .../sdk/io/kinesis/ShardRecordsIteratorTest.java | 221 +- .../io/kinesis/SimplifiedKinesisClientTest.java | 420 +- sdks/java/io/mongodb/pom.xml | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 8 +- .../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 321 +- .../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +- .../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 48 + sdks/java/io/mqtt/pom.xml | 2 +- .../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 53 +- sdks/java/io/pom.xml | 66 +- sdks/java/io/redis/pom.xml | 90 + .../sdk/io/redis/RedisConnectionConfiguration.java | 122 + .../java/org/apache/beam/sdk/io/redis/RedisIO.java | 451 + .../org/apache/beam/sdk/io/redis/package-info.java | 22 + .../org/apache/beam/sdk/io/redis/RedisIOTest.java | 109 + sdks/java/io/solr/pom.xml | 145 + .../beam/sdk/io/solr/AuthorizedSolrClient.java | 91 + .../apache/beam/sdk/io/solr/JavaBinCodecCoder.java | 98 + .../java/org/apache/beam/sdk/io/solr/SolrIO.java | 705 ++ .../org/apache/beam/sdk/io/solr/package-info.java | 20 + .../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 81 + .../org/apache/beam/sdk/io/solr/SolrIOTest.java | 269 + .../apache/beam/sdk/io/solr/SolrIOTestUtils.java | 132 + .../test/resources/cloud-minimal/conf/schema.xml | 29 + .../resources/cloud-minimal/conf/solrconfig.xml | 48 + sdks/java/io/tika/pom.xml | 113 + .../org/apache/beam/sdk/io/tika/ParseResult.java | 144 + .../java/org/apache/beam/sdk/io/tika/TikaIO.java | 284 + .../org/apache/beam/sdk/io/tika/package-info.java | 22 + .../apache/beam/sdk/io/tika/ParseResultTest.java | 83 + .../org/apache/beam/sdk/io/tika/TikaIOTest.java | 149 + sdks/java/io/tika/src/test/resources/damaged.pdf | 2 + .../test/resources/valid/apache-beam-tika-pdf.zip | Bin 0 -> 11685 bytes .../src/test/resources/valid/apache-beam-tika.odt | Bin 0 -> 12540 bytes sdks/java/io/xml/pom.xml | 2 +- .../java/org/apache/beam/sdk/io/xml/XmlIO.java | 375 +- .../java/org/apache/beam/sdk/io/xml/XmlSink.java | 33 +- .../java/org/apache/beam/sdk/io/xml/XmlSource.java | 65 +- .../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 16 +- .../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 74 +- sdks/java/java8tests/pom.xml | 2 +- sdks/java/javadoc/ant.xml | 1 + sdks/java/javadoc/pom.xml | 44 +- .../examples-java8/generate-sources.sh | 2 +- sdks/java/maven-archetypes/examples-java8/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 65 +- .../maven-archetypes/examples/generate-sources.sh | 2 +- sdks/java/maven-archetypes/examples/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 49 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 12 +- .../resources/projects/basic/reference/pom.xml | 12 +- sdks/java/nexmark/pom.xml | 265 + .../java/org/apache/beam/sdk/nexmark/Main.java | 303 + .../java/org/apache/beam/sdk/nexmark/Monitor.java | 78 + .../beam/sdk/nexmark/NexmarkConfiguration.java | 721 ++ .../apache/beam/sdk/nexmark/NexmarkLauncher.java | 1157 +++ .../apache/beam/sdk/nexmark/NexmarkOptions.java | 403 + .../org/apache/beam/sdk/nexmark/NexmarkPerf.java | 207 + .../org/apache/beam/sdk/nexmark/NexmarkSuite.java | 112 + .../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 674 ++ .../org/apache/beam/sdk/nexmark/model/Auction.java | 187 + .../apache/beam/sdk/nexmark/model/AuctionBid.java | 85 + .../beam/sdk/nexmark/model/AuctionCount.java | 84 + .../beam/sdk/nexmark/model/AuctionPrice.java | 88 + .../org/apache/beam/sdk/nexmark/model/Bid.java | 177 + .../beam/sdk/nexmark/model/BidsPerSession.java | 87 + .../beam/sdk/nexmark/model/CategoryPrice.java | 97 + .../org/apache/beam/sdk/nexmark/model/Done.java | 80 + .../org/apache/beam/sdk/nexmark/model/Event.java | 171 + .../beam/sdk/nexmark/model/IdNameReserve.java | 98 + .../apache/beam/sdk/nexmark/model/KnownSize.java | 26 + .../beam/sdk/nexmark/model/NameCityStateId.java | 103 + .../org/apache/beam/sdk/nexmark/model/Person.java | 163 + .../apache/beam/sdk/nexmark/model/SellerPrice.java | 89 + .../beam/sdk/nexmark/model/package-info.java | 22 + .../org/apache/beam/sdk/nexmark/package-info.java | 21 + .../sdk/nexmark/queries/AbstractSimulator.java | 211 + .../beam/sdk/nexmark/queries/NexmarkQuery.java | 270 + .../sdk/nexmark/queries/NexmarkQueryModel.java | 117 + .../apache/beam/sdk/nexmark/queries/Query0.java | 70 + .../beam/sdk/nexmark/queries/Query0Model.java | 64 + .../apache/beam/sdk/nexmark/queries/Query1.java | 67 + .../apache/beam/sdk/nexmark/queries/Query10.java | 367 + .../apache/beam/sdk/nexmark/queries/Query11.java | 79 + .../apache/beam/sdk/nexmark/queries/Query12.java | 80 + .../beam/sdk/nexmark/queries/Query1Model.java | 76 + .../apache/beam/sdk/nexmark/queries/Query2.java | 79 + .../beam/sdk/nexmark/queries/Query2Model.java | 80 + .../apache/beam/sdk/nexmark/queries/Query3.java | 301 + .../beam/sdk/nexmark/queries/Query3Model.java | 124 + .../apache/beam/sdk/nexmark/queries/Query4.java | 116 + .../beam/sdk/nexmark/queries/Query4Model.java | 186 + .../apache/beam/sdk/nexmark/queries/Query5.java | 138 + .../beam/sdk/nexmark/queries/Query5Model.java | 176 + .../apache/beam/sdk/nexmark/queries/Query6.java | 155 + .../beam/sdk/nexmark/queries/Query6Model.java | 133 + .../apache/beam/sdk/nexmark/queries/Query7.java | 90 + .../beam/sdk/nexmark/queries/Query7Model.java | 130 + .../apache/beam/sdk/nexmark/queries/Query8.java | 98 + .../beam/sdk/nexmark/queries/Query8Model.java | 148 + .../apache/beam/sdk/nexmark/queries/Query9.java | 44 + .../beam/sdk/nexmark/queries/Query9Model.java | 44 + .../beam/sdk/nexmark/queries/WinningBids.java | 418 + .../sdk/nexmark/queries/WinningBidsSimulator.java | 206 + .../beam/sdk/nexmark/queries/package-info.java | 22 + .../sdk/nexmark/sources/BoundedEventSource.java | 190 + .../apache/beam/sdk/nexmark/sources/Generator.java | 609 ++ .../beam/sdk/nexmark/sources/GeneratorConfig.java | 298 + .../sdk/nexmark/sources/UnboundedEventSource.java | 329 + .../beam/sdk/nexmark/sources/package-info.java | 22 + .../nexmark/src/main/resources/log4j.properties | 55 + .../apache/beam/sdk/nexmark/queries/QueryTest.java | 185 + .../nexmark/sources/BoundedEventSourceTest.java | 70 + .../beam/sdk/nexmark/sources/GeneratorTest.java | 110 + .../nexmark/sources/UnboundedEventSourceTest.java | 105 + sdks/java/pom.xml | 5 +- sdks/pom.xml | 6 +- sdks/python/MANIFEST.in | 1 + sdks/python/apache_beam/__init__.py | 47 +- sdks/python/apache_beam/coders/coder_impl.py | 26 +- sdks/python/apache_beam/coders/coders.py | 134 +- sdks/python/apache_beam/coders/coders_test.py | 2 +- .../apache_beam/coders/coders_test_common.py | 21 +- sdks/python/apache_beam/coders/observable_test.py | 1 - .../coders/proto2_coder_test_messages_pb2.py | 6 +- .../apache_beam/coders/standard_coders_test.py | 11 +- sdks/python/apache_beam/coders/stream.pxd | 2 +- sdks/python/apache_beam/coders/stream.pyx | 2 +- sdks/python/apache_beam/coders/stream_test.py | 1 - sdks/python/apache_beam/coders/typecoders.py | 1 - .../apache_beam/examples/complete/autocomplete.py | 21 +- .../examples/complete/autocomplete_test.py | 31 +- .../apache_beam/examples/complete/estimate_pi.py | 16 +- .../examples/complete/estimate_pi_test.py | 14 +- .../examples/complete/game/game_stats.py | 393 + .../examples/complete/game/game_stats_test.py | 81 + .../examples/complete/game/hourly_team_score.py | 256 +- .../examples/complete/game/leader_board.py | 349 + .../examples/complete/game/leader_board_test.py | 69 + .../examples/complete/game/user_score.py | 195 +- .../complete/juliaset/juliaset/juliaset.py | 48 +- .../complete/juliaset/juliaset/juliaset_test.py | 6 +- .../examples/complete/juliaset/juliaset_main.py | 2 - .../examples/complete/juliaset/setup.py | 7 +- sdks/python/apache_beam/examples/complete/tfidf.py | 38 +- .../apache_beam/examples/complete/tfidf_test.py | 38 +- .../examples/complete/top_wikipedia_sessions.py | 17 +- .../complete/top_wikipedia_sessions_test.py | 10 +- .../examples/cookbook/bigquery_schema.py | 159 +- .../examples/cookbook/bigquery_side_input.py | 56 +- .../examples/cookbook/bigquery_side_input_test.py | 39 +- .../examples/cookbook/bigquery_tornadoes.py | 28 +- .../cookbook/bigquery_tornadoes_it_test.py | 14 +- .../examples/cookbook/bigquery_tornadoes_test.py | 19 +- .../python/apache_beam/examples/cookbook/coders.py | 16 +- .../apache_beam/examples/cookbook/coders_test.py | 14 +- .../examples/cookbook/custom_ptransform.py | 29 +- .../examples/cookbook/custom_ptransform_test.py | 11 +- .../examples/cookbook/datastore_wordcount.py | 57 +- .../apache_beam/examples/cookbook/filters.py | 21 +- .../examples/cookbook/group_with_coder.py | 47 +- .../examples/cookbook/group_with_coder_test.py | 10 +- .../apache_beam/examples/cookbook/mergecontacts.py | 121 +- .../examples/cookbook/mergecontacts_test.py | 6 +- .../examples/cookbook/multiple_output_pardo.py | 84 +- .../cookbook/multiple_output_pardo_test.py | 13 +- .../apache_beam/examples/snippets/snippets.py | 576 +- .../apache_beam/examples/snippets/snippets_test.py | 470 +- .../apache_beam/examples/streaming_wordcap.py | 22 +- .../apache_beam/examples/streaming_wordcount.py | 66 +- .../apache_beam/examples/windowed_wordcount.py | 96 + sdks/python/apache_beam/examples/wordcount.py | 13 +- .../apache_beam/examples/wordcount_debugging.py | 65 +- .../examples/wordcount_debugging_test.py | 3 +- .../python/apache_beam/examples/wordcount_fnapi.py | 151 + .../apache_beam/examples/wordcount_it_test.py | 16 + .../apache_beam/examples/wordcount_minimal.py | 35 +- .../apache_beam/examples/wordcount_minimal_test.py | 3 +- sdks/python/apache_beam/examples/wordcount_test.py | 3 +- sdks/python/apache_beam/internal/gcp/auth.py | 1 - sdks/python/apache_beam/internal/gcp/json_value.py | 46 +- .../apache_beam/internal/gcp/json_value_test.py | 3 +- sdks/python/apache_beam/internal/pickler.py | 2 +- sdks/python/apache_beam/internal/util.py | 6 +- sdks/python/apache_beam/io/avroio.py | 164 +- sdks/python/apache_beam/io/avroio_test.py | 54 +- sdks/python/apache_beam/io/concat_source_test.py | 1 - sdks/python/apache_beam/io/filebasedsink.py | 22 +- sdks/python/apache_beam/io/filebasedsink_test.py | 19 +- sdks/python/apache_beam/io/filebasedsource.py | 234 +- sdks/python/apache_beam/io/filebasedsource_test.py | 8 +- sdks/python/apache_beam/io/filesystem.py | 78 +- sdks/python/apache_beam/io/filesystem_test.py | 5 +- sdks/python/apache_beam/io/filesystems.py | 1 - sdks/python/apache_beam/io/filesystems_test.py | 4 +- sdks/python/apache_beam/io/gcp/bigquery.py | 458 +- sdks/python/apache_beam/io/gcp/bigquery_test.py | 266 +- .../io/gcp/datastore/v1/adaptive_throttler.py | 94 + .../io/gcp/datastore/v1/adaptive_throttler_test.py | 95 + .../apache_beam/io/gcp/datastore/v1/datastoreio.py | 136 +- .../io/gcp/datastore/v1/datastoreio_test.py | 78 +- .../apache_beam/io/gcp/datastore/v1/helper.py | 64 +- .../apache_beam/io/gcp/datastore/v1/helper_test.py | 27 +- .../python/apache_beam/io/gcp/datastore/v1/util.py | 95 + .../apache_beam/io/gcp/datastore/v1/util_test.py | 67 + .../apache_beam/io/gcp/gcsfilesystem_test.py | 1 + sdks/python/apache_beam/io/gcp/gcsio.py | 29 +- sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +- .../clients/bigquery/bigquery_v2_messages.py | 3 - .../clients/storage/storage_v1_messages.py | 3 - sdks/python/apache_beam/io/gcp/pubsub.py | 167 +- sdks/python/apache_beam/io/gcp/pubsub_test.py | 97 +- .../apache_beam/io/gcp/tests/bigquery_matcher.py | 6 +- .../io/gcp/tests/bigquery_matcher_test.py | 5 +- sdks/python/apache_beam/io/gcp/tests/utils.py | 63 + sdks/python/apache_beam/io/gcp/tests/utils_test.py | 110 + sdks/python/apache_beam/io/iobase.py | 110 +- sdks/python/apache_beam/io/localfilesystem_test.py | 4 +- sdks/python/apache_beam/io/range_trackers.py | 184 +- sdks/python/apache_beam/io/range_trackers_test.py | 223 +- sdks/python/apache_beam/io/source_test_utils.py | 95 +- .../apache_beam/io/source_test_utils_test.py | 2 +- sdks/python/apache_beam/io/sources_test.py | 1 - sdks/python/apache_beam/io/textio.py | 245 +- sdks/python/apache_beam/io/textio_test.py | 499 +- sdks/python/apache_beam/io/tfrecordio.py | 5 +- sdks/python/apache_beam/io/tfrecordio_test.py | 8 +- sdks/python/apache_beam/io/vcfio.py | 436 + sdks/python/apache_beam/io/vcfio_test.py | 519 + sdks/python/apache_beam/metrics/cells_test.py | 2 +- sdks/python/apache_beam/metrics/execution.py | 5 +- sdks/python/apache_beam/metrics/execution_test.py | 8 +- sdks/python/apache_beam/metrics/metric.py | 3 +- sdks/python/apache_beam/metrics/metric_test.py | 18 +- sdks/python/apache_beam/metrics/metricbase.py | 4 + .../python/apache_beam/options/pipeline_options.py | 66 +- .../apache_beam/options/pipeline_options_test.py | 46 +- .../options/pipeline_options_validator_test.py | 11 +- sdks/python/apache_beam/options/value_provider.py | 2 - .../apache_beam/options/value_provider_test.py | 93 +- sdks/python/apache_beam/pipeline.py | 385 +- sdks/python/apache_beam/pipeline_test.py | 143 +- sdks/python/apache_beam/portability/__init__.py | 18 + .../python/apache_beam/portability/api/__init__.py | 21 + sdks/python/apache_beam/pvalue.py | 111 +- sdks/python/apache_beam/pvalue_test.py | 8 + sdks/python/apache_beam/runners/api/__init__.py | 32 - .../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 ----- .../runners/api/beam_fn_api_pb2_grpc.py | 205 - .../apache_beam/runners/api/beam_runner_api_pb2.py | 2872 ------ sdks/python/apache_beam/runners/common.py | 6 +- sdks/python/apache_beam/runners/common_test.py | 2 +- .../runners/dataflow/dataflow_metrics.py | 2 +- .../runners/dataflow/dataflow_runner.py | 320 +- .../runners/dataflow/dataflow_runner_test.py | 142 +- .../runners/dataflow/internal/apiclient.py | 148 +- .../runners/dataflow/internal/apiclient_test.py | 135 +- .../clients/dataflow/dataflow_v1b3_client.py | 259 +- .../clients/dataflow/dataflow_v1b3_messages.py | 92 +- .../internal/clients/dataflow/message_matchers.py | 1 - .../clients/dataflow/message_matchers_test.py | 3 +- .../runners/dataflow/internal/dependency.py | 127 +- .../runners/dataflow/internal/dependency_test.py | 22 +- .../apache_beam/runners/dataflow/internal/names.py | 2 + .../runners/dataflow/native_io/iobase.py | 6 +- .../runners/dataflow/native_io/iobase_test.py | 59 +- .../runners/dataflow/native_io/streaming_create.py | 72 + .../runners/dataflow/ptransform_overrides.py | 52 + .../runners/dataflow/template_runner_test.py | 5 +- .../runners/dataflow/test_dataflow_runner.py | 10 +- .../apache_beam/runners/direct/bundle_factory.py | 2 +- .../consumer_tracking_pipeline_visitor_test.py | 2 +- .../apache_beam/runners/direct/direct_metrics.py | 2 +- .../runners/direct/direct_metrics_test.py | 8 +- .../apache_beam/runners/direct/direct_runner.py | 146 +- .../runners/direct/direct_runner_test.py | 41 + .../runners/direct/evaluation_context.py | 90 +- sdks/python/apache_beam/runners/direct/executor.py | 242 +- .../runners/direct/helper_transforms.py | 2 +- .../runners/direct/transform_evaluator.py | 462 +- .../apache_beam/runners/direct/transform_result.py | 41 - sdks/python/apache_beam/runners/direct/util.py | 70 + .../runners/direct/watermark_manager.py | 100 +- .../apache_beam/runners/experimental/__init__.py | 16 + .../experimental/python_rpc_direct/__init__.py | 22 + .../python_rpc_direct/python_rpc_direct_runner.py | 110 + .../experimental/python_rpc_direct/server.py | 111 + sdks/python/apache_beam/runners/job/__init__.py | 16 + sdks/python/apache_beam/runners/job/manager.py | 52 + sdks/python/apache_beam/runners/job/utils.py | 32 + .../python/apache_beam/runners/pipeline_context.py | 31 +- .../runners/portability/fn_api_runner.py | 1096 +- .../runners/portability/fn_api_runner_test.py | 134 +- .../runners/portability/maptask_executor_runner.py | 19 +- .../portability/maptask_executor_runner_test.py | 12 +- .../runners/portability/universal_local_runner.py | 409 + .../portability/universal_local_runner_main.py | 44 + .../portability/universal_local_runner_test.py | 86 + sdks/python/apache_beam/runners/runner.py | 61 +- sdks/python/apache_beam/runners/runner_test.py | 2 +- .../apache_beam/runners/worker/bundle_processor.py | 590 ++ .../apache_beam/runners/worker/data_plane.py | 71 +- .../apache_beam/runners/worker/data_plane_test.py | 9 +- .../apache_beam/runners/worker/log_handler.py | 17 +- .../apache_beam/runners/worker/log_handler_test.py | 22 +- .../apache_beam/runners/worker/opcounters.py | 10 +- .../apache_beam/runners/worker/opcounters_test.py | 1 - .../apache_beam/runners/worker/operation_specs.py | 20 +- .../apache_beam/runners/worker/operations.pxd | 4 +- .../apache_beam/runners/worker/operations.py | 87 +- .../apache_beam/runners/worker/sdk_worker.py | 560 +- .../apache_beam/runners/worker/sdk_worker_main.py | 30 +- .../apache_beam/runners/worker/sdk_worker_test.py | 106 +- .../apache_beam/runners/worker/statesampler.pyx | 68 +- .../runners/worker/statesampler_fake.py | 17 +- .../runners/worker/statesampler_test.py | 5 +- .../apache_beam/testing/data/vcf/valid-4.0.vcf | 23 + .../apache_beam/testing/data/vcf/valid-4.0.vcf.bz2 | Bin 0 -> 781 bytes .../apache_beam/testing/data/vcf/valid-4.0.vcf.gz | Bin 0 -> 727 bytes .../testing/data/vcf/valid-4.1-large.vcf | 10000 +++++++++++++++++++ .../testing/data/vcf/valid-4.1-large.vcf.gz | Bin 0 -> 156715 bytes .../apache_beam/testing/data/vcf/valid-4.2.vcf | 42 + .../apache_beam/testing/data/vcf/valid-4.2.vcf.gz | Bin 0 -> 1240 bytes .../apache_beam/testing/pipeline_verifiers.py | 1 - .../apache_beam/testing/pipeline_verifiers_test.py | 5 +- sdks/python/apache_beam/testing/test_pipeline.py | 54 +- .../apache_beam/testing/test_pipeline_test.py | 4 +- sdks/python/apache_beam/testing/test_stream.py | 6 +- .../python/apache_beam/testing/test_stream_test.py | 69 + sdks/python/apache_beam/testing/test_utils.py | 63 +- sdks/python/apache_beam/testing/test_utils_test.py | 86 + sdks/python/apache_beam/testing/util.py | 21 +- sdks/python/apache_beam/testing/util_test.py | 4 +- sdks/python/apache_beam/transforms/combiners.py | 17 +- .../apache_beam/transforms/combiners_test.py | 70 +- sdks/python/apache_beam/transforms/core.py | 561 +- sdks/python/apache_beam/transforms/create_test.py | 6 +- sdks/python/apache_beam/transforms/display.py | 91 +- sdks/python/apache_beam/transforms/display_test.py | 6 +- sdks/python/apache_beam/transforms/ptransform.py | 202 +- .../apache_beam/transforms/ptransform_test.py | 78 +- .../apache_beam/transforms/sideinputs_test.py | 5 +- sdks/python/apache_beam/transforms/timeutil.py | 1 - sdks/python/apache_beam/transforms/trigger.py | 79 +- sdks/python/apache_beam/transforms/trigger_test.py | 19 +- sdks/python/apache_beam/transforms/util.py | 273 +- sdks/python/apache_beam/transforms/util_test.py | 108 + sdks/python/apache_beam/transforms/window.py | 68 +- sdks/python/apache_beam/transforms/window_test.py | 158 +- .../transforms/write_ptransform_test.py | 11 +- sdks/python/apache_beam/typehints/decorators.py | 129 +- .../typehints/native_type_compatibility.py | 166 + .../typehints/native_type_compatibility_test.py | 92 + sdks/python/apache_beam/typehints/opcodes.py | 19 +- .../apache_beam/typehints/trivial_inference.py | 83 +- .../typehints/trivial_inference_test.py | 16 +- sdks/python/apache_beam/typehints/typecheck.py | 6 +- .../apache_beam/typehints/typed_pipeline_test.py | 55 +- sdks/python/apache_beam/typehints/typehints.py | 79 +- .../python/apache_beam/typehints/typehints_test.py | 3 +- sdks/python/apache_beam/utils/annotations_test.py | 1 + sdks/python/apache_beam/utils/counters.py | 107 +- sdks/python/apache_beam/utils/counters_test.py | 78 + sdks/python/apache_beam/utils/plugin.py | 42 + sdks/python/apache_beam/utils/processes_test.py | 1 - sdks/python/apache_beam/utils/proto_utils.py | 17 + sdks/python/apache_beam/utils/retry.py | 9 +- sdks/python/apache_beam/utils/retry_test.py | 5 +- sdks/python/apache_beam/utils/timestamp.py | 5 + sdks/python/apache_beam/utils/urns.py | 49 +- sdks/python/apache_beam/version.py | 2 +- sdks/python/container/Dockerfile | 27 + sdks/python/container/boot.go | 123 + sdks/python/container/pom.xml | 154 + sdks/python/gen_protos.py | 141 + sdks/python/generate_pydoc.sh | 140 +- sdks/python/pom.xml | 9 +- sdks/python/run_postcommit.sh | 20 - sdks/python/run_pylint.sh | 49 +- sdks/python/run_validatesrunner.sh | 71 + sdks/python/setup.py | 65 +- sdks/python/tox.ini | 32 +- 1777 files changed, 160041 insertions(+), 41448 deletions(-) -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
