This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch samza-runner in repository https://gitbox.apache.org/repos/asf/beam.git
commit 349419af55a1e6ad08502e6b71d7046e6727c9a9 Merge: 1840a58 e27bc31 Author: Kenn Knowles <[email protected]> AuthorDate: Tue May 29 21:12:34 2018 -0700 Merge pull request #5505: [BEAM-3079] Rebase Samza runner with master .github/PULL_REQUEST_TEMPLATE.md | 12 +- .gitignore | 8 +- .test-infra/jenkins/PreCommit_Pipeline.groovy | 129 - .test-infra/jenkins/common_job_properties.groovy | 190 +- .test-infra/jenkins/job_Inventory.groovy | 66 + ...groovy => job_PerformanceTests_Dataflow.groovy} | 0 .../job_PerformanceTests_FileBasedIO_IT.groovy | 146 + ...job_PerformanceTests_FileBasedIO_IT_HDFS.groovy | 153 + .../job_PerformanceTests_HadoopInputFormat.groovy | 68 + .../jenkins/job_PerformanceTests_JDBC.groovy | 68 + .../job_PerformanceTests_MongoDBIO_IT.groovy | 66 + .../jenkins/job_PerformanceTests_Python.groovy | 61 + .../jenkins/job_PerformanceTests_Spark.groovy | 49 + .../jenkins/job_PostCommit_Go_GradleBuild.groovy | 51 + .../jenkins/job_PostCommit_Java_GradleBuild.groovy | 56 + ...job_PostCommit_Java_ValidatesRunner_Apex.groovy | 52 + ...PostCommit_Java_ValidatesRunner_Dataflow.groovy | 58 + ...ob_PostCommit_Java_ValidatesRunner_Flink.groovy | 50 + ...PostCommit_Java_ValidatesRunner_Gearpump.groovy | 57 + ...ob_PostCommit_Java_ValidatesRunner_Spark.groovy | 52 + ...ommit_Python_ValidatesContainer_Dataflow.groovy | 42 + ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 46 + .../jenkins/job_PostCommit_Python_Verify.groovy | 47 + .../jenkins/job_PostRelease_NightlySnapshot.groovy | 63 + .../jenkins/job_PreCommit_Go_GradleBuild.groovy | 44 + .../jenkins/job_PreCommit_Java_GradleBuild.groovy | 49 + .../job_PreCommit_Python_GradleBuild.groovy | 50 + ...e.groovy => job_PreCommit_Website_Merge.groovy} | 0 ...e.groovy => job_PreCommit_Website_Stage.groovy} | 0 ...st.groovy => job_PreCommit_Website_Test.groovy} | 0 .../jenkins/job_ReleaseCandidate_Python.groovy | 42 + .../job_Release_Gradle_NightlySnapshot.groovy | 59 + .test-infra/jenkins/job_beam_Java_Build.groovy | 74 - .../jenkins/job_beam_Java_CodeHealth.groovy | 39 - .../jenkins/job_beam_Java_IntegrationTest.groovy | 63 - .test-infra/jenkins/job_beam_Java_UnitTest.groovy | 40 - .../job_beam_PerformanceTests_Analysis.groovy | 85 + ...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 119 - .../jenkins/job_beam_PerformanceTests_JDBC.groovy | 63 - .../job_beam_PerformanceTests_Python.groovy | 58 - .../jenkins/job_beam_PerformanceTests_Spark.groovy | 44 - ...job_beam_PostCommit_Java_JDKVersionsTest.groovy | 60 - .../job_beam_PostCommit_Java_MavenInstall.groovy | 65 - ...eam_PostCommit_Java_MavenInstall_Windows.groovy | 46 - ...eam_PostCommit_Java_ValidatesRunner_Apex.groovy | 48 - ...PostCommit_Java_ValidatesRunner_Dataflow.groovy | 45 - ...am_PostCommit_Java_ValidatesRunner_Flink.groovy | 43 - ...PostCommit_Java_ValidatesRunner_Gearpump.groovy | 49 - ...am_PostCommit_Java_ValidatesRunner_Spark.groovy | 44 - ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 54 - .../job_beam_PostCommit_Python_Verify.groovy | 55 - .../job_beam_PostRelease_NightlySnapshot.groovy | 52 - .../job_beam_PreCommit_Go_MavenInstall.groovy | 56 - .../job_beam_PreCommit_Java_GradleBuild.groovy | 59 - .../job_beam_PreCommit_Java_MavenInstall.groovy | 56 - .../jenkins/job_beam_PreCommit_Pipeline.groovy | 84 - .../job_beam_PreCommit_Python_MavenInstall.groovy | 57 - .../jenkins/job_beam_Python_UnitTest.groovy | 40 - .../job_beam_Release_NightlySnapshot.groovy | 61 - .test-infra/jenkins/test_wordcount.sh | 125 - .../jenkins/verify_performance_test_results.py | 324 + .../verify_performance_test_results_test.py | 89 + .../kubernetes/mongodb/load-balancer/mongo.yml | 49 + .../mongodb/load-balancer/pkb-config.yml | 32 + .test-infra/kubernetes/mongodb/node-port/mongo.yml | 50 + .../kubernetes/mongodb/node-port/pkb-config.yml | 30 + .../postgres/postgres-service-for-local-dev.yml | 27 + LICENSE | 99 + README.md | 11 +- assembly.xml | 112 + build.gradle | 307 +- build_rules.gradle | 1034 +- examples/java/README.md | 10 +- examples/java/build.gradle | 82 +- examples/java/pom.xml | 13 +- .../apache/beam/examples/DebuggingWordCount.java | 11 +- .../org/apache/beam/examples/MinimalWordCount.java | 4 +- .../apache/beam/examples/WindowedWordCount.java | 9 +- .../java/org/apache/beam/examples/WordCount.java | 11 +- .../apache/beam/examples/common/ExampleUtils.java | 63 +- .../beam/examples/complete/AutoComplete.java | 3 + .../org/apache/beam/examples/complete/TfIdf.java | 8 +- .../beam/examples/complete/TrafficRoutes.java | 2 +- .../beam/examples/complete/game/GameStats.java | 2 +- .../examples/complete/game/HourlyTeamScore.java | 2 +- .../beam/examples/complete/game/LeaderBoard.java | 2 +- .../examples/complete/game/StatefulTeamScore.java | 33 +- .../beam/examples/complete/game/UserScore.java | 3 +- .../examples/complete/game/injector/Injector.java | 31 +- .../complete/game/injector/InjectorUtils.java | 1 + .../complete/game/utils/WriteToBigQuery.java | 1 + .../beam/examples/cookbook/BigQueryTornadoes.java | 12 +- .../examples/cookbook/CombinePerKeyExamples.java | 2 +- .../beam/examples/cookbook/FilterExamples.java | 6 +- .../beam/examples/cookbook/TriggerExample.java | 9 +- .../apache/beam/examples/snippets/Snippets.java | 415 + .../org/apache/beam/examples/subprocess/Echo.cc | 36 + .../apache/beam/examples/subprocess/EchoAgain.cc | 36 + .../examples/subprocess/ExampleEchoPipeline.java | 135 + .../subprocess/SubProcessPipelineOptions.java | 97 + .../configuration/SubProcessConfiguration.java | 83 + .../kernel/SubProcessCommandLineArgs.java | 76 + .../subprocess/kernel/SubProcessIOFiles.java | 131 + .../subprocess/kernel/SubProcessKernel.java | 311 + .../subprocess/utils/CallingSubProcessUtils.java | 110 + .../examples/subprocess/utils/ExecutableFile.java | 77 + .../beam/examples/subprocess/utils/FileUtils.java | 164 + .../beam/examples/website_snippets/Snippets.java | 86 - .../beam/examples/DebuggingWordCountTest.java | 2 +- .../apache/beam/examples/WindowedWordCountIT.java | 30 +- .../java/org/apache/beam/examples/WordCountIT.java | 4 +- .../complete/game/StatefulTeamScoreTest.java | 2 - .../examples/cookbook/BigQueryTornadoesIT.java | 2 +- .../examples/cookbook/BigQueryTornadoesTest.java | 2 +- .../beam/examples/snippets/SnippetsTest.java | 142 + .../subprocess/ExampleEchoPipelineTest.java | 204 + .../examples/website_snippets/SnippetsTest.java | 113 - examples/pom.xml | 2 +- gradle.properties | 5 + gradle/wrapper/gradle-wrapper.jar | Bin 54712 -> 54731 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- model/fn-execution/build.gradle | 12 +- model/fn-execution/pom.xml | 12 +- .../fn-execution/src/main/proto/beam_fn_api.proto | 173 +- .../src/main/proto/beam_provision_api.proto | 5 +- .../beam/model/fnexecution/v1/standard_coders.yaml | 195 + .../beam/model/fnexecution/v1/standard_coders.yaml | 195 - model/job-management/build.gradle | 12 +- model/job-management/pom.xml | 30 +- model/pipeline/build.gradle | 10 +- model/pipeline/pom.xml | 12 +- .../pipeline/src/main/proto/beam_runner_api.proto | 222 +- .../src/main/proto/standard_window_fns.proto | 25 +- model/pom.xml | 2 +- pom.xml | 563 +- release/TestScripts.groovy | 140 - release/build.gradle | 40 + release/quickstart-java-direct.groovy | 64 - .../src/main/groovy/MobileGamingCommands.groovy | 144 + release/src/main/groovy/QuickstartArchetype.groovy | 44 + release/src/main/groovy/TestScripts.groovy | 220 + .../main/groovy/mobilegaming-java-dataflow.groovy | 111 + .../main/groovy/mobilegaming-java-direct.groovy | 105 + .../main/groovy/python_release_automation_utils.sh | 135 + .../src/main/groovy/quickstart-java-apex.groovy | 45 + .../main/groovy/quickstart-java-dataflow.groovy | 54 + .../src/main/groovy/quickstart-java-direct.groovy | 43 + .../main/groovy/quickstart-java-flinklocal.groovy | 43 + .../main/groovy/quickstart-java-gearpump.groovy | 43 + .../src/main/groovy/quickstart-java-spark.groovy | 43 + .../run_release_candidate_python_mobile_gaming.sh | 188 + .../run_release_candidate_python_quickstart.sh | 231 + runners/apex/build.gradle | 70 +- runners/apex/pom.xml | 19 +- .../org/apache/beam/runners/apex/ApexRunner.java | 18 +- .../apache/beam/runners/apex/ApexRunnerResult.java | 15 +- .../apache/beam/runners/apex/ApexYarnLauncher.java | 24 +- .../apache/beam/runners/apex/TestApexRunner.java | 8 +- .../translation/FlattenPCollectionTranslator.java | 3 +- .../runners/apex/translation/ParDoTranslator.java | 2 +- .../apex/translation/TranslationContext.java | 4 +- .../translation/operators/ApexParDoOperator.java | 2 +- .../operators/ApexProcessFnOperator.java | 2 +- .../translation/operators/ApexTimerInternals.java | 1 - .../apex/translation/utils/ApexStateInternals.java | 17 +- .../translation/utils/StateInternalsProxy.java | 5 +- .../apache/beam/runners/apex/ApexRunnerTest.java | 30 +- .../beam/runners/apex/ApexYarnLauncherTest.java | 2 +- .../beam/runners/apex/examples/WordCountTest.java | 14 +- .../apex/translation/ParDoTranslatorTest.java | 2 + .../apex/translation/SideInputTranslationTest.java | 2 +- .../operators/ApexTimerInternalsTest.java | 2 +- .../translation/utils/ApexStateInternalsTest.java | 6 - runners/core-construction-java/build.gradle | 31 +- runners/core-construction-java/pom.xml | 53 +- .../core/construction/ArtifactServiceStager.java | 80 +- .../beam/runners/core/construction/BeamUrns.java | 29 + .../core/construction/CoderTranslation.java | 91 +- .../runners/core/construction/CoderTranslator.java | 12 +- .../construction/CoderTranslatorRegistrar.java | 37 + .../core/construction/CoderTranslators.java | 20 +- .../core/construction/CombineTranslation.java | 131 +- .../runners/core/construction/Environments.java | 98 +- .../construction/ExecutableStageTranslation.java | 43 + .../core/construction/ImpulseTranslation.java | 65 + .../core/construction/JavaReadViaImpulse.java | 176 + .../core/construction/ModelCoderRegistrar.java | 97 + .../runners/core/construction/ModelCoders.java | 115 + .../construction/PCollectionViewTranslation.java | 12 +- .../core/construction/PTransformTranslation.java | 60 +- .../core/construction/ParDoTranslation.java | 109 +- .../runners/core/construction/SplittableParDo.java | 121 +- .../core/construction/SyntheticComponents.java | 44 + .../UnboundedReadFromBoundedSource.java | 11 +- .../core/construction/WindowIntoTranslation.java | 10 +- .../construction/WindowingStrategyTranslation.java | 86 +- .../core/construction/WriteFilesTranslation.java | 4 +- .../core/construction/graph/ExecutableStage.java | 204 + .../core/construction/graph/FusedPipeline.java | 114 + .../graph/GreedyPCollectionFusers.java | 309 + .../construction/graph/GreedyPipelineFuser.java | 372 + .../core/construction/graph/GreedyStageFuser.java | 183 + .../graph/ImmutableExecutableStage.java | 86 + .../runners/core/construction/graph/Networks.java | 289 + .../construction/graph/OutputDeduplicator.java | 346 + .../core/construction/graph/PipelineNode.java | 56 + .../core/construction/graph/ProtoOverrides.java | 102 + .../core/construction/graph/QueryablePipeline.java | 316 + .../construction/graph/SideInputReference.java | 61 + .../core/construction/graph/package-info.java | 24 + .../core/construction/metrics/MetricFiltering.java | 4 +- .../construction/ArtifactServiceStagerTest.java | 18 +- .../core/construction/CoderTranslationTest.java | 23 +- .../core/construction/CombineTranslationTest.java | 50 +- .../runners/core/construction/CommonCoderTest.java | 355 + .../CreatePCollectionViewTranslationTest.java | 160 +- .../core/construction/EnvironmentsTest.java | 226 + .../InMemoryArtifactStagerService.java | 19 +- .../core/construction/JavaReadViaImpulseTest.java | 181 + .../runners/core/construction/ModelCodersTest.java | 120 + .../construction/PCollectionTranslationTest.java | 35 +- .../PCollectionViewTranslationTest.java | 74 + .../core/construction/PTransformMatchersTest.java | 7 +- .../core/construction/ParDoTranslationTest.java | 15 +- .../PipelineOptionsTranslationTest.java | 23 +- .../core/construction/PipelineTranslationTest.java | 2 +- .../core/construction/ReadTranslationTest.java | 2 +- .../SingleInputOutputOverrideFactoryTest.java | 5 +- .../core/construction/SplittableParDoTest.java | 7 +- .../construction/TestStreamTranslationTest.java | 118 +- .../core/construction/UnconsumedReadsTest.java | 4 +- .../construction/WriteFilesTranslationTest.java | 109 +- .../construction/graph/ExecutableStageMatcher.java | 144 + .../construction/graph/ExecutableStageTest.java | 175 + .../core/construction/graph/FusedPipelineTest.java | 143 + .../graph/GreedyPipelineFuserTest.java | 1098 + .../construction/graph/GreedyStageFuserTest.java | 1037 + .../graph/ImmutableExecutableStageTest.java | 107 + .../core/construction/graph/NetworksTest.java | 293 + .../construction/graph/OutputDeduplicatorTest.java | 509 + .../construction/graph/ProtoOverridesTest.java | 257 + .../construction/graph/QueryablePipelineTest.java | 424 + .../construction/metrics/MetricFilteringTest.java | 2 - runners/core-java/build.gradle | 33 +- runners/core-java/pom.xml | 27 +- .../org/apache/beam/runners/core/DoFnRunner.java | 6 + .../org/apache/beam/runners/core/DoFnRunners.java | 4 +- .../runners/core/GroupByKeyViaGroupByKeyOnly.java | 5 +- .../beam/runners/core/InMemoryStateInternals.java | 83 +- .../runners/core/LateDataDroppingDoFnRunner.java | 43 +- .../beam/runners/core/MergingActiveWindowSet.java | 11 +- ...TimeBoundedSplittableProcessElementInvoker.java | 141 +- .../apache/beam/runners/core/ProcessFnRunner.java | 8 +- .../runners/core/PushbackSideInputDoFnRunner.java | 6 + .../beam/runners/core/ReduceFnContextFactory.java | 18 +- .../apache/beam/runners/core/ReduceFnRunner.java | 15 +- .../apache/beam/runners/core/SimpleDoFnRunner.java | 152 +- .../core/SimplePushbackSideInputDoFnRunner.java | 8 +- .../core/SplittableParDoViaKeyedWorkItems.java | 4 +- .../core/SplittableProcessElementInvoker.java | 2 +- .../apache/beam/runners/core/StateInternals.java | 5 +- .../org/apache/beam/runners/core/StateMerging.java | 95 - .../org/apache/beam/runners/core/StateTags.java | 2 +- .../beam/runners/core/StatefulDoFnRunner.java | 5 + .../apache/beam/runners/core/TimerInternals.java | 10 +- .../apache/beam/runners/core/WatermarkHold.java | 74 +- .../beam/runners/core/metrics/CounterCell.java | 3 + .../runners/core/metrics/MetricsContainerImpl.java | 7 +- .../core/metrics/MetricsContainerStepMap.java | 20 +- .../beam/runners/core/metrics/MetricsPusher.java | 121 + .../runners/core/metrics/MetricsTranslation.java | 144 + .../beam/runners/core/metrics/NoOpMetricsSink.java | 34 + .../runners/core/triggers/TriggerStateMachine.java | 8 +- .../TriggerStateMachineContextFactory.java | 3 - .../runners/core/InMemoryStateInternalsTest.java | 6 - .../core/LateDataDroppingDoFnRunnerTest.java | 22 +- ...BoundedSplittableProcessElementInvokerTest.java | 111 +- .../beam/runners/core/ReduceFnRunnerTest.java | 327 +- .../apache/beam/runners/core/ReduceFnTester.java | 49 +- .../SimplePushbackSideInputDoFnRunnerTest.java | 6 + .../runners/core/SplittableParDoProcessFnTest.java | 37 +- .../beam/runners/core/StateInternalsTest.java | 49 +- .../beam/runners/core/StatefulDoFnRunnerTest.java | 11 +- .../beam/runners/core/TimerInternalsTest.java | 82 +- .../runners/core/metrics/MetricUpdateMatchers.java | 6 +- .../core/metrics/MetricsContainerImplTest.java | 2 +- .../core/metrics/MetricsContainerStepMapTest.java | 50 +- .../runners/core/metrics/MetricsPusherTest.java | 86 + .../core/metrics/MetricsTranslationTest.java | 191 + .../beam/runners/core/metrics/TestMetricsSink.java | 50 + .../core/triggers/TriggerStateMachineTester.java | 9 +- runners/direct-java/build.gradle | 139 +- runners/direct-java/pom.xml | 86 +- .../direct/BoundedReadEvaluatorFactory.java | 34 +- .../beam/runners/direct/BundleProcessor.java | 3 +- .../java/org/apache/beam/runners/direct/Clock.java | 5 +- .../beam/runners/direct/CommittedBundle.java | 16 +- .../beam/runners/direct/CommittedResult.java | 12 +- .../direct/CopyOnAccessInMemoryStateInternals.java | 15 +- .../runners/direct/DirectExecutionContext.java | 7 +- .../apache/beam/runners/direct/DirectGraph.java | 70 +- .../beam/runners/direct/DirectGraphVisitor.java | 2 +- .../apache/beam/runners/direct/DirectMetrics.java | 52 +- .../apache/beam/runners/direct/DirectOptions.java | 8 + .../beam/runners/direct/DirectRegistrar.java | 1 + .../apache/beam/runners/direct/DirectRunner.java | 94 +- .../runners/direct/DirectTransformExecutor.java | 8 +- ...LifecycleManagerRemovingTransformEvaluator.java | 4 + .../beam/runners/direct/EmptyInputProvider.java | 2 +- .../beam/runners/direct/EvaluationContext.java | 59 +- .../beam/runners/direct/ExecutableGraph.java | 43 + .../direct/ExecutorServiceParallelExecutor.java | 62 +- .../direct/GroupAlsoByWindowEvaluatorFactory.java | 36 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 62 +- .../runners/direct/ImpulseEvaluatorFactory.java | 104 + .../beam/runners/direct/MultiStepCombine.java | 15 +- .../apache/beam/runners/direct/ParDoEvaluator.java | 30 +- .../beam/runners/direct/ParDoEvaluatorFactory.java | 17 +- .../beam/runners/direct/QuiescenceDriver.java | 21 +- .../beam/runners/direct/ReadEvaluatorFactory.java | 20 +- .../beam/runners/direct/RootProviderRegistry.java | 35 +- .../beam/runners/direct/SideInputContainer.java | 9 +- .../SplittableProcessElementsEvaluatorFactory.java | 104 +- .../direct/StatefulParDoEvaluatorFactory.java | 6 +- .../runners/direct/TransformEvaluatorRegistry.java | 34 +- .../direct/UnboundedReadEvaluatorFactory.java | 85 +- .../beam/runners/direct/WatermarkManager.java | 393 +- .../runners/direct/WindowEvaluatorFactory.java | 2 +- .../runners/direct/WriteWithShardingFactory.java | 2 +- .../runners/direct/portable/BundleFactory.java | 49 + .../BundleFactoryOutputRecieverFactory.java | 69 + .../runners/direct/portable/BundleProcessor.java | 34 + .../runners/direct/portable/CommittedBundle.java | 85 + .../runners/direct/portable/CommittedResult.java | 70 + .../direct/portable/CompletionCallback.java | 49 + .../CopyOnAccessInMemoryStateInternals.java | 476 + .../direct/portable/DirectExecutionContext.java | 113 + .../runners/direct/portable/DirectGroupByKey.java | 133 + .../direct/portable/DirectJobBundleFactory.java | 178 + .../runners/direct/portable/DirectMetrics.java | 339 + .../direct/portable/DirectTimerInternals.java | 113 + .../direct/portable/DirectTransformExecutor.java | 158 + .../direct/portable/DisplayDataValidator.java | 67 + .../runners/direct/portable/EvaluationContext.java | 370 + .../direct/portable/ExecutorServiceFactory.java | 33 + .../portable/ExecutorServiceParallelExecutor.java | 379 + .../direct/portable/FlattenEvaluatorFactory.java | 79 + .../GroupAlsoByWindowEvaluatorFactory.java | 282 + .../portable/GroupByKeyOnlyEvaluatorFactory.java | 161 + .../portable/ImmutableListBundleFactory.java | 170 + .../direct/portable/ImpulseEvaluatorFactory.java | 109 + .../direct/portable/PCollectionViewWindow.java | 67 + .../direct/portable/PCollectionViewWriter.java | 34 + .../portable/PassthroughTransformEvaluator.java | 48 + .../runners/direct/portable/PipelineExecutor.java | 64 + .../runners/direct/portable/PortableGraph.java | 70 + .../runners/direct/portable/QuiescenceDriver.java | 332 + .../portable/RemoteStageEvaluatorFactory.java | 89 + .../runners/direct/portable/RootInputProvider.java | 45 + .../direct/portable/RootProviderRegistry.java | 63 + .../beam/runners/direct/portable/SourceShard.java | 33 + .../direct/portable/StateAndTimerProvider.java | 32 + .../beam/runners/direct/portable/StepAndKey.java | 70 + .../direct/portable/StepTransformResult.java | 140 + .../direct/portable/TransformEvaluator.java | 45 + .../direct/portable/TransformEvaluatorFactory.java | 60 + .../portable/TransformEvaluatorRegistry.java | 122 + .../runners/direct/portable/TransformExecutor.java | 22 + .../direct/portable/TransformExecutorFactory.java | 32 + .../direct/portable/TransformExecutorService.java | 41 + .../direct/portable/TransformExecutorServices.java | 179 + .../runners/direct/portable/TransformResult.java | 103 + .../runners/direct/portable/UncommittedBundle.java | 58 + .../direct/portable/WatermarkCallbackExecutor.java | 177 + .../direct/portable/WindowEvaluatorFactory.java | 127 + .../artifact/LocalArtifactStagingLocation.java | 117 + .../LocalFileSystemArtifactRetrievalService.java | 120 + .../LocalFileSystemArtifactStagerService.java | 280 + .../direct/portable/artifact/package-info.java | 20 + .../runners/direct/portable/job/PreparingJob.java | 61 + .../portable/job/ReferenceRunnerJobServer.java | 88 + .../portable/job/ReferenceRunnerJobService.java | 178 + .../runners/direct/portable/job/package-info.java | 23 + .../beam/runners/direct/portable/package-info.java | 25 + .../direct/BoundedReadEvaluatorFactoryTest.java | 18 +- .../runners/direct/CloningBundleFactoryTest.java | 1 + .../beam/runners/direct/CommittedResultTest.java | 10 +- .../runners/direct/DirectGraphVisitorTest.java | 16 +- .../apache/beam/runners/direct/DirectGraphs.java | 12 +- .../beam/runners/direct/DirectMetricsTest.java | 39 +- .../runners/direct/DirectRunnerApiSurfaceTest.java | 8 + .../beam/runners/direct/DirectRunnerTest.java | 35 + .../direct/DirectTransformExecutorTest.java | 1 - .../beam/runners/direct/EvaluationContextTest.java | 15 +- .../ExecutorServiceParallelExecutorTest.java | 116 + .../direct/FlattenEvaluatorFactoryTest.java | 22 +- .../direct/ImpulseEvaluatorFactoryTest.java | 120 + .../beam/runners/direct/ParDoEvaluatorTest.java | 2 + .../direct/StatefulParDoEvaluatorFactoryTest.java | 9 +- .../beam/runners/direct/StructuralKeyTest.java | 90 - .../direct/UnboundedReadEvaluatorFactoryTest.java | 73 +- .../beam/runners/direct/WatermarkManagerTest.java | 787 +- .../runners/direct/WindowEvaluatorFactoryTest.java | 4 +- .../BundleFactoryOutputRecieverFactoryTest.java | 259 + .../direct/portable/CommittedResultTest.java | 130 + .../CopyOnAccessInMemoryStateInternalsTest.java | 591 + .../portable/DirectJobBundleFactoryTest.java | 178 + .../runners/direct/portable/DirectMetricsTest.java | 233 + .../direct/portable/DirectTimerInternalsTest.java | 132 + .../portable/DirectTransformExecutorTest.java | 326 + .../direct/portable/EvaluationContextTest.java | 330 + .../direct/portable/ExecutableGraphBuilder.java | 82 + .../portable/FlattenEvaluatorFactoryTest.java | 122 + .../GroupByKeyOnlyEvaluatorFactoryTest.java | 257 + .../portable/ImmutableListBundleFactoryTest.java | 234 + .../portable/ImpulseEvaluatorFactoryTest.java | 131 + .../beam/runners/direct/portable/MockClock.java | 65 + .../runners/direct/portable/PortableGraphTest.java | 178 + .../portable/RemoteStageEvaluatorFactoryTest.java | 172 + .../direct/portable/StepTransformResultTest.java | 96 + .../portable/TransformExecutorServicesTest.java | 143 + .../portable/WatermarkCallbackExecutorTest.java | 136 + .../portable/WindowEvaluatorFactoryTest.java | 314 + .../artifact/LocalArtifactStagingLocationTest.java | 125 + ...ocalFileSystemArtifactRetrievalServiceTest.java | 240 + .../LocalFileSystemArtifactStagerServiceTest.java | 301 + .../job/ReferenceRunnerJobServiceTest.java | 146 + runners/extensions-java/metrics/build.gradle | 34 + runners/extensions-java/metrics/pom.xml | 75 + .../extensions/metrics/MetricsHttpSink.java | 99 + .../runners/extensions/metrics/package-info.java | 26 + .../extensions/metrics/MetricsHttpSinkTest.java | 181 + runners/extensions-java/pom.xml | 38 + runners/flink/build.gradle | 65 +- runners/flink/pom.xml | 58 +- .../beam/runners/flink/ArtifactSourcePool.java | 61 + .../FlinkBatchPortablePipelineTranslator.java | 619 + .../flink/FlinkBatchTransformTranslators.java | 98 +- .../runners/flink/FlinkExecutionEnvironments.java | 174 + .../flink/FlinkPipelineExecutionEnvironment.java | 150 +- .../beam/runners/flink/FlinkPipelineOptions.java | 12 + .../flink/FlinkPortablePipelineTranslator.java | 41 + .../org/apache/beam/runners/flink/FlinkRunner.java | 14 +- .../beam/runners/flink/FlinkRunnerResult.java | 7 +- .../FlinkStreamingPortablePipelineTranslator.java | 543 + .../flink/FlinkStreamingTransformTranslators.java | 362 +- .../runners/flink/FlinkTransformOverrides.java | 3 +- .../flink/PipelineTranslationOptimizer.java | 16 +- .../apache/beam/runners/flink/TestFlinkRunner.java | 1 + .../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 6 + .../flink/metrics/FlinkMetricContainer.java | 26 +- .../functions/FlinkExecutableStageContext.java | 55 + .../functions/FlinkExecutableStageFunction.java | 162 + .../FlinkExecutableStagePruningFunction.java | 45 + .../functions/FlinkPartialReduceFunction.java | 1 + .../translation/functions/FlinkReduceFunction.java | 1 + .../functions/SideInputInitializer.java | 16 +- .../functions/SortingFlinkCombineRunner.java | 10 +- .../utils/FlinkPipelineTranslatorUtils.java | 42 + .../translation/wrappers/ImpulseInputFormat.java | 99 + .../wrappers/streaming/DoFnOperator.java | 172 +- .../streaming/ExecutableStageDoFnOperator.java | 228 + .../wrappers/streaming/SplittableDoFnOperator.java | 33 +- .../wrappers/streaming/WindowDoFnOperator.java | 33 +- .../streaming/io/BoundedSourceWrapper.java | 231 - .../streaming/io/UnboundedSourceWrapper.java | 123 +- .../state/FlinkBroadcastStateInternals.java | 11 +- .../state/FlinkKeyGroupStateInternals.java | 11 +- .../streaming/state/FlinkSplitStateInternals.java | 9 - .../streaming/state/FlinkStateInternals.java | 11 +- .../FlinkPipelineExecutionEnvironmentTest.java | 68 + .../beam/runners/flink/PipelineOptionsTest.java | 25 +- .../beam/runners/flink/ReadSourceITCase.java | 85 - .../runners/flink/ReadSourceStreamingITCase.java | 74 - .../runners/flink/ReadSourceStreamingTest.java | 80 + .../apache/beam/runners/flink/ReadSourceTest.java | 85 + .../flink/streaming/BoundedSourceRestoreTest.java | 236 + .../flink/streaming/DedupingOperatorTest.java | 30 +- .../runners/flink/streaming/DoFnOperatorTest.java | 511 +- .../streaming/ExecutableStageDoFnOperatorTest.java | 369 + .../FlinkBroadcastStateInternalsTest.java | 8 - .../streaming/FlinkKeyGroupStateInternalsTest.java | 14 - .../streaming/FlinkSplitStateInternalsTest.java | 8 - .../flink/streaming/GroupByNullKeyTest.java | 2 +- .../flink/streaming/StreamRecordStripper.java | 49 + .../streaming/TopWikipediaSessionsITCase.java | 167 - .../flink/streaming/TopWikipediaSessionsTest.java | 174 + .../streaming/UnboundedSourceWrapperTest.java | 82 +- .../flink/streaming/WindowDoFnOperatorTest.java | 227 + .../FlinkExecutableStageFunctionTest.java | 225 + .../translation/types/CoderTypeSerializerTest.java | 1 - runners/gcp/gcemd/build.gradle | 22 + runners/gcp/gcemd/pom.xml | 5 +- runners/gcp/gcsproxy/build.gradle | 22 + runners/gcp/gcsproxy/pom.xml | 5 +- runners/gcp/pom.xml | 2 +- runners/gearpump/README.md | 61 - runners/gearpump/build.gradle | 70 +- runners/gearpump/pom.xml | 24 +- .../runners/gearpump/GearpumpPipelineOptions.java | 3 - .../beam/runners/gearpump/GearpumpRunner.java | 4 - .../beam/runners/gearpump/TestGearpumpRunner.java | 1 - .../CreateGearpumpPCollectionViewTranslator.java | 1 - .../translators/GearpumpPipelineTranslator.java | 3 - .../gearpump/translators/GroupByKeyTranslator.java | 2 +- .../translators/ParDoMultiOutputTranslator.java | 2 - .../translators/ReadUnboundedTranslator.java | 1 - .../gearpump/translators/TransformTranslator.java | 1 - .../gearpump/translators/TranslationContext.java | 3 - .../translators/WindowAssignTranslator.java | 2 - .../translators/functions/DoFnFunction.java | 17 +- .../translators/io/BoundedSourceWrapper.java | 1 - .../gearpump/translators/io/GearpumpSource.java | 4 +- .../translators/io/UnboundedSourceWrapper.java | 1 - .../translators/utils/DoFnRunnerFactory.java | 1 - .../translators/utils/NoOpStepContext.java | 1 - .../translators/utils/TranslatorUtils.java | 7 +- .../beam/runners/gearpump/PipelineOptionsTest.java | 3 +- .../FlattenPCollectionsTranslatorTest.java | 3 +- .../translators/GroupByKeyTranslatorTest.java | 2 - .../translators/ReadBoundedTranslatorTest.java | 2 +- .../translators/ReadUnboundedTranslatorTest.java | 2 +- .../translators/io/GearpumpSourceTest.java | 2 - .../gearpump/translators/io/ValueSoureTest.java | 2 - .../translators/utils/TranslatorUtilsTest.java | 2 - runners/google-cloud-dataflow-java/build.gradle | 158 +- runners/google-cloud-dataflow-java/pom.xml | 19 +- .../dataflow/BatchStatefulParDoOverrides.java | 1 + .../beam/runners/dataflow/BatchViewOverrides.java | 625 +- .../beam/runners/dataflow/DataflowMetrics.java | 50 +- .../beam/runners/dataflow/DataflowPipelineJob.java | 15 +- .../dataflow/DataflowPipelineTranslator.java | 61 +- .../beam/runners/dataflow/DataflowRunner.java | 145 +- .../dataflow/PrimitiveParDoSingleFactory.java | 46 +- .../beam/runners/dataflow/TestDataflowRunner.java | 20 +- .../beam/runners/dataflow/TransformTranslator.java | 5 + .../beam/runners/dataflow/internal/IsmFormat.java | 2 +- .../runners/dataflow/util/DataflowTransport.java | 6 +- ...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 + .../beam/runners/dataflow/util/MonitoringUtil.java | 3 +- .../beam/runners/dataflow/util/PackageUtil.java | 64 +- .../beam/runners/dataflow/util/PropertyNames.java | 1 + .../runners/dataflow/util/RandomAccessData.java | 4 +- .../beam/runners/dataflow/util/TimeUtil.java | 19 +- .../runners/dataflow/BatchViewOverridesTest.java | 4 - .../beam/runners/dataflow/DataflowMetricsTest.java | 65 +- .../dataflow/DataflowPipelineTranslatorTest.java | 128 +- .../beam/runners/dataflow/DataflowRunnerTest.java | 39 +- .../runners/dataflow/TestDataflowRunnerTest.java | 34 + .../runners/dataflow/util/CloudObjectsTest.java | 2 - runners/java-fn-execution/build.gradle | 42 +- runners/java-fn-execution/pom.xml | 83 +- .../GrpcContextHeaderAccessorProvider.java | 76 + .../beam/runners/fnexecution/GrpcFnServer.java | 3 +- .../beam/runners/fnexecution/HeaderAccessor.java | 25 + .../fnexecution/InProcessServerFactory.java | 4 +- .../beam/runners/fnexecution/ServerFactory.java | 50 +- .../artifact/ArtifactRetrievalService.java | 24 + .../fnexecution/artifact/ArtifactSource.java | 41 + .../runners/fnexecution/artifact/package-info.java | 23 + .../fnexecution/control/ControlClientPool.java | 80 + .../control/DockerJobBundleFactory.java | 362 + .../fnexecution/control/FnApiControlClient.java | 105 +- .../control/FnApiControlClientPoolService.java | 72 +- .../control/InstructionRequestHandler.java | 26 + .../fnexecution/control/JobBundleFactory.java | 31 + .../fnexecution/control/MapControlClientPool.java | 78 + .../fnexecution/control/OutputReceiverFactory.java | 33 + .../control/ProcessBundleDescriptors.java | 348 + .../runners/fnexecution/control/RemoteBundle.java | 52 + .../fnexecution/control/RemoteOutputReceiver.java | 38 + .../fnexecution/control/SdkHarnessClient.java | 469 +- .../fnexecution/control/StageBundleFactory.java | 38 + .../runners/fnexecution/data/FnDataService.java | 6 +- .../runners/fnexecution/data/GrpcDataService.java | 40 +- .../fnexecution/data/RemoteInputDestination.java | 37 + .../fnexecution/environment/DockerCommand.java | 153 + .../environment/DockerContainerEnvironment.java | 86 + .../environment/DockerEnvironmentFactory.java | 178 + .../environment/EnvironmentFactory.java | 29 + .../environment/InProcessEnvironmentFactory.java | 123 + .../fnexecution/environment/RemoteEnvironment.java | 67 + .../fnexecution/environment/package-info.java | 22 + .../environment/testing/NeedsDocker.java | 21 + .../environment/testing/package-info.java | 20 + .../graph/LengthPrefixUnknownCoders.java | 183 - .../jobsubmission/InMemoryJobService.java | 264 + .../fnexecution/jobsubmission/JobInvocation.java | 59 + .../fnexecution/jobsubmission/JobInvoker.java | 34 + .../fnexecution/jobsubmission/JobPreparation.java | 42 + .../fnexecution/jobsubmission/package-info.java | 22 + .../runners/fnexecution/provisioning/JobInfo.java | 38 + .../fnexecution/state/GrpcStateService.java | 161 + .../runners/fnexecution/state/StateDelegator.java | 51 + .../fnexecution/state/StateRequestHandler.java | 46 + .../fnexecution/state/StateRequestHandlers.java | 286 + .../runners/fnexecution/state/package-info.java | 20 + .../wire/LengthPrefixUnknownCoders.java | 172 + .../beam/runners/fnexecution/wire/WireCoders.java | 105 + .../runners/fnexecution/wire/package-info.java | 20 + .../GrpcContextHeaderAccessorProviderTest.java | 106 + .../runners/fnexecution/InProcessSdkHarness.java | 111 + .../runners/fnexecution/ServerFactoryTest.java | 25 +- .../control/DockerJobBundleFactoryTest.java | 195 + .../control/FnApiControlClientPoolServiceTest.java | 82 +- .../control/FnApiControlClientTest.java | 50 +- .../fnexecution/control/RemoteExecutionTest.java | 343 + .../fnexecution/control/SdkHarnessClientTest.java | 554 +- .../fnexecution/data/GrpcDataServiceTest.java | 2 + .../data/RemoteInputDestinationTest.java | 56 + .../fnexecution/environment/DockerCommandTest.java | 79 + .../environment/DockerEnvironmentFactoryTest.java | 117 + .../environment/RemoteEnvironmentTest.java | 71 + .../graph/LengthPrefixUnknownCodersTest.java | 147 - .../jobsubmission/InMemoryJobServiceTest.java | 138 + .../logging/GrpcLoggingServiceTest.java | 2 +- .../fnexecution/state/GrpcStateServiceTest.java | 133 + .../wire/LengthPrefixUnknownCodersTest.java | 147 + runners/local-artifact-service-java/build.gradle | 45 - runners/local-artifact-service-java/pom.xml | 120 - .../LocalFileSystemArtifactStagerService.java | 286 - .../apache/beam/artifact/local/package-info.java | 22 - .../LocalFileSystemArtifactStagerServiceTest.java | 301 - runners/local-java/build.gradle | 9 +- runners/local-java/pom.xml | 31 +- .../java/org/apache/beam/runners/local/Bundle.java | 27 +- runners/pom.xml | 14 +- runners/reference/java/build.gradle | 23 +- runners/reference/java/pom.xml | 35 +- .../beam/runners/reference/CloseableResource.java | 120 + .../reference/JobServicePipelineResult.java | 149 + .../beam/runners/reference/PortableRunner.java | 245 + .../testing/InProcessManagedChannelFactory.java | 36 + .../runners/reference/testing/TestJobService.java | 78 + .../runners/reference/testing/package-info.java | 20 + .../runners/reference/CloseableResourceTest.java | 109 + .../beam/runners/reference/PortableRunnerTest.java | 84 + runners/reference/job-server/build.gradle | 45 - runners/reference/job-server/pom.xml | 144 - .../beam/runners/reference/job/PreparingJob.java | 58 - .../reference/job/ReferenceRunnerJobServer.java | 82 - .../reference/job/ReferenceRunnerJobService.java | 183 - .../beam/runners/reference/job/package-info.java | 23 - .../job/ReferenceRunnerJobServiceTest.java | 145 - runners/reference/pom.xml | 3 +- runners/spark/build.gradle | 115 +- runners/spark/pom.xml | 120 +- .../runners/spark/SparkNativePipelineVisitor.java | 16 +- .../org/apache/beam/runners/spark/SparkRunner.java | 8 +- .../spark/aggregators/NamedAggregators.java | 9 +- .../spark/aggregators/metrics/package-info.java | 22 - .../beam/runners/spark/coders/CoderHelpers.java | 14 +- .../apache/beam/runners/spark/io/ConsoleIO.java | 23 +- .../beam/runners/spark/io/EmptyCheckpointMark.java | 2 +- .../apache/beam/runners/spark/io/SourceRDD.java | 2 +- .../runners/spark/metrics/SparkBeamMetric.java | 28 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 20 +- .../spark/stateful/SparkStateInternals.java | 6 - .../runners/spark/translation/BoundedDataset.java | 7 +- .../spark/translation/DoFnRunnerWithMetrics.java | 6 + .../spark/translation/EvaluationContext.java | 12 +- .../spark/translation/MultiDoFnFunction.java | 2 +- .../spark/translation/SparkAbstractCombineFn.java | 9 +- .../spark/translation/SparkContextFactory.java | 2 +- .../spark/translation/SparkGlobalCombineFn.java | 26 +- .../spark/translation/SparkKeyedCombineFn.java | 28 +- .../spark/translation/TransformTranslator.java | 4 +- .../spark/translation/TranslationUtils.java | 6 +- .../spark/translation/WindowingHelpers.java | 8 +- .../SparkRunnerStreamingContextFactory.java | 1 + .../streaming/StreamingTransformTranslator.java | 20 +- .../runners/spark/util/SparkSideInputReader.java | 27 +- .../runners/spark/ProvidedSparkContextTest.java | 2 +- .../aggregators/metrics/sink/InMemoryMetrics.java | 18 +- .../runners/spark/metrics/SparkBeamMetricTest.java | 8 +- .../spark/metrics/SparkMetricsPusherTest.java | 128 + .../ResumeFromCheckpointStreamingTest.java | 22 +- .../streaming/StreamingSourceMetricsTest.java | 9 +- sdks/CONTAINERS.md | 141 +- sdks/go/BUILD.md | 8 +- sdks/go/README.md | 93 + sdks/go/build.gradle | 9 +- sdks/go/cmd/beamctl/cmd/provision.go | 58 + sdks/go/cmd/beamctl/cmd/root.go | 2 +- sdks/go/cmd/specialize/main.go | 240 + sdks/go/cmd/symtab/main.go | 101 + sdks/go/container/Dockerfile | 30 + sdks/go/container/boot.go | 144 + sdks/go/container/build.gradle | 53 + sdks/go/container/pom.xml | 157 + sdks/go/data/haiku/old_pond.txt | 3 + sdks/go/data/shakespeare/hamlet.txt | 6045 ++++++ sdks/go/data/shakespeare/kinglear-hashtag.txt | 5525 +++++ sdks/go/data/shakespeare/kinglear.txt | 5525 +++++ sdks/go/data/shakespeare/romeoandjuliet.txt | 4766 +++++ sdks/go/examples/build.gradle | 73 + .../examples/complete/autocomplete/autocomplete.go | 71 + sdks/go/examples/contains/contains.go | 99 + sdks/go/examples/cookbook/combine/combine.go | 106 + sdks/go/examples/cookbook/filter/filter.go | 106 + sdks/go/examples/cookbook/join/join.go | 111 + sdks/go/examples/cookbook/max/max.go | 88 + sdks/go/examples/cookbook/tornadoes/tornadoes.go | 119 + .../debugging_wordcount/debugging_wordcount.go | 163 + sdks/go/examples/forest/forest.go | 81 + sdks/go/examples/grades/grades.go | 105 + .../minimal_wordcount/minimal_wordcount.go | 102 + sdks/go/examples/multiout/multiout.go | 79 + sdks/go/examples/pingpong/pingpong.go | 131 + sdks/go/examples/streaming_wordcap/wordcap.go | 81 + .../windowed_wordcount/windowed_wordcount.go | 131 + sdks/go/examples/wordcap/wordcap.go | 75 + sdks/go/examples/wordcount/wordcount.go | 177 + sdks/go/examples/yatzy/yatzy.go | 139 + sdks/go/gogradle.lock | 697 + sdks/go/pkg/beam/coder.go | 226 + sdks/go/pkg/beam/coder_test.go | 43 + sdks/go/pkg/beam/combine.go | 70 + sdks/go/pkg/beam/core/funcx/doc.go | 22 + sdks/go/pkg/beam/core/funcx/fn.go | 457 + sdks/go/pkg/beam/core/funcx/fn_test.go | 241 + sdks/go/pkg/beam/core/funcx/output.go | 76 + sdks/go/pkg/beam/core/funcx/output_test.go | 48 + sdks/go/pkg/beam/core/funcx/sideinput.go | 100 + sdks/go/pkg/beam/core/funcx/sideinput_test.go | 70 + sdks/go/pkg/beam/core/funcx/signature.go | 212 + sdks/go/pkg/beam/core/funcx/signature_test.go | 212 + sdks/go/pkg/beam/core/graph/bind.go | 320 + sdks/go/pkg/beam/core/graph/bind_test.go | 208 + sdks/go/pkg/beam/core/graph/coder/coder.go | 293 + sdks/go/pkg/beam/core/graph/coder/int.go | 71 + sdks/go/pkg/beam/core/graph/coder/int_test.go | 88 + sdks/go/pkg/beam/core/graph/coder/time.go | 43 + sdks/go/pkg/beam/core/graph/coder/time_test.go | 57 + sdks/go/pkg/beam/core/graph/coder/varint.go | 98 + sdks/go/pkg/beam/core/graph/coder/varint_test.go | 94 + sdks/go/pkg/beam/core/graph/coder/windows.go | 47 + sdks/go/pkg/beam/core/graph/doc.go | 22 + sdks/go/pkg/beam/core/graph/edge.go | 430 + sdks/go/pkg/beam/core/graph/fn.go | 304 + sdks/go/pkg/beam/core/graph/graph.go | 126 + sdks/go/pkg/beam/core/graph/graph_test.go | 68 + sdks/go/pkg/beam/core/graph/mtime/time.go | 124 + sdks/go/pkg/beam/core/graph/node.go | 85 + sdks/go/pkg/beam/core/graph/scope.go | 39 + sdks/go/pkg/beam/core/graph/window/fn.go | 111 + sdks/go/pkg/beam/core/graph/window/strategy.go | 38 + sdks/go/pkg/beam/core/graph/window/windows.go | 78 + sdks/go/pkg/beam/core/metrics/metrics.go | 475 + sdks/go/pkg/beam/core/metrics/metrics_test.go | 358 + sdks/go/pkg/beam/core/runtime/coderx/float.go | 75 + sdks/go/pkg/beam/core/runtime/coderx/float_test.go | 50 + sdks/go/pkg/beam/core/runtime/coderx/int.go | 99 + sdks/go/pkg/beam/core/runtime/coderx/varint.go | 139 + .../go/pkg/beam/core/runtime/coderx/varint_test.go | 84 + sdks/go/pkg/beam/core/runtime/exec/coder.go | 416 + sdks/go/pkg/beam/core/runtime/exec/cogbk.go | 172 + sdks/go/pkg/beam/core/runtime/exec/combine.go | 219 + sdks/go/pkg/beam/core/runtime/exec/combine_test.go | 68 + sdks/go/pkg/beam/core/runtime/exec/datasink.go | 97 + sdks/go/pkg/beam/core/runtime/exec/datasource.go | 207 + sdks/go/pkg/beam/core/runtime/exec/decode.go | 102 + sdks/go/pkg/beam/core/runtime/exec/discard.go | 53 + sdks/go/pkg/beam/core/runtime/exec/emit.go | 128 + sdks/go/pkg/beam/core/runtime/exec/encode.go | 102 + sdks/go/pkg/beam/core/runtime/exec/flatten.go | 75 + sdks/go/pkg/beam/core/runtime/exec/flatten_test.go | 49 + sdks/go/pkg/beam/core/runtime/exec/fn.go | 214 + sdks/go/pkg/beam/core/runtime/exec/fn_test.go | 266 + sdks/go/pkg/beam/core/runtime/exec/fullvalue.go | 143 + .../pkg/beam/core/runtime/exec/fullvalue_test.go | 124 + sdks/go/pkg/beam/core/runtime/exec/input.go | 187 + sdks/go/pkg/beam/core/runtime/exec/multiplex.go | 62 + .../pkg/beam/core/runtime/exec/multiplex_test.go | 55 + .../beam/core/runtime/exec/optimized/callers.go | 632 + .../beam/core/runtime/exec/optimized/callers.tmpl | 69 + .../beam/core/runtime/exec/optimized/decoders.go | 2407 +++ .../beam/core/runtime/exec/optimized/decoders.tmpl | 146 + .../beam/core/runtime/exec/optimized/emitters.go | 14217 +++++++++++++ .../beam/core/runtime/exec/optimized/emitters.tmpl | 114 + .../beam/core/runtime/exec/optimized/encoders.go | 2299 ++ .../beam/core/runtime/exec/optimized/encoders.tmpl | 146 + .../go/pkg/beam/core/runtime/exec/optimized/gen.go | 29 + .../pkg/beam/core/runtime/exec/optimized/inputs.go | 21272 +++++++++++++++++++ .../beam/core/runtime/exec/optimized/inputs.tmpl | 181 + sdks/go/pkg/beam/core/runtime/exec/pardo.go | 228 + sdks/go/pkg/beam/core/runtime/exec/pardo_test.go | 97 + sdks/go/pkg/beam/core/runtime/exec/plan.go | 184 + sdks/go/pkg/beam/core/runtime/exec/status.go | 27 + sdks/go/pkg/beam/core/runtime/exec/translate.go | 528 + sdks/go/pkg/beam/core/runtime/exec/unit.go | 75 + sdks/go/pkg/beam/core/runtime/exec/unit_test.go | 111 + sdks/go/pkg/beam/core/runtime/exec/util.go | 72 + sdks/go/pkg/beam/core/runtime/exec/window.go | 92 + sdks/go/pkg/beam/core/runtime/exec/window_test.go | 115 + sdks/go/pkg/beam/core/runtime/graphx/coder.go | 419 + sdks/go/pkg/beam/core/runtime/graphx/coder_test.go | 106 + sdks/go/pkg/beam/core/runtime/graphx/cogbk.go | 106 + sdks/go/pkg/beam/core/runtime/graphx/doc.go | 29 + sdks/go/pkg/beam/core/runtime/graphx/serialize.go | 1010 + sdks/go/pkg/beam/core/runtime/graphx/translate.go | 508 + .../pkg/beam/core/runtime/graphx/translate_test.go | 90 + sdks/go/pkg/beam/core/runtime/graphx/tree.go | 121 + sdks/go/pkg/beam/core/runtime/graphx/user.go | 110 + sdks/go/pkg/beam/core/runtime/graphx/v1/gen.go | 18 + sdks/go/pkg/beam/core/runtime/graphx/v1/v1.pb.go | 887 + sdks/go/pkg/beam/core/runtime/graphx/v1/v1.proto | 244 + sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 338 + .../pkg/beam/core/runtime/harness/datamgr_test.go | 104 + sdks/go/pkg/beam/core/runtime/harness/gen.go | 18 + sdks/go/pkg/beam/core/runtime/harness/harness.go | 270 + sdks/go/pkg/beam/core/runtime/harness/init/init.go | 94 + sdks/go/pkg/beam/core/runtime/harness/logging.go | 152 + sdks/go/pkg/beam/core/runtime/harness/session.go | 242 + .../go/pkg/beam/core/runtime/harness/session.proto | 69 + .../core/runtime/harness/session/session.pb.go | 442 + sdks/go/pkg/beam/core/runtime/init.go | 44 + sdks/go/pkg/beam/core/runtime/options.go | 118 + sdks/go/pkg/beam/core/runtime/options_test.go | 49 + sdks/go/pkg/beam/core/runtime/symbols.go | 93 + sdks/go/pkg/beam/core/runtime/types.go | 62 + sdks/go/pkg/beam/core/runtime/types_test.go | 74 + sdks/go/pkg/beam/core/typex/class.go | 184 + sdks/go/pkg/beam/core/typex/class_test.go | 118 + sdks/go/pkg/beam/core/typex/fulltype.go | 407 + sdks/go/pkg/beam/core/typex/fulltype_test.go | 125 + sdks/go/pkg/beam/core/typex/special.go | 73 + sdks/go/pkg/beam/core/util/dot/dot.go | 130 + sdks/go/pkg/beam/core/util/hooks/hooks.go | 195 + sdks/go/pkg/beam/core/util/hooks/hooks_test.go | 65 + sdks/go/pkg/beam/core/util/ioutilx/read.go | 43 + sdks/go/pkg/beam/core/util/protox/any.go | 101 + sdks/go/pkg/beam/core/util/protox/any_test.go | 83 + sdks/go/pkg/beam/core/util/protox/base64.go | 50 + sdks/go/pkg/beam/core/util/protox/protox.go | 28 + sdks/go/pkg/beam/core/util/reflectx/call.go | 121 + sdks/go/pkg/beam/core/util/reflectx/calls.go | 1341 ++ sdks/go/pkg/beam/core/util/reflectx/calls.tmpl | 72 + sdks/go/pkg/beam/core/util/reflectx/functions.go | 45 + sdks/go/pkg/beam/core/util/reflectx/json.go | 32 + sdks/go/pkg/beam/core/util/reflectx/tags.go | 74 + sdks/go/pkg/beam/core/util/reflectx/types.go | 111 + sdks/go/pkg/beam/core/util/stringx/bytes.go | 28 + sdks/go/pkg/beam/core/util/symtab/symtab.go | 128 + sdks/go/pkg/beam/create.go | 105 + sdks/go/pkg/beam/create_test.go | 88 + sdks/go/pkg/beam/doc.go | 23 + sdks/go/pkg/beam/doc_test.go | 166 + sdks/go/pkg/beam/encoding.go | 154 + sdks/go/pkg/beam/external.go | 58 + sdks/go/pkg/beam/flatten.go | 65 + sdks/go/pkg/beam/forward.go | 127 + sdks/go/pkg/beam/gbk.go | 94 + sdks/go/pkg/beam/impulse.go | 41 + sdks/go/pkg/beam/io/bigqueryio/bigquery.go | 243 + sdks/go/pkg/beam/io/bigqueryio/bigquery_test.go | 38 + sdks/go/pkg/beam/io/datastoreio/datastore.go | 237 + sdks/go/pkg/beam/io/datastoreio/datastore_test.go | 99 + sdks/go/pkg/beam/io/filesystem/filesystem.go | 81 + sdks/go/pkg/beam/io/filesystem/gcs/gcs.go | 139 + sdks/go/pkg/beam/io/filesystem/local/local.go | 56 + sdks/go/pkg/beam/io/filesystem/memfs/memory.go | 108 + .../go/pkg/beam/io/filesystem/memfs/memory_test.go | 74 + sdks/go/pkg/beam/io/filesystem/util.go | 45 + sdks/go/pkg/beam/io/pubsubio/pubsubio.go | 95 + sdks/go/pkg/beam/io/pubsubio/v1/gen.go | 21 + sdks/go/pkg/beam/io/pubsubio/v1/v1.pb.go | 134 + sdks/go/pkg/beam/io/pubsubio/v1/v1.proto | 39 + sdks/go/pkg/beam/io/textio/textio.go | 180 + sdks/go/pkg/beam/log/log.go | 183 + sdks/go/pkg/beam/log/standard.go | 37 + sdks/go/pkg/beam/metrics.go | 75 + sdks/go/pkg/beam/metrics_test.go | 118 + .../beam/model/fnexecution_v1/beam_fn_api.pb.go | 801 +- .../model/fnexecution_v1/beam_provision_api.pb.go | 6 +- sdks/go/pkg/beam/model/gen.go | 3 +- .../model/jobmanagement_v1/beam_artifact_api.pb.go | 4 +- .../beam/model/jobmanagement_v1/beam_job_api.pb.go | 10 +- .../beam/model/pipeline_v1/beam_runner_api.pb.go | 1077 +- sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 4 +- .../model/pipeline_v1/standard_window_fns.pb.go | 174 +- sdks/go/pkg/beam/option.go | 67 + sdks/go/pkg/beam/options/gcpopts/options.go | 38 + sdks/go/pkg/beam/options/jobopts/options.go | 96 + sdks/go/pkg/beam/pardo.go | 314 + sdks/go/pkg/beam/partition.go | 123 + sdks/go/pkg/beam/partition_test.go | 155 + sdks/go/pkg/beam/pcollection.go | 90 + sdks/go/pkg/beam/pipeline.go | 87 + sdks/go/pkg/beam/provision/provision.go | 86 + sdks/go/pkg/beam/provision/provison.go | 80 - sdks/go/pkg/beam/runner.go | 51 + sdks/go/pkg/beam/runners/dataflow/dataflow.go | 386 + sdks/go/pkg/beam/runners/dataflow/messages.go | 189 + sdks/go/pkg/beam/runners/dataflow/translate.go | 484 + sdks/go/pkg/beam/runners/direct/buffer.go | 165 + sdks/go/pkg/beam/runners/direct/direct.go | 286 + sdks/go/pkg/beam/runners/direct/gbk.go | 144 + sdks/go/pkg/beam/runners/direct/impulse.go | 65 + sdks/go/pkg/beam/runners/dot/dot.go | 55 + sdks/go/pkg/beam/runners/flink/flink.go | 38 + sdks/go/pkg/beam/runners/session/session.go | 337 + .../beam/runners/universal/runnerlib/compile.go | 74 + .../beam/runners/universal/runnerlib/execute.go | 86 + .../go/pkg/beam/runners/universal/runnerlib/job.go | 144 + .../pkg/beam/runners/universal/runnerlib/stage.go | 51 + sdks/go/pkg/beam/runners/universal/universal.go | 61 + sdks/go/pkg/beam/testing/passert/hash.go | 69 + sdks/go/pkg/beam/testing/passert/passert.go | 214 + sdks/go/pkg/beam/testing/ptest/ptest.go | 60 + sdks/go/pkg/beam/transforms/filter/distinct.go | 44 + .../go/pkg/beam/transforms/filter/distinct_test.go | 70 + sdks/go/pkg/beam/transforms/filter/filter.go | 90 + sdks/go/pkg/beam/transforms/filter/filter_test.go | 90 + sdks/go/pkg/beam/transforms/stats/count.go | 39 + sdks/go/pkg/beam/transforms/stats/count_test.go | 69 + sdks/go/pkg/beam/transforms/stats/max.go | 45 + sdks/go/pkg/beam/transforms/stats/max_switch.go | 155 + sdks/go/pkg/beam/transforms/stats/max_switch.tmpl | 48 + sdks/go/pkg/beam/transforms/stats/max_test.go | 134 + sdks/go/pkg/beam/transforms/stats/mean.go | 95 + sdks/go/pkg/beam/transforms/stats/mean_test.go | 121 + sdks/go/pkg/beam/transforms/stats/min.go | 45 + sdks/go/pkg/beam/transforms/stats/min_switch.go | 155 + sdks/go/pkg/beam/transforms/stats/min_switch.tmpl | 48 + sdks/go/pkg/beam/transforms/stats/min_test.go | 121 + sdks/go/pkg/beam/transforms/stats/sum.go | 45 + sdks/go/pkg/beam/transforms/stats/sum_switch.go | 119 + sdks/go/pkg/beam/transforms/stats/sum_switch.tmpl | 45 + sdks/go/pkg/beam/transforms/stats/sum_test.go | 117 + sdks/go/pkg/beam/transforms/stats/util.go | 48 + sdks/go/pkg/beam/transforms/top/top.go | 180 + sdks/go/pkg/beam/transforms/top/top_test.go | 126 + sdks/go/pkg/beam/util.go | 127 + sdks/go/pkg/beam/util/gcsx/gcs.go | 51 + sdks/go/pkg/beam/util/grpcx/dial.go | 12 +- sdks/go/pkg/beam/util/grpcx/hook.go | 87 + sdks/go/pkg/beam/util/pubsubx/pubsub.go | 110 + sdks/go/pkg/beam/util/syscallx/syscall.go | 6 +- sdks/go/pkg/beam/util/syscallx/syscall_default.go | 4 +- sdks/go/pkg/beam/validate.go | 79 + sdks/go/pkg/beam/windowing.go | 42 + sdks/go/pkg/beam/x/beamx/run.go | 43 + sdks/go/pkg/beam/x/debug/head.go | 68 + sdks/go/pkg/beam/x/debug/print.go | 97 + sdks/go/pkg/beam/x/hooks/perf/perf.go | 170 + sdks/go/pom.xml | 14 +- sdks/go/test/build.gradle | 48 + sdks/go/test/integration/driver.go | 98 + sdks/go/test/integration/wordcount/wordcount.go | 81 + .../test/integration/wordcount/wordcount_test.go | 85 + sdks/go/test/regression/pardo.go | 66 + sdks/go/test/regression/pardo_test.go | 34 + sdks/go/test/run_integration_tests.sh | 92 + sdks/java/build-tools/build.gradle | 7 - sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/checkstyle.xml | 31 +- .../src/main/resources/beam/findbugs-filter.xml | 78 +- .../src/main/resources/beam/suppressions.xml | 12 +- sdks/java/container/build.gradle | 12 +- sdks/java/container/pom.xml | 5 +- sdks/java/core/build.gradle | 79 +- sdks/java/core/pom.xml | 50 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 2 +- .../apache/beam/sdk/coders/BeamRecordCoder.java | 111 - .../beam/sdk/coders/BigEndianIntegerCoder.java | 3 +- .../beam/sdk/coders/BigEndianShortCoder.java | 107 + .../org/apache/beam/sdk/coders/ByteArrayCoder.java | 2 +- .../java/org/apache/beam/sdk/coders/ByteCoder.java | 2 +- .../java/org/apache/beam/sdk/coders/Coder.java | 2 +- .../org/apache/beam/sdk/coders/CoderProviders.java | 2 +- .../org/apache/beam/sdk/coders/CoderRegistry.java | 13 +- .../org/apache/beam/sdk/coders/FloatCoder.java | 118 + .../org/apache/beam/sdk/coders/IterableCoder.java | 10 + .../apache/beam/sdk/coders/IterableLikeCoder.java | 2 +- .../java/org/apache/beam/sdk/coders/RowCoder.java | 195 + .../apache/beam/sdk/coders/SerializableCoder.java | 48 +- .../org/apache/beam/sdk/coders/SnappyCoder.java | 65 + .../beam/sdk/coders/StringDelegateCoder.java | 2 +- .../apache/beam/sdk/coders/StringUtf8Coder.java | 2 +- .../org/apache/beam/sdk/coders/VarIntCoder.java | 2 +- .../org/apache/beam/sdk/coders/VarLongCoder.java | 4 +- .../main/java/org/apache/beam/sdk/io/AvroIO.java | 6 +- .../main/java/org/apache/beam/sdk/io/AvroSink.java | 2 - .../java/org/apache/beam/sdk/io/AvroSource.java | 2 +- .../org/apache/beam/sdk/io/CompressedSource.java | 4 +- .../java/org/apache/beam/sdk/io/Compression.java | 7 +- .../org/apache/beam/sdk/io/CountingSource.java | 2 + .../java/org/apache/beam/sdk/io/FileBasedSink.java | 30 +- .../main/java/org/apache/beam/sdk/io/FileIO.java | 118 +- .../java/org/apache/beam/sdk/io/FileSystems.java | 25 +- .../org/apache/beam/sdk/io/LocalFileSystem.java | 14 +- .../org/apache/beam/sdk/io/OffsetBasedSource.java | 4 +- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 2 +- .../main/java/org/apache/beam/sdk/io/TextIO.java | 20 +- .../java/org/apache/beam/sdk/io/TextSource.java | 2 +- .../java/org/apache/beam/sdk/io/WriteFiles.java | 5 +- .../java/org/apache/beam/sdk/io/fs/ResourceId.java | 1 + .../apache/beam/sdk/io/fs/ResourceIdTester.java | 150 + .../org/apache/beam/sdk/io/range/ByteKeyRange.java | 6 +- .../beam/sdk/metrics/DistributionResult.java | 16 +- .../org/apache/beam/sdk/metrics/GaugeResult.java | 8 +- .../org/apache/beam/sdk/metrics/MetricName.java | 30 +- .../beam/sdk/metrics/MetricQueryResults.java | 6 +- .../org/apache/beam/sdk/metrics/MetricResult.java | 10 +- .../org/apache/beam/sdk/metrics/MetricResults.java | 4 +- .../org/apache/beam/sdk/metrics/MetricsSink.java | 28 + .../java/org/apache/beam/sdk/options/Default.java | 11 + .../beam/sdk/options/ExperimentalOptions.java | 14 + .../apache/beam/sdk/options/PipelineOptions.java | 47 +- .../beam/sdk/options/PipelineOptionsFactory.java | 392 +- .../beam/sdk/options/PortablePipelineOptions.java | 46 + .../beam/sdk/options/ProxyInvocationHandler.java | 37 +- .../apache/beam/sdk/runners/PTransformMatcher.java | 8 + .../beam/sdk/runners/TransformHierarchy.java | 36 +- .../java/org/apache/beam/sdk/schemas/Schema.java | 533 + .../org/apache/beam/sdk/schemas/package-info.java | 28 + .../org/apache/beam/sdk/state/StateContexts.java | 28 + .../java/org/apache/beam/sdk/state/StateSpecs.java | 42 +- .../java/org/apache/beam/sdk/state/TimerSpecs.java | 1 + .../apache/beam/sdk/testing/CoderProperties.java | 6 +- .../apache/beam/sdk/testing/CombineFnTester.java | 1 + .../java/org/apache/beam/sdk/testing/PAssert.java | 21 + .../beam/sdk/testing/SerializableMatchers.java | 10 +- .../apache/beam/sdk/testing/SuccessOrFailure.java | 19 + .../org/apache/beam/sdk/testing/TestPipeline.java | 40 +- .../org/apache/beam/sdk/testing/TestStream.java | 2 +- .../beam/sdk/testing/UsesAttemptedMetrics.java | 2 +- .../beam/sdk/testing/UsesCommittedMetrics.java | 2 +- .../org/apache/beam/sdk/testing/UsesImpulse.java | 21 + .../beam/sdk/testing/UsesParDoLifecycle.java | 24 + .../apache/beam/sdk/testing/WindowFnTestUtils.java | 23 +- .../beam/sdk/transforms/ApproximateQuantiles.java | 16 +- .../beam/sdk/transforms/ApproximateUnique.java | 104 +- .../org/apache/beam/sdk/transforms/Combine.java | 2 + .../org/apache/beam/sdk/transforms/CombineFns.java | 18 + .../org/apache/beam/sdk/transforms/Create.java | 5 +- .../org/apache/beam/sdk/transforms/Distinct.java | 137 +- .../java/org/apache/beam/sdk/transforms/DoFn.java | 139 +- .../beam/sdk/transforms/DoFnOutputReceivers.java | 84 + .../org/apache/beam/sdk/transforms/DoFnTester.java | 306 +- .../beam/sdk/transforms/GroupIntoBatches.java | 2 +- .../org/apache/beam/sdk/transforms/Impulse.java | 58 + .../org/apache/beam/sdk/transforms/JsonToRow.java | 109 + .../java/org/apache/beam/sdk/transforms/Mean.java | 4 - .../java/org/apache/beam/sdk/transforms/ParDo.java | 28 +- .../java/org/apache/beam/sdk/transforms/Regex.java | 13 + .../java/org/apache/beam/sdk/transforms/View.java | 1 + .../java/org/apache/beam/sdk/transforms/Wait.java | 149 + .../java/org/apache/beam/sdk/transforms/Watch.java | 164 +- .../reflect/ByteBuddyDoFnInvokerFactory.java | 68 +- .../reflect/ByteBuddyOnTimerInvokerFactory.java | 3 +- .../beam/sdk/transforms/reflect/DoFnInvoker.java | 88 +- .../beam/sdk/transforms/reflect/DoFnSignature.java | 177 +- .../sdk/transforms/reflect/DoFnSignatures.java | 111 +- .../transforms/reflect/OnTimerMethodSpecifier.java | 2 +- .../splittabledofn/ByteKeyRangeTracker.java | 158 + .../splittabledofn/HasDefaultTracker.java | 2 +- .../splittabledofn/OffsetRangeTracker.java | 12 +- .../splittabledofn/RestrictionTracker.java | 76 +- .../sdk/transforms/windowing/CalendarWindows.java | 13 +- .../sdk/transforms/windowing/GlobalWindows.java | 5 + .../windowing/MergeOverlappingIntervalWindows.java | 3 + .../beam/sdk/transforms/windowing/PaneInfo.java | 12 +- .../beam/sdk/transforms/windowing/Trigger.java | 4 +- .../java/org/apache/beam/sdk/util/ApiSurface.java | 25 +- .../java/org/apache/beam/sdk/util/BackOff.java | 4 + .../apache/beam/sdk/util/BucketingFunction.java | 6 +- .../util/BufferedElementCountingOutputStream.java | 15 +- .../java/org/apache/beam/sdk/util/ClassPath.java | 6 +- .../java/org/apache/beam/sdk/util/CoderUtils.java | 14 +- .../apache/beam/sdk/util/ExplicitShardedFile.java | 4 +- .../sdk/util/ExposedByteArrayOutputStream.java | 10 +- .../org/apache/beam/sdk/util/FluentBackoff.java | 2 + .../org/apache/beam/sdk/util/InstanceBuilder.java | 6 +- .../org/apache/beam/sdk/util/JsonToRowUtils.java | 54 + .../java/org/apache/beam/sdk/util/MoreFutures.java | 254 + .../org/apache/beam/sdk/util/MovingFunction.java | 4 +- .../apache/beam/sdk/util/MutationDetectors.java | 15 +- .../apache/beam/sdk/util/RowJsonDeserializer.java | 269 + .../apache/beam/sdk/util/RowJsonValidation.java | 67 + .../beam/sdk/util/RowJsonValueExtractors.java | 198 + .../beam/sdk/util/SerializableThrowable.java | 18 + .../java/org/apache/beam/sdk/util/Sleeper.java | 2 +- .../org/apache/beam/sdk/util/ThrowingRunnable.java | 24 + .../org/apache/beam/sdk/util/ThrowingSupplier.java | 26 + .../org/apache/beam/sdk/util/WeightedValue.java | 1 + .../org/apache/beam/sdk/util/WindowedValue.java | 283 +- .../java/org/apache/beam/sdk/util/ZipFiles.java | 19 +- .../sdk/util/common/ElementByteSizeObserver.java | 4 +- .../beam/sdk/util/common/ReflectHelpers.java | 52 +- .../org/apache/beam/sdk/values/BeamRecord.java | 319 - .../org/apache/beam/sdk/values/BeamRecordType.java | 96 - .../org/apache/beam/sdk/values/PCollection.java | 4 +- .../apache/beam/sdk/values/PCollectionTuple.java | 4 +- .../java/org/apache/beam/sdk/values/PValue.java | 1 + .../main/java/org/apache/beam/sdk/values/Row.java | 551 + .../apache/beam/sdk/values/TimestampedValue.java | 6 + .../java/org/apache/beam/sdk/values/TupleTag.java | 2 +- .../org/apache/beam/sdk/values/TypeDescriptor.java | 2 +- .../apache/beam/sdk/values/WindowingStrategy.java | 4 +- .../beam/sdk/values/reflect/ByteBuddyUtils.java | 111 + .../sdk/values/reflect/DefaultRowTypeFactory.java | 93 + .../beam/sdk/values/reflect/FieldValueGetter.java | 36 + .../sdk/values/reflect/GeneratedGetterFactory.java | 106 + .../beam/sdk/values/reflect/GetterFactory.java | 33 + .../beam/sdk/values/reflect/InferredRowCoder.java | 117 + .../beam/sdk/values/reflect/ReflectionGetter.java | 59 + .../values/reflect/ReflectionGetterFactory.java | 57 + .../beam/sdk/values/reflect/ReflectionUtils.java | 73 + .../apache/beam/sdk/values/reflect/RowFactory.java | 141 + .../beam/sdk/values/reflect/RowTypeFactory.java | 44 + .../beam/sdk/values/reflect/RowTypeGetters.java | 54 + .../beam/sdk/values/reflect/package-info.java | 22 + .../apache/beam/sdk/state/StateContextsTest.java | 95 + .../org/apache/beam/SdkCoreApiSurfaceTest.java | 20 +- .../java/org/apache/beam/sdk/PipelineTest.java | 2 + .../org/apache/beam/sdk/coders/AvroCoderTest.java | 9 +- .../beam/sdk/coders/BigDecimalCoderTest.java | 9 +- .../beam/sdk/coders/BigIntegerCoderTest.java | 5 +- .../apache/beam/sdk/coders/CoderRegistryTest.java | 25 +- .../beam/sdk/coders/CollectionCoderTest.java | 5 +- .../apache/beam/sdk/coders/CommonCoderTest.java | 351 - .../apache/beam/sdk/coders/DelegateCoderTest.java | 7 +- .../apache/beam/sdk/coders/DoubleCoderTest.java | 4 +- .../apache/beam/sdk/coders/InstantCoderTest.java | 2 +- .../apache/beam/sdk/coders/IterableCoderTest.java | 4 +- .../beam/sdk/coders/LengthPrefixCoderTest.java | 29 +- .../org/apache/beam/sdk/coders/ListCoderTest.java | 4 +- .../beam/sdk/coders/SerializableCoderTest.java | 68 +- .../beam/sdk/coders/StructuralByteArrayTest.java | 9 +- .../beam/sdk/coders/StructuredCoderTest.java | 3 +- .../org/apache/beam/sdk/coders/RowCoderTest.java | 126 + .../java/org/apache/beam/sdk/io/AvroIOTest.java | 18 +- .../sdk/io/BoundedReadFromUnboundedSourceTest.java | 257 +- .../org/apache/beam/sdk/io/CountingSourceTest.java | 8 +- .../org/apache/beam/sdk/io/FileBasedSinkTest.java | 19 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 6 +- .../java/org/apache/beam/sdk/io/FileIOTest.java | 110 +- .../apache/beam/sdk/io/GenerateSequenceTest.java | 11 +- .../apache/beam/sdk/io/LocalResourceIdTest.java | 13 +- .../test/java/org/apache/beam/sdk/io/ReadTest.java | 5 + .../java/org/apache/beam/sdk/io/SimpleSink.java | 12 +- .../org/apache/beam/sdk/io/TFRecordIOTest.java | 13 +- .../org/apache/beam/sdk/io/TextIOReadTest.java | 16 +- .../java/org/apache/beam/sdk/io/TextIOTest.java | 31 - .../org/apache/beam/sdk/io/TextIOWriteTest.java | 94 +- .../org/apache/beam/sdk/io/WriteFilesTest.java | 36 +- .../apache/beam/sdk/io/fs/ResourceIdTester.java | 150 - .../apache/beam/sdk/io/range/ByteKeyRangeTest.java | 2 +- .../beam/sdk/metrics/MetricResultsMatchers.java | 46 +- .../org/apache/beam/sdk/metrics/MetricsTest.java | 546 +- .../beam/sdk/options/ExperimentalOptionsTest.java | 39 + .../sdk/options/PipelineOptionsFactoryTest.java | 121 +- .../sdk/options/ProxyInvocationHandlerTest.java | 26 +- .../apache/beam/sdk/options/ValueProviderTest.java | 4 +- .../beam/sdk/runners/PipelineRunnerTest.java | 2 +- .../sdk/runners/dataflow/TestCountingSource.java | 239 - .../runners/dataflow/TestCountingSourceTest.java | 72 - .../org/apache/beam/sdk/schemas/SchemaTest.java | 175 + .../beam/sdk/testing/CoderPropertiesTest.java | 45 +- .../beam/sdk/testing/CombineFnTesterTest.java | 33 +- .../apache/beam/sdk/testing/ExpectedLogsTest.java | 2 +- .../sdk/testing/InterceptingUrlClassLoader.java | 13 +- .../org/apache/beam/sdk/testing/PAssertTest.java | 13 +- .../beam/sdk/testing/SerializableMatchersTest.java | 43 +- .../beam/sdk/testing/SourceTestUtilsTest.java | 4 +- .../apache/beam/sdk/testing/TestPipelineTest.java | 41 +- .../apache/beam/sdk/testing/ThreadLeakTracker.java | 83 + .../sdk/transforms/ApproximateQuantilesTest.java | 2 - .../beam/sdk/transforms/ApproximateUniqueTest.java | 69 +- .../apache/beam/sdk/transforms/CombineFnsTest.java | 16 + .../apache/beam/sdk/transforms/CombineTest.java | 2138 +- .../org/apache/beam/sdk/transforms/CountTest.java | 10 +- .../org/apache/beam/sdk/transforms/CreateTest.java | 10 +- .../apache/beam/sdk/transforms/DistinctTest.java | 284 +- .../apache/beam/sdk/transforms/DoFnTesterTest.java | 1 + .../org/apache/beam/sdk/transforms/FilterTest.java | 93 +- .../beam/sdk/transforms/FlatMapElementsTest.java | 72 +- .../apache/beam/sdk/transforms/GroupByKeyTest.java | 860 +- .../beam/sdk/transforms/GroupIntoBatchesTest.java | 2 +- .../apache/beam/sdk/transforms/ImpulseTest.java | 49 + .../apache/beam/sdk/transforms/JsonToRowTest.java | 94 + .../org/apache/beam/sdk/transforms/KvSwapTest.java | 3 +- .../beam/sdk/transforms/MapElementsTest.java | 125 +- .../beam/sdk/transforms/ParDoLifecycleTest.java | 45 +- .../org/apache/beam/sdk/transforms/ParDoTest.java | 5353 ++--- .../apache/beam/sdk/transforms/PartitionTest.java | 34 +- .../org/apache/beam/sdk/transforms/RegexTest.java | 2 - .../apache/beam/sdk/transforms/ReshuffleTest.java | 5 +- .../org/apache/beam/sdk/transforms/SampleTest.java | 28 +- .../beam/sdk/transforms/SimpleFunctionTest.java | 35 +- .../beam/sdk/transforms/SplittableDoFnTest.java | 6 +- .../apache/beam/sdk/transforms/ToStringTest.java | 12 +- .../org/apache/beam/sdk/transforms/ValuesTest.java | 6 +- .../org/apache/beam/sdk/transforms/ViewTest.java | 33 +- .../org/apache/beam/sdk/transforms/WaitTest.java | 331 + .../org/apache/beam/sdk/transforms/WatchTest.java | 143 +- .../apache/beam/sdk/transforms/WithKeysTest.java | 33 + .../beam/sdk/transforms/WithTimestampsTest.java | 30 + .../transforms/display/DisplayDataMatchers.java | 2 +- .../display/DisplayDataMatchersTest.java | 70 +- .../sdk/transforms/display/DisplayDataTest.java | 18 +- .../beam/sdk/transforms/join/CoGroupByKeyTest.java | 44 +- .../sdk/transforms/reflect/DoFnInvokersTest.java | 87 +- .../reflect/DoFnSignaturesProcessElementTest.java | 2 +- .../reflect/DoFnSignaturesSplittableDoFnTest.java | 24 +- .../sdk/transforms/reflect/DoFnSignaturesTest.java | 370 +- .../reflect/DoFnSignaturesTestUtils.java | 2 +- .../transforms/reflect/OnTimerInvokersTest.java | 2 +- .../reflect/testhelper/DoFnInvokersTestHelper.java | 4 + .../splittabledofn/ByteKeyRangeTrackerTest.java | 204 + .../splittabledofn/OffsetRangeTrackerTest.java | 73 +- .../beam/sdk/transforms/windowing/WindowTest.java | 16 +- .../sdk/transforms/windowing/WindowingTest.java | 12 +- .../org/apache/beam/sdk/util/ApiSurfaceTest.java | 2 +- .../beam/sdk/util/BucketingFunctionTest.java | 2 +- .../BufferedElementCountingOutputStreamTest.java | 41 +- .../sdk/util/ExposedByteArrayInputStreamTest.java | 5 +- .../sdk/util/ExposedByteArrayOutputStreamTest.java | 5 +- .../org/apache/beam/sdk/util/MoreFuturesTest.java | 81 + .../apache/beam/sdk/util/MovingFunctionTest.java | 8 +- .../beam/sdk/util/MutationDetectorsTest.java | 41 +- .../org/apache/beam/sdk/util/NameUtilsTest.java | 4 +- .../beam/sdk/util/RowJsonDeserializerTest.java | 594 + .../beam/sdk/util/SerializableUtilsTest.java | 4 +- .../apache/beam/sdk/util/WindowedValueTest.java | 33 +- .../org/apache/beam/sdk/util/ZipFilesTest.java | 5 +- .../java/org/apache/beam/sdk/values/RowTest.java | 290 + .../org/apache/beam/sdk/values/TupleTagTest.java | 8 +- .../values/reflect/DefaultSchemaFactoryTest.java | 100 + .../values/reflect/GeneratedGetterFactoryTest.java | 107 + .../sdk/values/reflect/InferredRowCoderTest.java | 98 + .../reflect/ReflectionGetterFactoryTest.java | 107 + .../sdk/values/reflect/ReflectionGetterTest.java | 95 + .../beam/sdk/values/reflect/RowFactoryTest.java | 147 + .../beam/sdk/values/reflect/SchemaGettersTest.java | 43 + .../google-cloud-platform-core/build.gradle | 13 +- .../extensions/google-cloud-platform-core/pom.xml | 28 +- .../sdk/extensions/gcp/options/GcpOptions.java | 2 +- .../sdk/extensions/gcp/storage/GcsFileSystem.java | 4 +- .../extensions/gcp/storage/GcsPathValidator.java | 2 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 19 +- .../sdk/extensions/gcp/options/GcpOptionsTest.java | 2 - .../extensions/gcp/storage/GcsResourceIdTest.java | 2 + .../java/org/apache/beam/sdk/util/GcsUtilTest.java | 2 +- .../sdk/util/RetryHttpRequestInitializerTest.java | 4 +- sdks/java/extensions/jackson/build.gradle | 12 +- sdks/java/extensions/jackson/pom.xml | 10 +- .../extensions/jackson/JacksonTransformsTest.java | 31 +- sdks/java/extensions/join-library/build.gradle | 15 +- sdks/java/extensions/join-library/pom.xml | 10 +- .../sdk/extensions/joinlibrary/InnerJoinTest.java | 7 +- .../extensions/joinlibrary/OuterFullJoinTest.java | 8 +- .../extensions/joinlibrary/OuterLeftJoinTest.java | 7 +- .../extensions/joinlibrary/OuterRightJoinTest.java | 7 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/extensions/protobuf/build.gradle | 12 +- sdks/java/extensions/protobuf/pom.xml | 30 +- .../sdk/extensions/protobuf/ByteStringCoder.java | 2 +- sdks/java/extensions/sketching/build.gradle | 39 + sdks/java/extensions/sketching/pom.xml | 19 +- .../extensions/sketching/ApproximateDistinct.java | 50 +- .../extensions/sketching/SketchFrequencies.java | 52 +- .../sdk/extensions/sketching/TDigestQuantiles.java | 349 + .../sketching/ApproximateDistinctTest.java | 4 - .../sketching/SketchFrequenciesTest.java | 7 +- .../extensions/sketching/TDigestQuantilesTest.java | 180 + sdks/java/extensions/sorter/build.gradle | 15 +- sdks/java/extensions/sorter/pom.xml | 14 +- .../extensions/sorter/BufferedExternalSorter.java | 2 +- .../beam/sdk/extensions/sorter/ExternalSorter.java | 6 +- .../beam/sdk/extensions/sorter/InMemorySorter.java | 8 +- .../beam/sdk/extensions/sorter/SortValues.java | 13 +- .../sorter/BufferedExternalSorterTest.java | 8 +- .../sdk/extensions/sorter/ExternalSorterTest.java | 8 +- .../sdk/extensions/sorter/InMemorySorterTest.java | 10 +- .../beam/sdk/extensions/sorter/SortValuesTest.java | 8 +- .../sdk/extensions/sorter/SorterTestUtils.java | 4 +- sdks/java/extensions/sql/build.gradle | 139 +- sdks/java/extensions/sql/pom.xml | 78 +- .../extensions/sql/src/main/codegen/config.fmpp | 93 +- .../sql/src/main/codegen/data/Parser.tdd | 75 - .../sql/src/main/codegen/includes/license.ftl | 17 - .../sql/src/main/codegen/includes/parserImpls.ftl | 297 +- .../beam/sdk/extensions/sql/BeamRecordSqlType.java | 250 - .../apache/beam/sdk/extensions/sql/BeamSql.java | 259 +- .../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 80 +- .../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 - .../sdk/extensions/sql/BeamSqlSeekableTable.java | 20 +- .../beam/sdk/extensions/sql/BeamSqlTable.java | 33 +- .../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 12 +- .../beam/sdk/extensions/sql/QueryTransform.java | 186 + .../beam/sdk/extensions/sql/RowSqlTypes.java | 178 + .../beam/sdk/extensions/sql/SchemaHelper.java | 66 + .../sdk/extensions/sql/example/BeamSqlExample.java | 61 +- .../extensions/sql/example/BeamSqlPojoExample.java | 151 + .../sdk/extensions/sql/example/model/Customer.java | 45 + .../sdk/extensions/sql/example/model/Order.java | 39 + .../extensions/sql/example/model/package-info.java | 20 + .../sdk/extensions/sql/example/package-info.java | 5 +- .../sdk/extensions/sql/impl/BeamCalciteSchema.java | 92 + .../sql/impl/BeamCalciteSchemaFactory.java | 46 + .../sdk/extensions/sql/impl/BeamCalciteTable.java | 93 + .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 111 +- .../beam/sdk/extensions/sql/impl/JdbcDriver.java | 103 + .../interpreter/BeamSqlExpressionExecutor.java | 20 +- .../sql/impl/interpreter/BeamSqlFnExecutor.java | 126 +- .../operator/BeamSqlCaseExpression.java | 25 +- .../operator/BeamSqlCastExpression.java | 137 +- .../operator/BeamSqlCorrelVariableExpression.java | 56 + .../operator/BeamSqlDefaultExpression.java | 38 + .../interpreter/operator/BeamSqlDotExpression.java | 57 + .../interpreter/operator/BeamSqlExpression.java | 26 +- .../operator/BeamSqlInputRefExpression.java | 12 +- .../interpreter/operator/BeamSqlPrimitive.java | 129 +- .../interpreter/operator/BeamSqlUdfExpression.java | 25 +- .../operator/BeamSqlWindowEndExpression.java | 12 +- .../operator/BeamSqlWindowExpression.java | 18 +- .../operator/BeamSqlWindowStartExpression.java | 16 +- .../sql/impl/interpreter/operator/UdafImpl.java | 28 +- .../arithmetic/BeamSqlArithmeticExpression.java | 30 +- .../arithmetic/BeamSqlDivideExpression.java | 7 +- .../arithmetic/BeamSqlMinusExpression.java | 7 +- .../operator/arithmetic/BeamSqlModExpression.java | 7 +- .../arithmetic/BeamSqlMultiplyExpression.java | 7 +- .../operator/arithmetic/BeamSqlPlusExpression.java | 7 +- .../operator/arithmetic/package-info.java | 4 +- .../operator/array/BeamSqlArrayExpression.java | 52 + .../operator/array/BeamSqlArrayItemExpression.java | 49 + .../interpreter/operator/array/package-info.java | 24 + .../collection/BeamSqlCardinalityExpression.java | 50 + .../collection/BeamSqlSingleElementExpression.java | 64 + .../operator/collection/package-info.java | 24 + .../comparison/BeamSqlCompareExpression.java | 72 +- .../comparison/BeamSqlEqualsExpression.java | 10 +- .../comparison/BeamSqlGreaterThanExpression.java | 10 +- .../BeamSqlGreaterThanOrEqualsExpression.java | 10 +- .../comparison/BeamSqlIsNotNullExpression.java | 18 +- .../comparison/BeamSqlIsNullExpression.java | 18 +- .../comparison/BeamSqlLessThanExpression.java | 10 +- .../BeamSqlLessThanOrEqualsExpression.java | 10 +- .../comparison/BeamSqlNotEqualsExpression.java | 10 +- .../operator/comparison/package-info.java | 4 +- .../date/BeamSqlCurrentDateExpression.java | 17 +- .../date/BeamSqlCurrentTimeExpression.java | 19 +- .../date/BeamSqlCurrentTimestampExpression.java | 15 +- .../operator/date/BeamSqlDateCeilExpression.java | 23 +- .../operator/date/BeamSqlDateFloorExpression.java | 23 +- .../date/BeamSqlDatetimeMinusExpression.java | 24 +- .../date/BeamSqlDatetimePlusExpression.java | 75 +- .../operator/date/BeamSqlExtractExpression.java | 80 +- .../date/BeamSqlIntervalMultiplyExpression.java | 51 +- .../BeamSqlTimestampMinusIntervalExpression.java | 22 +- .../BeamSqlTimestampMinusTimestampExpression.java | 34 +- .../interpreter/operator/date/TimeUnitUtils.java | 9 +- .../interpreter/operator/date/package-info.java | 4 +- .../operator/logical/BeamSqlAndExpression.java | 17 +- .../operator/logical/BeamSqlLogicalExpression.java | 5 +- .../operator/logical/BeamSqlNotExpression.java | 15 +- .../operator/logical/BeamSqlOrExpression.java | 21 +- .../interpreter/operator/logical/package-info.java | 4 +- .../operator/map/BeamSqlMapExpression.java | 53 + .../operator/map/BeamSqlMapItemExpression.java | 49 + .../interpreter/operator/map/package-info.java | 24 + .../operator/math/BeamSqlAbsExpression.java | 30 +- .../operator/math/BeamSqlAcosExpression.java | 11 +- .../operator/math/BeamSqlAsinExpression.java | 11 +- .../operator/math/BeamSqlAtan2Expression.java | 16 +- .../operator/math/BeamSqlAtanExpression.java | 11 +- .../operator/math/BeamSqlCeilExpression.java | 11 +- .../operator/math/BeamSqlCosExpression.java | 11 +- .../operator/math/BeamSqlCotExpression.java | 11 +- .../operator/math/BeamSqlDegreesExpression.java | 11 +- .../operator/math/BeamSqlExpExpression.java | 11 +- .../operator/math/BeamSqlFloorExpression.java | 11 +- .../operator/math/BeamSqlLnExpression.java | 11 +- .../operator/math/BeamSqlLogExpression.java | 11 +- .../operator/math/BeamSqlMathBinaryExpression.java | 28 +- .../operator/math/BeamSqlMathUnaryExpression.java | 22 +- .../operator/math/BeamSqlPiExpression.java | 14 +- .../operator/math/BeamSqlPowerExpression.java | 16 +- .../operator/math/BeamSqlRadiansExpression.java | 11 +- .../operator/math/BeamSqlRandExpression.java | 12 +- .../math/BeamSqlRandIntegerExpression.java | 18 +- .../operator/math/BeamSqlRoundExpression.java | 49 +- .../operator/math/BeamSqlSignExpression.java | 43 +- .../operator/math/BeamSqlSinExpression.java | 11 +- .../operator/math/BeamSqlTanExpression.java | 11 +- .../operator/math/BeamSqlTruncateExpression.java | 54 +- .../interpreter/operator/math/package-info.java | 4 +- .../impl/interpreter/operator/package-info.java | 4 +- .../reinterpret/BeamSqlReinterpretExpression.java | 33 +- .../DatetimeReinterpretConversions.java | 15 +- .../reinterpret/IntegerReinterpretConversions.java | 4 +- .../reinterpret/ReinterpretConversion.java | 27 +- .../operator/reinterpret/Reinterpreter.java | 12 +- .../operator/reinterpret/package-info.java | 4 +- .../operator/row/BeamSqlFieldAccessExpression.java | 70 + .../interpreter/operator/row/package-info.java | 24 + .../string/BeamSqlCharLengthExpression.java | 13 +- .../operator/string/BeamSqlConcatExpression.java | 24 +- .../operator/string/BeamSqlInitCapExpression.java | 13 +- .../operator/string/BeamSqlLowerExpression.java | 13 +- .../operator/string/BeamSqlOverlayExpression.java | 29 +- .../operator/string/BeamSqlPositionExpression.java | 25 +- .../string/BeamSqlStringUnaryExpression.java | 7 +- .../string/BeamSqlSubstringExpression.java | 24 +- .../operator/string/BeamSqlTrimExpression.java | 35 +- .../operator/string/BeamSqlUpperExpression.java | 13 +- .../interpreter/operator/string/package-info.java | 4 +- .../sql/impl/interpreter/package-info.java | 4 +- .../beam/sdk/extensions/sql/impl/package-info.java | 4 +- .../extensions/sql/impl/parser/BeamSqlParser.java | 50 - .../sql/impl/parser/ColumnConstraint.java | 42 - .../sql/impl/parser/ColumnDefinition.java | 56 - .../extensions/sql/impl/parser/ParserUtils.java | 64 - .../sql/impl/parser/SqlCheckConstraint.java | 75 + .../sql/impl/parser/SqlColumnDeclaration.java | 73 + .../extensions/sql/impl/parser/SqlCreateTable.java | 207 +- .../extensions/sql/impl/parser/SqlDDLKeywords.java | 27 - .../extensions/sql/impl/parser/SqlDdlNodes.java | 64 + .../extensions/sql/impl/parser/SqlDropObject.java | 88 + .../extensions/sql/impl/parser/SqlDropTable.java | 36 + .../extensions/sql/impl/parser/UnparseUtil.java | 59 - .../extensions/sql/impl/parser/package-info.java | 4 +- .../sql/impl/planner/BeamQueryPlanner.java | 183 +- .../sql/impl/planner/BeamRelDataTypeSystem.java | 10 +- .../extensions/sql/impl/planner/BeamRuleSets.java | 57 +- .../extensions/sql/impl/planner/package-info.java | 3 +- .../sql/impl/rel/BeamAggregationRel.java | 267 +- .../sql/impl/rel/BeamEnumerableConverter.java | 169 + .../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 43 +- .../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 109 +- .../extensions/sql/impl/rel/BeamIOSourceRel.java | 47 +- .../extensions/sql/impl/rel/BeamIntersectRel.java | 35 +- .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 408 +- .../sql/impl/rel/BeamLogicalConvention.java | 12 +- .../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 28 +- .../extensions/sql/impl/rel/BeamProjectRel.java | 61 +- .../sdk/extensions/sql/impl/rel/BeamRelNode.java | 17 +- .../sql/impl/rel/BeamSetOperatorRelBase.java | 50 +- .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 132 +- .../extensions/sql/impl/rel/BeamSqlRelUtils.java | 27 +- .../extensions/sql/impl/rel/BeamUncollectRel.java | 88 + .../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 59 +- .../sdk/extensions/sql/impl/rel/BeamUnnestRel.java | 158 + .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 59 +- .../sdk/extensions/sql/impl/rel/package-info.java | 9 +- .../sql/impl/rule/AggregateWindowFactory.java | 152 + .../sql/impl/rule/AggregateWindowField.java | 51 + .../sql/impl/rule/BeamAggregationRule.java | 134 +- .../sql/impl/rule/BeamEnumerableConverterRule.java | 45 + .../extensions/sql/impl/rule/BeamFilterRule.java | 8 +- .../extensions/sql/impl/rule/BeamIOSinkRule.java | 53 +- .../extensions/sql/impl/rule/BeamIOSourceRule.java | 49 - .../sql/impl/rule/BeamIntersectRule.java | 18 +- .../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 22 +- .../extensions/sql/impl/rule/BeamMinusRule.java | 14 +- .../extensions/sql/impl/rule/BeamProjectRule.java | 37 +- .../sdk/extensions/sql/impl/rule/BeamSortRule.java | 14 +- .../sql/impl/rule/BeamUncollectRule.java | 49 + .../extensions/sql/impl/rule/BeamUnionRule.java | 14 +- .../extensions/sql/impl/rule/BeamUnnestRule.java | 58 + .../extensions/sql/impl/rule/BeamValuesRule.java | 14 +- .../sdk/extensions/sql/impl/rule/package-info.java | 8 +- .../extensions/sql/impl/schema/BaseBeamTable.java | 18 +- .../sdk/extensions/sql/impl/schema/BeamIOType.java | 8 +- .../sql/impl/schema/BeamPCollectionTable.java | 30 +- .../extensions/sql/impl/schema/BeamTableUtils.java | 86 +- .../extensions/sql/impl/schema/package-info.java | 5 +- .../impl/transform/BeamAggregationTransforms.java | 289 +- .../impl/transform/BeamBuiltinAggregations.java | 312 +- .../sql/impl/transform/BeamJoinTransforms.java | 245 +- .../impl/transform/BeamSetOperatorsTransforms.java | 47 +- .../sql/impl/transform/BeamSqlFilterFn.java | 15 +- .../impl/transform/BeamSqlOutputToConsoleFn.java | 12 +- .../sql/impl/transform/BeamSqlProjectFn.java | 47 +- .../impl/transform/agg/CovarianceAccumulator.java | 139 + .../sql/impl/transform/agg/CovarianceFn.java | 112 + .../impl/transform/agg/VarianceAccumulator.java | 87 + .../sql/impl/transform/agg/VarianceFn.java | 134 + .../sql/impl/transform/agg/package-info.java | 24 + .../sql/impl/transform/package-info.java | 4 +- .../sql/impl/utils/BigDecimalConverter.java | 53 + .../extensions/sql/impl/utils/CalciteUtils.java | 200 +- .../extensions/sql/impl/utils/SqlTypeUtils.java | 14 +- .../extensions/sql/impl/utils/package-info.java | 4 +- .../beam/sdk/extensions/sql/meta/Column.java | 51 - .../apache/beam/sdk/extensions/sql/meta/Table.java | 40 +- .../beam/sdk/extensions/sql/meta/package-info.java | 4 +- .../sql/meta/provider/BeamSqlTableProvider.java | 71 + .../meta/provider/InMemoryMetaTableProvider.java | 42 + .../extensions/sql/meta/provider/MetaUtils.java | 40 - .../sql/meta/provider/TableProvider.java | 33 +- .../meta/provider/bigquery/BeamBigQueryTable.java | 74 + .../provider/bigquery/BigQueryTableProvider.java | 55 + .../sql/meta/provider/bigquery/package-info.java | 20 + .../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 101 +- .../sql/meta/provider/kafka/BeamKafkaTable.java | 90 +- .../meta/provider/kafka/KafkaTableProvider.java | 37 +- .../sql/meta/provider/kafka/package-info.java | 4 +- .../extensions/sql/meta/provider/package-info.java | 4 +- .../meta/provider/pubsub/PubsubIOJsonTable.java | 202 + .../provider/pubsub/PubsubJsonTableProvider.java | 95 + .../meta/provider/pubsub/PubsubMessageToRow.java | 144 + .../sql/meta/provider/pubsub/package-info.java | 24 + .../sql/meta/provider/text/BeamTextCSVTable.java | 39 +- .../provider/text/BeamTextCSVTableIOReader.java | 38 +- .../provider/text/BeamTextCSVTableIOWriter.java | 42 +- .../sql/meta/provider/text/BeamTextTable.java | 13 +- .../sql/meta/provider/text/TextTableProvider.java | 43 +- .../sql/meta/provider/text/package-info.java | 4 +- .../sql/meta/store/InMemoryMetaStore.java | 63 +- .../sdk/extensions/sql/meta/store/MetaStore.java | 31 +- .../extensions/sql/meta/store/package-info.java | 4 +- .../beam/sdk/extensions/sql/package-info.java | 4 +- .../resources/META-INF/services/java.sql.Driver | 1 + .../sql/src/main/resources/saffron.properties | 21 + .../sdk/extensions/sql/BeamRecordSqlTypeTest.java | 115 - .../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 63 - .../sdk/extensions/sql/BeamSqlCliPubsubTest.java | 54 + .../beam/sdk/extensions/sql/BeamSqlCliTest.java | 210 +- .../sql/BeamSqlDslAggregationCovarianceTest.java | 101 + .../extensions/sql/BeamSqlDslAggregationTest.java | 674 +- .../sql/BeamSqlDslAggregationVarianceTest.java | 99 + .../sdk/extensions/sql/BeamSqlDslArrayTest.java | 398 + .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 175 +- .../sdk/extensions/sql/BeamSqlDslFilterTest.java | 76 +- .../sdk/extensions/sql/BeamSqlDslJoinTest.java | 338 +- .../extensions/sql/BeamSqlDslNestedRowsTest.java | 282 + .../sdk/extensions/sql/BeamSqlDslProjectTest.java | 148 +- .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 177 +- .../beam/sdk/extensions/sql/BeamSqlMapTest.java | 169 + .../sdk/extensions/sql/BeamSqlNonAsciiTest.java | 38 +- .../extensions/sql/InferredRowCoderSqlTest.java | 226 + .../beam/sdk/extensions/sql/JsonToRowSqlTest.java | 83 + .../sdk/extensions/sql/SqlSchemaFactoryTest.java | 108 + .../apache/beam/sdk/extensions/sql/TestUtils.java | 213 +- .../sdk/extensions/sql/impl/JdbcDriverTest.java | 113 + .../impl/interpreter/BeamSqlFnExecutorTest.java | 345 +- .../interpreter/BeamSqlFnExecutorTestBase.java | 64 +- .../operator/BeamNullExperssionTest.java | 31 +- .../operator/BeamSqlAndOrExpressionTest.java | 19 +- .../operator/BeamSqlCaseExpressionTest.java | 27 +- .../operator/BeamSqlCastExpressionTest.java | 84 +- .../operator/BeamSqlCompareExpressionTest.java | 135 +- .../operator/BeamSqlDotExpressionTest.java | 76 + .../operator/BeamSqlInputRefExpressionTest.java | 22 +- .../interpreter/operator/BeamSqlPrimitiveTest.java | 36 +- .../operator/BeamSqlReinterpretExpressionTest.java | 58 +- .../operator/BeamSqlUdfExpressionTest.java | 16 +- .../BeamSqlArithmeticExpressionTest.java | 115 +- .../operator/array/BeamSqlArrayExpressionTest.java | 78 + .../array/BeamSqlArrayItemExpressionTest.java | 90 + .../BeamSqlCardinalityExpressionTest.java | 86 + .../BeamSqlSingleElementExpressionTest.java | 87 + .../date/BeamSqlCurrentDateExpressionTest.java | 9 +- .../date/BeamSqlCurrentTimeExpressionTest.java | 12 +- .../BeamSqlCurrentTimestampExpressionTest.java | 12 +- .../date/BeamSqlDateCeilExpressionTest.java | 23 +- .../date/BeamSqlDateExpressionTestBase.java | 34 +- .../date/BeamSqlDateFloorExpressionTest.java | 22 +- .../date/BeamSqlDatetimeMinusExpressionTest.java | 66 +- .../date/BeamSqlDatetimePlusExpressionTest.java | 65 +- .../date/BeamSqlExtractExpressionTest.java | 75 +- .../BeamSqlIntervalMultiplyExpressionTest.java | 29 +- ...eamSqlTimestampMinusIntervalExpressionTest.java | 59 +- ...amSqlTimestampMinusTimestampExpressionTest.java | 138 +- .../operator/date/TimeUnitUtilsTest.java | 47 +- .../operator/logical/BeamSqlNotExpressionTest.java | 17 +- .../math/BeamSqlMathBinaryExpressionTest.java | 131 +- .../math/BeamSqlMathUnaryExpressionTest.java | 255 +- .../DatetimeReinterpretConversionsTest.java | 44 +- .../IntegerReinterpretConversionsTest.java | 46 +- .../reinterpret/ReinterpretConversionTest.java | 69 +- .../operator/reinterpret/ReinterpreterTest.java | 76 +- .../row/BeamSqlFieldAccessExpressionTest.java | 86 + .../string/BeamSqlCharLengthExpressionTest.java | 16 +- .../string/BeamSqlConcatExpressionTest.java | 17 +- .../string/BeamSqlInitCapExpressionTest.java | 24 +- .../string/BeamSqlLowerExpressionTest.java | 14 +- .../string/BeamSqlOverlayExpressionTest.java | 32 +- .../string/BeamSqlPositionExpressionTest.java | 24 +- .../string/BeamSqlStringUnaryExpressionTest.java | 8 +- .../string/BeamSqlSubstringExpressionTest.java | 47 +- .../operator/string/BeamSqlTrimExpressionTest.java | 52 +- .../string/BeamSqlUpperExpressionTest.java | 14 +- .../sql/impl/parser/BeamDDLNestedTypesTest.java | 136 + .../extensions/sql/impl/parser/BeamDDLTest.java | 160 + .../sql/impl/parser/BeamSqlParserTest.java | 167 - .../sql/impl/parser/ParserTestUtils.java | 33 + .../sdk/extensions/sql/impl/rel/BaseRelTest.java | 25 +- .../sql/impl/rel/BeamEnumerableConverterTest.java | 161 + .../sql/impl/rel/BeamIntersectRelTest.java | 107 +- .../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 182 +- .../rel/BeamJoinRelUnboundedVsBoundedTest.java | 337 +- .../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 243 +- .../extensions/sql/impl/rel/BeamMinusRelTest.java | 107 +- .../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 85 +- .../extensions/sql/impl/rel/BeamSortRelTest.java | 369 +- .../extensions/sql/impl/rel/BeamUnionRelTest.java | 97 +- .../extensions/sql/impl/rel/BeamValuesRelTest.java | 91 +- .../sdk/extensions/sql/impl/rel/CheckSize.java | 15 +- .../sql/impl/schema/BeamSqlRowCoderTest.java | 73 +- .../transform/BeamAggregationTransformTest.java | 299 +- .../schema/transform/BeamTransformBaseTest.java | 190 +- .../transform/agg/VarianceAccumulatorTest.java | 121 + .../sql/impl/transform/agg/VarianceFnTest.java | 104 + .../sql/impl/utils/BigDecimalConverterTest.java | 51 + .../sql/impl/utils/SqlTypeUtilsTest.java | 16 +- .../BeamSqlArithmeticOperatorsIntegrationTest.java | 190 +- ...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 133 +- .../BeamSqlComparisonOperatorsIntegrationTest.java | 454 +- ...BeamSqlConditionalFunctionsIntegrationTest.java | 51 +- .../BeamSqlDateFunctionsIntegrationTest.java | 331 +- .../BeamSqlLogicalFunctionsIntegrationTest.java | 25 +- .../BeamSqlMathFunctionsIntegrationTest.java | 413 +- .../BeamSqlStringFunctionsIntegrationTest.java | 40 +- .../bigquery/BigQueryTableProviderTest.java | 67 + .../meta/provider/kafka/BeamKafkaCSVTableTest.java | 82 +- .../provider/kafka/KafkaTableProviderTest.java | 27 +- .../sql/meta/provider/pubsub/PubsubJsonIT.java | 207 + .../pubsub/PubsubJsonTableProviderTest.java | 137 + .../provider/pubsub/PubsubMessageToRowTest.java | 187 + .../meta/provider/text/BeamTextCSVTableTest.java | 125 +- .../meta/provider/text/TextTableProviderTest.java | 24 +- .../sql/meta/store/InMemoryMetaStoreTest.java | 95 +- .../extensions/sql/mock/MockedBoundedTable.java | 69 +- .../beam/sdk/extensions/sql/mock/MockedTable.java | 17 +- .../extensions/sql/mock/MockedUnboundedTable.java | 44 +- .../extensions/sql/utils/QuickCheckGenerators.java | 138 + .../beam/sdk/extensions/sql/utils/RowAsserts.java | 52 + sdks/java/fn-execution/build.gradle | 16 +- sdks/java/fn-execution/pom.xml | 36 +- .../java/org/apache/beam/sdk/fn/IdGenerator.java | 25 + .../java/org/apache/beam/sdk/fn/IdGenerators.java | 40 + .../sdk/fn/data/BeamFnDataInboundObserver.java | 2 +- .../data/CompletableFutureInboundDataClient.java | 73 + .../fn/data/SettableFutureInboundDataClient.java | 63 - .../sdk/fn/function/CloseableThrowingConsumer.java | 22 + .../beam/sdk/fn/function/ThrowingBiConsumer.java | 33 + .../beam/sdk/fn/function/ThrowingBiFunction.java | 32 + .../beam/sdk/fn/function/ThrowingConsumer.java | 31 + .../beam/sdk/fn/function/ThrowingFunction.java | 32 + .../beam/sdk/fn/function/ThrowingRunnable.java | 30 + .../apache/beam/sdk/fn/function/package-info.java | 20 + .../java/org/apache/beam/sdk/fn/package-info.java | 19 + .../sdk/fn/stream/BufferingStreamObserver.java | 2 +- .../org/apache/beam/sdk/fn/stream/DataStreams.java | 123 +- .../beam/sdk/fn/stream/DirectStreamObserver.java | 55 +- .../beam/sdk/fn/stream/StreamObserverFactory.java | 2 +- .../sdk/fn/stream/SynchronizedStreamObserver.java | 64 + .../fn/test/InProcessManagedChannelFactory.java | 41 + .../org/apache/beam/sdk/fn/test/TestExecutors.java | 89 + .../org/apache/beam/sdk/fn/test/TestStreams.java | 173 + .../org/apache/beam/sdk/fn/test/package-info.java | 20 + .../sdk/fn/windowing/EncodedBoundedWindow.java | 3 +- .../org/apache/beam/sdk/fn/IdGeneratorsTest.java | 61 + .../BeamFnDataBufferingOutboundObserverTest.java | 4 +- .../CompletableFutureInboundDataClientTest.java | 165 + .../beam/sdk/fn/stream/AdvancingPhaserTest.java | 5 +- .../sdk/fn/stream/BufferingStreamObserverTest.java | 2 +- .../apache/beam/sdk/fn/stream/DataStreamsTest.java | 60 +- .../sdk/fn/stream/DirectStreamObserverTest.java | 53 +- .../java/org/apache/beam/sdk/fn/test/Consumer.java | 26 - .../java/org/apache/beam/sdk/fn/test/Supplier.java | 26 - .../org/apache/beam/sdk/fn/test/TestExecutors.java | 88 - .../apache/beam/sdk/fn/test/TestExecutorsTest.java | 4 +- .../org/apache/beam/sdk/fn/test/TestStreams.java | 171 - .../apache/beam/sdk/fn/test/TestStreamsTest.java | 4 +- sdks/java/harness/build.gradle | 62 +- sdks/java/harness/pom.xml | 61 +- .../beam/fn/harness/AssignWindowsRunner.java | 111 + .../beam/fn/harness/BeamFnDataReadRunner.java | 2 +- .../beam/fn/harness/BeamFnDataWriteRunner.java | 2 +- .../beam/fn/harness/BoundedSourceRunner.java | 2 +- .../org/apache/beam/fn/harness/FlattenRunner.java | 2 +- .../apache/beam/fn/harness/FnApiDoFnRunner.java | 662 +- .../java/org/apache/beam/fn/harness/FnHarness.java | 13 +- .../org/apache/beam/fn/harness/IdGenerator.java | 33 - .../org/apache/beam/fn/harness/MapFnRunners.java | 180 + .../beam/fn/harness/PTransformRunnerFactory.java | 2 +- .../beam/fn/harness/WindowMappingFnRunner.java | 68 + .../fn/harness/control/BeamFnControlClient.java | 94 +- .../fn/harness/control/ProcessBundleHandler.java | 30 +- .../data/BeamFnDataBufferingOutboundObserver.java | 4 +- .../data/CompletableFutureInboundDataClient.java | 73 - .../fn/harness/fn/CloseableThrowingConsumer.java | 23 - .../beam/fn/harness/fn/ThrowingBiConsumer.java | 33 - .../beam/fn/harness/fn/ThrowingBiFunction.java | 32 - .../beam/fn/harness/fn/ThrowingConsumer.java | 32 - .../beam/fn/harness/fn/ThrowingFunction.java | 32 - .../beam/fn/harness/fn/ThrowingRunnable.java | 30 - .../apache/beam/fn/harness/fn/package-info.java | 22 - .../fn/harness/logging/BeamFnLoggingClient.java | 4 +- .../apache/beam/fn/harness/state/BagUserState.java | 66 +- .../harness/state/BeamFnStateGrpcClientCache.java | 16 +- .../state/LazyCachingIteratorToIterable.java | 17 + .../beam/fn/harness/state/MultimapSideInput.java | 85 + .../fn/harness/state/StateFetchingIterators.java | 32 +- .../beam/fn/harness/AssignWindowsRunnerTest.java | 369 + .../beam/fn/harness/BeamFnDataReadRunnerTest.java | 5 +- .../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 3 +- .../beam/fn/harness/BoundedSourceRunnerTest.java | 2 +- .../apache/beam/fn/harness/FlattenRunnerTest.java | 84 +- .../beam/fn/harness/FnApiDoFnRunnerTest.java | 301 +- .../org/apache/beam/fn/harness/FnHarnessTest.java | 19 +- .../apache/beam/fn/harness/IdGeneratorTest.java | 40 - .../apache/beam/fn/harness/MapFnRunnersTest.java | 185 + .../beam/fn/harness/WindowMappingFnRunnerTest.java | 68 + .../harness/control/BeamFnControlClientTest.java | 173 +- .../harness/control/ProcessBundleHandlerTest.java | 144 +- .../data/BeamFnDataInboundObserverTest.java | 1 + .../CompletableFutureInboundDataClientTest.java | 166 - .../harness/logging/BeamFnLoggingClientTest.java | 2 - .../beam/fn/harness/state/BagUserStateTest.java | 59 +- .../state/BeamFnStateGrpcClientCacheTest.java | 13 +- .../state/LazyCachingIteratorToIterableTest.java | 14 + .../fn/harness/state/MultimapSideInputTest.java | 73 + .../harness/state/StateFetchingIteratorsTest.java | 4 +- sdks/java/io/amazon-web-services/build.gradle | 47 + sdks/java/io/amazon-web-services/pom.xml | 188 + .../apache/beam/sdk/io/aws/options/AwsModule.java | 237 + .../apache/beam/sdk/io/aws/options/AwsOptions.java | 83 + .../aws/options/AwsPipelineOptionsRegistrar.java | 38 + .../apache/beam/sdk/io/aws/options/S3Options.java | 84 + .../beam/sdk/io/aws/options/package-info.java | 22 + .../apache/beam/sdk/io/aws/s3/S3FileSystem.java | 691 + .../beam/sdk/io/aws/s3/S3FileSystemRegistrar.java | 43 + .../io/aws/s3/S3ReadableSeekableByteChannel.java | 172 + .../apache/beam/sdk/io/aws/s3/S3ResourceId.java | 194 + .../beam/sdk/io/aws/s3/S3WritableByteChannel.java | 181 + .../apache/beam/sdk/io/aws/s3/package-info.java | 21 + .../beam/sdk/io/aws/options/AwsModuleTest.java | 185 + .../beam/sdk/io/aws/s3/MatchResultMatcher.java | 118 + .../beam/sdk/io/aws/s3/S3FileSystemTest.java | 661 + .../beam/sdk/io/aws/s3/S3ResourceIdTest.java | 307 + .../org/apache/beam/sdk/io/aws/s3/S3TestUtils.java | 82 + .../sdk/io/aws/s3/S3WritableByteChannelTest.java | 158 + sdks/java/io/amqp/build.gradle | 16 +- sdks/java/io/amqp/pom.xml | 11 +- .../apache/beam/sdk/io/amqp/AmqpMessageCoder.java | 2 - .../io/amqp/AmqpMessageCoderProviderRegistrar.java | 2 - .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 2 - sdks/java/io/cassandra/build.gradle | 31 +- sdks/java/io/cassandra/pom.xml | 15 +- .../apache/beam/sdk/io/cassandra/CassandraIO.java | 102 +- .../beam/sdk/io/cassandra/CassandraService.java | 37 +- .../sdk/io/cassandra/CassandraServiceImpl.java | 264 +- .../apache/beam/sdk/io/cassandra/RingRange.java | 60 + .../beam/sdk/io/cassandra/SplitGenerator.java | 179 + .../apache/beam/sdk/io/cassandra/package-info.java | 4 +- .../beam/sdk/io/cassandra/CassandraIOIT.java | 60 +- .../beam/sdk/io/cassandra/CassandraIOTest.java | 70 +- .../sdk/io/cassandra/CassandraServiceImplTest.java | 100 +- .../sdk/io/cassandra/CassandraTestDataSet.java | 72 +- .../beam/sdk/io/cassandra/SplitGeneratorTest.java | 141 + sdks/java/io/common/build.gradle | 15 +- sdks/java/io/common/pom.xml | 14 +- .../org/apache/beam/sdk/io/common/HashingFn.java | 19 + .../org/apache/beam/sdk/io/common/IOITHelper.java | 39 + .../beam/sdk/io/common/IOTestPipelineOptions.java | 26 + .../org/apache/beam/sdk/io/common/TestRow.java | 12 +- .../elasticsearch-tests-2/build.gradle | 33 +- .../elasticsearch-tests-2/pom.xml | 2 +- .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 31 +- .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 26 +- .../elasticsearch-tests-5/build.gradle | 33 +- .../elasticsearch-tests-5/pom.xml | 8 +- .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 29 +- .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 34 +- .../java/org/elasticsearch/bootstrap/JarHell.java | 6 +- .../elasticsearch-tests-common/build.gradle | 28 +- .../elasticsearch-tests-common/pom.xml | 4 +- .../io/elasticsearch/ElasticSearchIOTestUtils.java | 93 +- .../io/elasticsearch/ElasticsearchIOITCommon.java | 14 +- .../elasticsearch/ElasticsearchIOTestCommon.java | 196 +- sdks/java/io/elasticsearch-tests/pom.xml | 10 +- sdks/java/io/elasticsearch/build.gradle | 16 +- sdks/java/io/elasticsearch/pom.xml | 9 +- .../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 205 +- sdks/java/io/file-based-io-tests/build.gradle | 33 +- sdks/java/io/file-based-io-tests/pom.xml | 28 +- .../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 23 +- .../beam/sdk/io/common/FileBasedIOITHelper.java | 17 +- .../apache/beam/sdk/io/parquet/ParquetIOIT.java | 132 + .../java/org/apache/beam/sdk/io/text/TextIOIT.java | 19 +- .../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 21 +- .../java/org/apache/beam/sdk/io/xml/XmlIOIT.java | 209 + sdks/java/io/google-cloud-platform/build.gradle | 55 +- sdks/java/io/google-cloud-platform/pom.xml | 50 +- .../beam/sdk/io/gcp/bigquery/BatchLoads.java | 63 +- .../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 6 +- .../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 28 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 248 +- .../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 98 +- .../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 2 +- .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 45 +- .../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 17 +- .../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 184 + .../beam/sdk/io/gcp/bigquery/CreateTables.java | 168 +- .../sdk/io/gcp/bigquery/DynamicDestinations.java | 2 +- .../gcp/bigquery/DynamicDestinationsHelpers.java | 1 + .../org/apache/beam/sdk/io/gcp/bigquery/Mode.java | 29 + .../io/gcp/bigquery/PassThroughThenCleanup.java | 10 + .../sdk/io/gcp/bigquery/StandardSQLTypeName.java | 54 + .../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 6 - .../beam/sdk/io/gcp/bigquery/TableDestination.java | 11 +- .../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 5 - .../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 3 - .../gcp/bigquery/WriteGroupedRecordsToFiles.java | 3 - .../beam/sdk/io/gcp/bigquery/WritePartition.java | 7 +- .../beam/sdk/io/gcp/bigquery/WriteRename.java | 2 +- .../beam/sdk/io/gcp/bigquery/WriteTables.java | 45 +- .../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 16 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 151 +- .../beam/sdk/io/gcp/bigtable/BigtableService.java | 4 +- .../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 24 +- .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +- .../beam/sdk/io/gcp/pubsub/PubsubClient.java | 37 +- .../beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java | 2 +- .../beam/sdk/io/gcp/pubsub/PubsubJsonClient.java | 4 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 2 +- .../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 171 + .../beam/sdk/io/gcp/pubsub/TestPubsubOptions.java | 28 + .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 291 + .../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 161 + .../sdk/io/gcp/spanner/CreateTransactionFn.java | 29 +- .../sdk/io/gcp/spanner/MutationCellCounter.java | 57 + .../beam/sdk/io/gcp/spanner/MutationGroup.java | 19 +- .../sdk/io/gcp/spanner/MutationGroupEncoder.java | 14 +- .../sdk/io/gcp/spanner/MutationSizeEstimator.java | 17 +- .../beam/sdk/io/gcp/spanner/MutationUtils.java | 38 + .../beam/sdk/io/gcp/spanner/NaiveSpannerRead.java | 111 + .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 - .../beam/sdk/io/gcp/spanner/OrderedCode.java | 20 +- .../beam/sdk/io/gcp/spanner/ReadOperation.java | 13 +- .../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 32 +- .../sdk/io/gcp/spanner/SerializedMutation.java | 2 + .../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 17 +- .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 23 +- .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 311 +- .../beam/sdk/io/gcp/spanner/SpannerSchema.java | 103 +- .../sdk/io/gcp/spanner/SpannerWriteResult.java | 72 + .../beam/sdk/io/gcp/spanner/Transaction.java | 10 +- .../cloud/spanner/FakeBatchTransactionId.java | 54 + .../google/cloud/spanner/FakePartitionFactory.java | 43 + .../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 6 +- .../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 3 +- .../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 43 +- .../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 33 - .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 35 +- .../beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java | 7 +- .../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 193 + .../sdk/io/gcp/bigquery/FakeBigQueryServices.java | 2 +- .../sdk/io/gcp/bigquery/FakeDatasetService.java | 6 +- .../beam/sdk/io/gcp/bigquery/FakeJobService.java | 43 +- .../sdk/io/gcp/bigquery/TableRowJsonCoderTest.java | 1 + .../sdk/io/gcp/bigtable/BigtableConfigTest.java | 39 +- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 257 +- .../io/gcp/bigtable/BigtableServiceImplTest.java | 19 +- .../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 16 +- .../beam/sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +- .../beam/sdk/io/gcp/datastore/V1WriteIT.java | 4 +- .../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 15 +- .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 5 - .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 20 +- .../sdk/io/gcp/pubsub/PubsubTestClientTest.java | 13 +- .../sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java | 26 +- .../io/gcp/pubsub/PubsubUnboundedSourceTest.java | 24 +- .../sdk/io/gcp/spanner/FakeServiceFactory.java | 13 + .../io/gcp/spanner/MutationGroupEncoderTest.java | 12 +- .../io/gcp/spanner/MutationSizeEstimatorTest.java | 48 + .../beam/sdk/io/gcp/spanner/OrderedCodeTest.java | 4 +- .../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 7 +- .../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 329 +- .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 105 +- .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 97 +- .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 142 +- .../beam/sdk/io/gcp/testing/package-info.java | 0 sdks/java/io/hadoop-common/build.gradle | 14 +- sdks/java/io/hadoop-common/pom.xml | 10 +- .../sdk/io/hadoop/SerializableConfiguration.java | 8 +- .../apache/beam/sdk/io/hadoop/WritableCoder.java | 10 +- .../io/hadoop/SerializableConfigurationTest.java | 6 +- sdks/java/io/hadoop-file-system/build.gradle | 18 +- sdks/java/io/hadoop-file-system/pom.xml | 12 +- .../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 4 +- .../beam/sdk/io/hdfs/HadoopFileSystemModule.java | 9 +- .../beam/sdk/io/hdfs/HadoopFileSystemOptions.java | 1 + .../apache/beam/sdk/io/hdfs/HadoopResourceId.java | 3 +- .../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 3 +- .../sdk/io/hdfs/HadoopFileSystemOptionsTest.java | 1 + .../sdk/io/hdfs/HadoopFileSystemRegistrarTest.java | 2 +- .../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 87 +- .../beam/sdk/io/hdfs/HadoopResourceIdTest.java | 6 +- sdks/java/io/hadoop-input-format/build.gradle | 44 +- sdks/java/io/hadoop-input-format/pom.xml | 46 +- .../io/hadoop/inputformat/HadoopInputFormatIO.java | 101 +- .../ConfigurableEmployeeInputFormat.java | 32 +- .../io/hadoop/inputformat/EmployeeInputFormat.java | 33 +- .../io/hadoop/inputformat/HIFIOCassandraIT.java | 11 +- .../sdk/io/hadoop/inputformat/HIFIOElasticIT.java | 16 +- .../hadoop/inputformat/HIFIOWithElasticTest.java | 28 +- .../HIFIOWithEmbeddedCassandraTest.java | 27 +- .../sdk/io/hadoop/inputformat/HIFITestOptions.java | 2 +- .../hadoop/inputformat/HadoopInputFormatIOIT.java | 21 +- .../inputformat/HadoopInputFormatIOTest.java | 30 +- .../ReuseObjectsEmployeeInputFormat.java | 44 +- .../io/hadoop/inputformat/TestEmployeeDataSet.java | 19 +- .../io/hadoop/inputformat/TestRowDBWritable.java | 6 +- sdks/java/io/hbase/build.gradle | 20 +- sdks/java/io/hbase/pom.xml | 35 +- .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 169 +- .../beam/sdk/io/hbase/HBaseMutationCoder.java | 4 +- .../org/apache/beam/sdk/io/hbase/HBaseUtils.java | 129 + .../org/apache/beam/sdk/io/hbase/package-info.java | 2 +- .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 93 +- .../beam/sdk/io/hbase/HBaseMutationCoderTest.java | 7 +- .../beam/sdk/io/hbase/SerializableScanTest.java | 5 +- sdks/java/io/hcatalog/build.gradle | 27 +- sdks/java/io/hcatalog/pom.xml | 154 +- .../apache/beam/sdk/io/hcatalog/HCatalogIOIT.java | 196 + .../beam/sdk/io/hcatalog/HCatalogIOTest.java | 7 +- .../beam/sdk/io/hcatalog/HCatalogIOTestUtils.java | 7 +- .../sdk/io/hcatalog/HiveDatabaseTestHelper.java | 56 + sdks/java/io/jdbc/build.gradle | 32 +- sdks/java/io/jdbc/pom.xml | 38 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 307 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 9 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 137 +- sdks/java/io/jms/build.gradle | 17 +- sdks/java/io/jms/pom.xml | 15 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 208 +- .../java/org/apache/beam/sdk/io/jms/JmsRecord.java | 7 +- .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 57 +- sdks/java/io/kafka/README.md | 17 +- sdks/java/io/kafka/build.gradle | 18 +- sdks/java/io/kafka/pom.xml | 23 +- .../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 97 +- .../CustomTimestampPolicyWithLimitedDelay.java | 101 + .../beam/sdk/io/kafka/KafkaCheckpointMark.java | 35 +- .../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 663 + .../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 1591 +- .../io/kafka/KafkaPublishTimestampFunction.java | 45 + .../org/apache/beam/sdk/io/kafka/KafkaRecord.java | 31 +- .../apache/beam/sdk/io/kafka/KafkaRecordCoder.java | 72 +- .../beam/sdk/io/kafka/KafkaTimestampType.java | 44 + .../beam/sdk/io/kafka/KafkaUnboundedReader.java | 733 + .../beam/sdk/io/kafka/KafkaUnboundedSource.java | 165 + .../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 138 + .../org/apache/beam/sdk/io/kafka/ProducerSpEL.java | 13 +- .../apache/beam/sdk/io/kafka/TimestampPolicy.java | 68 + .../beam/sdk/io/kafka/TimestampPolicyFactory.java | 193 + .../CustomTimestampPolicyWithLimitedDelayTest.java | 119 + .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 384 +- .../beam/sdk/io/kafka/KafkaRecordCoderTest.java | 40 + sdks/java/io/kinesis/build.gradle | 19 +- sdks/java/io/kinesis/pom.xml | 49 +- .../beam/sdk/io/kinesis/AWSClientsProvider.java | 10 +- .../beam/sdk/io/kinesis/BasicKinesisProvider.java | 90 + .../sdk/io/kinesis/DynamicCheckpointGenerator.java | 32 +- .../sdk/io/kinesis/GetKinesisRecordsResult.java | 19 +- .../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 466 +- .../beam/sdk/io/kinesis/KinesisPartitioner.java | 27 + .../apache/beam/sdk/io/kinesis/KinesisReader.java | 13 +- .../sdk/io/kinesis/KinesisReaderCheckpoint.java | 2 - .../apache/beam/sdk/io/kinesis/KinesisRecord.java | 2 - .../beam/sdk/io/kinesis/KinesisRecordCoder.java | 1 - .../io/kinesis/KinesisShardClosedException.java | 28 + .../apache/beam/sdk/io/kinesis/KinesisSource.java | 20 +- .../beam/sdk/io/kinesis/ShardCheckpoint.java | 2 - .../beam/sdk/io/kinesis/ShardReadersPool.java | 173 +- .../beam/sdk/io/kinesis/ShardRecordsIterator.java | 29 +- .../sdk/io/kinesis/SimplifiedKinesisClient.java | 11 +- .../apache/beam/sdk/io/kinesis/StartingPoint.java | 2 - .../sdk/io/kinesis/StartingPointShardsFinder.java | 191 + .../beam/sdk/io/kinesis/AmazonKinesisMock.java | 32 +- .../beam/sdk/io/kinesis/CustomOptionalTest.java | 2 - .../io/kinesis/DynamicCheckpointGeneratorTest.java | 33 +- .../apache/beam/sdk/io/kinesis/KinesisIOIT.java | 128 + .../beam/sdk/io/kinesis/KinesisMockReadTest.java | 4 +- .../beam/sdk/io/kinesis/KinesisMockWriteTest.java | 296 + .../beam/sdk/io/kinesis/KinesisProducerMock.java | 120 + .../io/kinesis/KinesisReaderCheckpointTest.java | 2 - .../beam/sdk/io/kinesis/KinesisReaderIT.java | 121 - .../beam/sdk/io/kinesis/KinesisReaderTest.java | 55 +- .../sdk/io/kinesis/KinesisRecordCoderTest.java | 4 +- .../beam/sdk/io/kinesis/KinesisServiceMock.java | 84 + .../beam/sdk/io/kinesis/RecordFilterTest.java | 2 - .../beam/sdk/io/kinesis/ShardCheckpointTest.java | 2 - .../beam/sdk/io/kinesis/ShardReadersPoolTest.java | 134 +- .../sdk/io/kinesis/ShardRecordsIteratorTest.java | 6 +- .../io/kinesis/SimplifiedKinesisClientTest.java | 36 +- .../io/kinesis/StartingPointShardsFinderTest.java | 262 + sdks/java/io/mongodb/build.gradle | 20 +- sdks/java/io/mongodb/pom.xml | 213 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 4 +- .../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 8 +- .../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 22 +- .../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 147 + .../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 27 +- sdks/java/io/mqtt/build.gradle | 16 +- sdks/java/io/mqtt/pom.xml | 9 +- .../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +- .../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 141 +- sdks/java/io/parquet/build.gradle | 45 + sdks/java/io/parquet/pom.xml | 140 + .../org/apache/beam/sdk/io/parquet/ParquetIO.java | 386 + .../apache/beam/sdk/io/parquet/package-info.java | 22 + .../apache/beam/sdk/io/parquet/ParquetIOTest.java | 136 + sdks/java/io/pom.xml | 6 +- sdks/java/io/redis/build.gradle | 16 +- sdks/java/io/redis/pom.xml | 9 +- .../sdk/io/redis/RedisConnectionConfiguration.java | 4 - .../java/org/apache/beam/sdk/io/redis/RedisIO.java | 2 +- sdks/java/io/solr/build.gradle | 21 +- sdks/java/io/solr/pom.xml | 43 +- .../beam/sdk/io/solr/AuthorizedSolrClient.java | 1 - .../java/org/apache/beam/sdk/io/solr/SolrIO.java | 214 +- .../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 3 +- .../org/apache/beam/sdk/io/solr/SolrIOTest.java | 113 + .../apache/beam/sdk/io/solr/SolrIOTestUtils.java | 36 +- sdks/java/io/tika/build.gradle | 23 +- sdks/java/io/tika/pom.xml | 27 +- sdks/java/io/xml/build.gradle | 19 +- sdks/java/io/xml/pom.xml | 50 +- .../java/org/apache/beam/sdk/io/xml/JAXBCoder.java | 14 +- .../java/org/apache/beam/sdk/io/xml/XmlIO.java | 4 +- .../java/org/apache/beam/sdk/io/xml/XmlSource.java | 3 +- .../org/apache/beam/sdk/io/xml/JAXBCoderTest.java | 22 +- .../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 18 +- .../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 25 +- sdks/java/java8tests/build.gradle | 38 - sdks/java/java8tests/pom.xml | 95 - .../options/PipelineOptionsFactoryJava8Test.java | 92 - .../beam/sdk/transforms/CombineJava8Test.java | 171 - .../beam/sdk/transforms/DistinctJava8Test.java | 97 - .../beam/sdk/transforms/FilterJava8Test.java | 114 - .../sdk/transforms/FlatMapElementsJava8Test.java | 84 - .../beam/sdk/transforms/MapElementsJava8Test.java | 97 - .../beam/sdk/transforms/PartitionJava8Test.java | 74 - .../sdk/transforms/SimpleFunctionJava8Test.java | 69 - .../beam/sdk/transforms/WithKeysJava8Test.java | 73 - .../sdk/transforms/WithTimestampsJava8Test.java | 71 - sdks/java/javadoc/ant.xml | 2 +- sdks/java/javadoc/build.gradle | 110 + sdks/java/javadoc/pom.xml | 73 +- sdks/java/maven-archetypes/examples/build.gradle | 51 +- .../maven-archetypes/examples/generate-sources.sh | 26 +- sdks/java/maven-archetypes/examples/pom.xml | 45 +- .../src/main/resources/archetype-resources/pom.xml | 46 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/build.gradle | 13 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- sdks/java/nexmark/build.gradle | 70 +- sdks/java/nexmark/pom.xml | 66 +- .../java/org/apache/beam/sdk/nexmark/Main.java | 4 +- .../apache/beam/sdk/nexmark/NexmarkLauncher.java | 312 +- .../apache/beam/sdk/nexmark/NexmarkOptions.java | 32 +- .../org/apache/beam/sdk/nexmark/NexmarkSuite.java | 31 +- .../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 17 +- .../org/apache/beam/sdk/nexmark/PubsubHelper.java | 229 + .../org/apache/beam/sdk/nexmark/model/Auction.java | 4 +- .../beam/sdk/nexmark/model/AuctionCount.java | 38 +- .../beam/sdk/nexmark/model/AuctionPrice.java | 26 +- .../org/apache/beam/sdk/nexmark/model/Bid.java | 25 +- .../beam/sdk/nexmark/model/BidsPerSession.java | 2 +- .../beam/sdk/nexmark/model/CategoryPrice.java | 2 +- .../org/apache/beam/sdk/nexmark/model/Event.java | 4 +- .../beam/sdk/nexmark/model/IdNameReserve.java | 2 +- .../beam/sdk/nexmark/model/NameCityStateId.java | 32 +- .../org/apache/beam/sdk/nexmark/model/Person.java | 4 +- .../apache/beam/sdk/nexmark/model/SellerPrice.java | 2 +- .../beam/sdk/nexmark/model/sql/BeamRecordSize.java | 134 - .../apache/beam/sdk/nexmark/model/sql/RowSize.java | 84 + .../beam/sdk/nexmark/model/sql/ToBeamRecord.java | 82 - .../apache/beam/sdk/nexmark/model/sql/ToRow.java | 85 + .../model/sql/adapter/ModelAdaptersMapping.java | 151 +- .../model/sql/adapter/ModelFieldsAdapter.java | 33 +- .../nexmark/model/sql/adapter/package-info.java | 2 +- .../sdk/nexmark/queries/AbstractSimulator.java | 1 - .../beam/sdk/nexmark/queries/AuctionOrBid.java | 29 + .../sdk/nexmark/queries/NexmarkQueryModel.java | 1 + .../beam/sdk/nexmark/queries/Query0Model.java | 1 - .../apache/beam/sdk/nexmark/queries/Query10.java | 1 - .../beam/sdk/nexmark/queries/Query1Model.java | 1 - .../beam/sdk/nexmark/queries/Query2Model.java | 1 - .../apache/beam/sdk/nexmark/queries/Query3.java | 6 +- .../beam/sdk/nexmark/queries/Query3Model.java | 6 +- .../apache/beam/sdk/nexmark/queries/Query4.java | 19 +- .../beam/sdk/nexmark/queries/Query4Model.java | 9 +- .../beam/sdk/nexmark/queries/Query5Model.java | 6 +- .../apache/beam/sdk/nexmark/queries/Query6.java | 11 +- .../beam/sdk/nexmark/queries/Query6Model.java | 35 +- .../beam/sdk/nexmark/queries/Query7Model.java | 1 - .../apache/beam/sdk/nexmark/queries/Query9.java | 9 +- .../beam/sdk/nexmark/queries/Query9Model.java | 1 - .../beam/sdk/nexmark/queries/WinningBids.java | 99 +- .../sdk/nexmark/queries/WinningBidsSimulator.java | 8 +- .../sdk/nexmark/queries/sql/NexmarkSqlQuery.java | 14 +- .../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 49 +- .../beam/sdk/nexmark/queries/sql/SqlQuery1.java | 36 +- .../beam/sdk/nexmark/queries/sql/SqlQuery2.java | 35 +- .../beam/sdk/nexmark/queries/sql/SqlQuery3.java | 162 + .../beam/sdk/nexmark/queries/sql/SqlQuery5.java | 114 + .../beam/sdk/nexmark/queries/sql/SqlQuery7.java | 101 + .../sdk/nexmark/sources/UnboundedEventSource.java | 2 - .../sources/generator/GeneratorCheckpoint.java | 1 - .../nexmark/sources/generator/GeneratorConfig.java | 1 - .../sources/generator/model/StringsGenerator.java | 10 +- .../sdk/nexmark/model/sql/BeamRecordSizeTest.java | 113 - .../beam/sdk/nexmark/model/sql/RowSizeTest.java | 113 + .../sdk/nexmark/model/sql/ToBeamRecordTest.java | 109 - .../beam/sdk/nexmark/model/sql/ToRowTest.java | 110 + .../sql/adapter/ModelAdaptersMappingTest.java | 41 +- .../apache/beam/sdk/nexmark/queries/QueryTest.java | 91 +- .../sdk/nexmark/queries/sql/SqlQuery0Test.java | 9 +- .../sdk/nexmark/queries/sql/SqlQuery1Test.java | 9 +- .../sdk/nexmark/queries/sql/SqlQuery2Test.java | 27 +- .../sdk/nexmark/queries/sql/SqlQuery3Test.java | 125 + .../sdk/nexmark/queries/sql/SqlQuery5Test.java | 103 + .../sdk/nexmark/queries/sql/SqlQuery7Test.java | 89 + .../nexmark/sources/UnboundedEventSourceTest.java | 1 - sdks/java/pom.xml | 18 +- sdks/pom.xml | 2 +- sdks/python/apache_beam/coders/__init__.py | 1 + sdks/python/apache_beam/coders/coder_impl.pxd | 11 +- sdks/python/apache_beam/coders/coder_impl.py | 155 +- sdks/python/apache_beam/coders/coders.py | 60 +- sdks/python/apache_beam/coders/coders_test.py | 16 +- .../apache_beam/coders/coders_test_common.py | 71 +- sdks/python/apache_beam/coders/fast_coders_test.py | 1 + sdks/python/apache_beam/coders/observable.py | 3 + sdks/python/apache_beam/coders/observable_test.py | 1 + sdks/python/apache_beam/coders/slow_coders_test.py | 1 + sdks/python/apache_beam/coders/slow_stream.py | 7 +- .../apache_beam/coders/standard_coders_test.py | 30 +- sdks/python/apache_beam/coders/stream_test.py | 3 + sdks/python/apache_beam/coders/typecoders.py | 15 +- sdks/python/apache_beam/coders/typecoders_test.py | 21 +- .../apache_beam/examples/complete/estimate_pi.py | 2 +- .../examples/complete/game/game_stats.py | 30 +- .../examples/complete/game/hourly_team_score.py | 14 +- .../examples/complete/game/leader_board.py | 22 +- .../complete/juliaset/juliaset/juliaset.py | 2 +- sdks/python/apache_beam/examples/complete/tfidf.py | 8 +- .../examples/cookbook/bigquery_side_input.py | 2 +- .../examples/cookbook/datastore_wordcount.py | 5 +- .../apache_beam/examples/snippets/snippets.py | 395 +- .../apache_beam/examples/snippets/snippets_test.py | 264 +- .../apache_beam/examples/streaming_wordcap.py | 60 - .../apache_beam/examples/streaming_wordcount.py | 92 +- .../examples/streaming_wordcount_debugging.py | 184 + .../examples/streaming_wordcount_it_test.py | 107 + .../apache_beam/examples/windowed_wordcount.py | 8 +- sdks/python/apache_beam/examples/wordcount.py | 8 +- .../apache_beam/examples/wordcount_debugging.py | 4 +- .../python/apache_beam/examples/wordcount_fnapi.py | 4 +- .../apache_beam/examples/wordcount_it_test.py | 2 +- .../apache_beam/examples/wordcount_minimal.py | 4 +- sdks/python/apache_beam/internal/__init__.py | 2 + sdks/python/apache_beam/internal/gcp/__init__.py | 2 + sdks/python/apache_beam/internal/gcp/auth.py | 11 +- sdks/python/apache_beam/internal/gcp/json_value.py | 21 +- .../apache_beam/internal/gcp/json_value_test.py | 8 +- sdks/python/apache_beam/internal/module_test.py | 15 + sdks/python/apache_beam/internal/pickler.py | 9 +- sdks/python/apache_beam/internal/pickler_test.py | 6 + sdks/python/apache_beam/internal/util.py | 10 +- sdks/python/apache_beam/internal/util_test.py | 1 + sdks/python/apache_beam/io/filebasedsink.py | 262 +- sdks/python/apache_beam/io/filebasedsink_test.py | 145 +- sdks/python/apache_beam/io/filebasedsource.py | 38 +- sdks/python/apache_beam/io/filebasedsource_test.py | 10 +- sdks/python/apache_beam/io/filesystem.py | 142 +- sdks/python/apache_beam/io/filesystem_test.py | 162 + sdks/python/apache_beam/io/filesystemio.py | 267 + sdks/python/apache_beam/io/filesystemio_test.py | 191 + sdks/python/apache_beam/io/filesystems.py | 57 +- sdks/python/apache_beam/io/filesystems_test.py | 6 + sdks/python/apache_beam/io/gcp/bigquery.py | 6 +- .../apache_beam/io/gcp/datastore/v1/helper.py | 5 +- sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 82 +- .../apache_beam/io/gcp/gcsfilesystem_test.py | 29 +- sdks/python/apache_beam/io/gcp/gcsio.py | 582 +- sdks/python/apache_beam/io/gcp/gcsio_test.py | 250 +- sdks/python/apache_beam/io/gcp/pubsub.py | 188 +- sdks/python/apache_beam/io/gcp/pubsub_test.py | 354 +- .../apache_beam/io/gcp/tests/pubsub_matcher.py | 112 + .../io/gcp/tests/pubsub_matcher_test.py | 94 + sdks/python/apache_beam/io/hadoopfilesystem.py | 183 +- .../python/apache_beam/io/hadoopfilesystem_test.py | 261 +- .../io/hdfs_integration_test/Dockerfile | 44 + .../io/hdfs_integration_test/docker-compose.yml | 60 + .../hdfs_integration_test/hdfs_integration_test.sh | 42 + .../io/hdfs_integration_test/hdfscli.cfg | 22 + sdks/python/apache_beam/io/iobase.py | 68 +- sdks/python/apache_beam/io/localfilesystem.py | 88 +- sdks/python/apache_beam/io/localfilesystem_test.py | 26 + sdks/python/apache_beam/io/range_trackers.py | 8 +- sdks/python/apache_beam/io/range_trackers_test.py | 5 +- sdks/python/apache_beam/io/textio.py | 4 +- sdks/python/apache_beam/io/tfrecordio.py | 47 +- sdks/python/apache_beam/io/tfrecordio_test.py | 486 +- sdks/python/apache_beam/io/utils.py | 65 + sdks/python/apache_beam/io/vcfio.py | 10 +- sdks/python/apache_beam/metrics/__init__.py | 2 + sdks/python/apache_beam/metrics/cells.py | 190 +- sdks/python/apache_beam/metrics/cells_test.py | 40 +- sdks/python/apache_beam/metrics/execution.pxd | 1 + sdks/python/apache_beam/metrics/execution.py | 73 +- sdks/python/apache_beam/metrics/execution_test.py | 69 +- sdks/python/apache_beam/metrics/metric.py | 39 +- sdks/python/apache_beam/metrics/metric_test.py | 57 +- sdks/python/apache_beam/metrics/metricbase.py | 35 +- sdks/python/apache_beam/options/__init__.py | 1 + .../python/apache_beam/options/pipeline_options.py | 42 +- .../apache_beam/options/pipeline_options_test.py | 2 + .../options/pipeline_options_validator.py | 3 + .../options/pipeline_options_validator_test.py | 3 + sdks/python/apache_beam/options/value_provider.py | 38 +- .../apache_beam/options/value_provider_test.py | 53 + sdks/python/apache_beam/pipeline.py | 200 +- sdks/python/apache_beam/pipeline_test.py | 104 +- sdks/python/apache_beam/portability/common_urns.py | 62 + sdks/python/apache_beam/portability/python_urns.py | 32 + sdks/python/apache_beam/pvalue.py | 74 +- sdks/python/apache_beam/runners/__init__.py | 1 - sdks/python/apache_beam/runners/common.pxd | 4 +- sdks/python/apache_beam/runners/common.py | 143 +- .../runners/dataflow/dataflow_metrics.py | 12 +- .../runners/dataflow/dataflow_metrics_test.py | 4 +- .../runners/dataflow/dataflow_runner.py | 230 +- .../runners/dataflow/dataflow_runner_test.py | 27 +- .../runners/dataflow/internal/apiclient.py | 147 +- .../runners/dataflow/internal/apiclient_test.py | 48 +- .../clients/dataflow/dataflow_v1b3_messages.py | 149 +- .../runners/dataflow/internal/dependency.py | 596 - .../runners/dataflow/internal/dependency_test.py | 443 - .../apache_beam/runners/dataflow/internal/names.py | 31 +- .../runners/dataflow/native_io/iobase_test.py | 3 +- .../runners/dataflow/ptransform_overrides.py | 6 +- .../runners/dataflow/test_dataflow_runner.py | 64 +- .../apache_beam/runners/direct/direct_metrics.py | 18 +- .../apache_beam/runners/direct/direct_runner.py | 394 +- .../runners/direct/direct_runner_test.py | 18 +- .../runners/direct/evaluation_context.py | 160 +- sdks/python/apache_beam/runners/direct/executor.py | 137 +- .../runners/direct/helper_transforms.py | 8 +- .../runners/direct/sdf_direct_runner.py | 26 +- .../runners/direct/sdf_direct_runner_test.py | 35 +- .../runners/direct/transform_evaluator.py | 169 +- sdks/python/apache_beam/runners/direct/util.py | 5 + .../runners/direct/watermark_manager.py | 19 +- .../python/apache_beam/runners/pipeline_context.py | 33 +- .../runners/portability/fn_api_runner.py | 346 +- .../runners/portability/fn_api_runner_test.py | 234 +- .../runners/portability/local_job_service.py | 294 + .../runners/portability/local_job_service_main.py | 45 + .../runners/portability/maptask_executor_runner.py | 472 - .../portability/maptask_executor_runner_test.py | 223 - .../runners/portability/portable_runner.py | 150 + .../runners/portability/portable_runner_test.py | 194 + .../runners/portability/portable_stager.py | 86 + .../runners/portability/portable_stager_test.py | 162 + .../apache_beam/runners/portability/stager.py | 557 + .../apache_beam/runners/portability/stager_test.py | 556 + .../runners/portability/universal_local_runner.py | 440 - .../portability/universal_local_runner_main.py | 45 - .../portability/universal_local_runner_test.py | 114 - sdks/python/apache_beam/runners/runner.py | 43 +- sdks/python/apache_beam/runners/runner_test.py | 24 + sdks/python/apache_beam/runners/sdf_common.py | 21 +- .../apache_beam/runners/worker/bundle_processor.py | 150 +- .../apache_beam/runners/worker/data_plane.py | 33 +- .../apache_beam/runners/worker/data_plane_test.py | 3 +- .../apache_beam/runners/worker/log_handler.py | 5 +- .../apache_beam/runners/worker/opcounters.pxd | 22 + .../apache_beam/runners/worker/opcounters.py | 128 + .../apache_beam/runners/worker/opcounters_test.py | 54 +- .../apache_beam/runners/worker/operation_specs.py | 53 +- .../apache_beam/runners/worker/operations.pxd | 1 + .../apache_beam/runners/worker/operations.py | 249 +- .../apache_beam/runners/worker/sdk_worker.py | 187 +- .../apache_beam/runners/worker/sdk_worker_main.py | 2 +- .../runners/worker/sdk_worker_main_test.py | 2 +- .../apache_beam/runners/worker/sideinputs.py | 46 +- .../apache_beam/runners/worker/sideinputs_test.py | 47 +- .../apache_beam/runners/worker/statesampler.py | 40 +- .../runners/worker/statesampler_fast.pyx | 54 +- .../runners/worker/statesampler_slow.py | 22 +- .../runners/worker/statesampler_test.py | 2 + .../runners/worker/worker_id_interceptor.py | 59 + .../runners/worker/worker_id_interceptor_test.py | 75 + .../apache_beam/testing/data/standard_coders.yaml | 56 +- .../testing/data/trigger_transcripts.yaml | 2 +- .../apache_beam/testing/pipeline_verifiers_test.py | 2 +- sdks/python/apache_beam/testing/test_pipeline.py | 7 +- .../python/apache_beam/testing/test_stream_test.py | 262 +- sdks/python/apache_beam/testing/test_utils.py | 47 + sdks/python/apache_beam/testing/test_utils_test.py | 57 +- sdks/python/apache_beam/testing/util.py | 67 +- sdks/python/apache_beam/testing/util_test.py | 18 +- sdks/python/apache_beam/tools/__init__.py | 21 + .../tools/distribution_counter_microbenchmark.py | 67 + .../apache_beam/tools/map_fn_microbenchmark.py | 66 + .../apache_beam/tools/sideinput_microbenchmark.py | 75 + sdks/python/apache_beam/tools/utils.py | 34 + sdks/python/apache_beam/transforms/__init__.py | 6 + sdks/python/apache_beam/transforms/combiners.py | 46 +- .../apache_beam/transforms/combiners_test.py | 4 +- sdks/python/apache_beam/transforms/core.py | 84 +- sdks/python/apache_beam/transforms/create_test.py | 6 + sdks/python/apache_beam/transforms/cy_combiners.py | 19 + .../cy_dataflow_distribution_counter.pxd | 44 + .../cy_dataflow_distribution_counter.pyx | 135 + .../dataflow_distribution_counter_test.py | 82 + sdks/python/apache_beam/transforms/display.py | 4 +- sdks/python/apache_beam/transforms/display_test.py | 6 +- sdks/python/apache_beam/transforms/ptransform.py | 160 +- .../apache_beam/transforms/ptransform_test.py | 142 +- .../transforms/py_dataflow_distribution_counter.py | 115 + sdks/python/apache_beam/transforms/trigger.py | 145 +- sdks/python/apache_beam/transforms/trigger_test.py | 65 +- sdks/python/apache_beam/transforms/util.py | 133 +- sdks/python/apache_beam/transforms/util_test.py | 6 + sdks/python/apache_beam/transforms/window.py | 22 +- .../transforms/write_ptransform_test.py | 6 +- sdks/python/apache_beam/typehints/decorators.py | 6 +- .../typehints/native_type_compatibility.py | 4 +- sdks/python/apache_beam/typehints/opcodes.py | 6 +- .../apache_beam/typehints/trivial_inference.py | 6 +- .../typehints/trivial_inference_test.py | 6 + sdks/python/apache_beam/typehints/typecheck.py | 86 +- .../apache_beam/typehints/typed_pipeline_test.py | 4 +- sdks/python/apache_beam/typehints/typehints.py | 43 +- .../python/apache_beam/typehints/typehints_test.py | 80 +- sdks/python/apache_beam/utils/annotations.py | 2 +- sdks/python/apache_beam/utils/counters.py | 40 +- sdks/python/apache_beam/utils/profiler.py | 9 +- sdks/python/apache_beam/utils/retry.py | 11 +- sdks/python/apache_beam/utils/timestamp.py | 77 +- sdks/python/apache_beam/utils/timestamp_test.py | 35 +- sdks/python/apache_beam/utils/urns.py | 47 +- sdks/python/apache_beam/utils/windowed_value.pxd | 3 +- sdks/python/apache_beam/utils/windowed_value.py | 133 +- .../apache_beam/utils/windowed_value_test.py | 11 +- sdks/python/apache_beam/version.py | 2 +- sdks/python/build.gradle | 175 +- sdks/python/container/Dockerfile | 8 +- sdks/python/container/boot.go | 1 + sdks/python/container/build.gradle | 22 +- sdks/python/container/pom.xml | 26 +- sdks/python/container/run_validatescontainer.sh | 89 + sdks/python/gen_protos.py | 40 +- sdks/python/generate_pydoc.sh | 192 - sdks/python/pom.xml | 2 +- sdks/python/run_postcommit.sh | 85 - sdks/python/run_pylint.sh | 103 - sdks/python/run_validatesrunner.sh | 71 - sdks/python/scripts/generate_pydoc.sh | 208 + sdks/python/scripts/run_mini_py3lint.sh | 52 + sdks/python/scripts/run_postcommit.sh | 81 + sdks/python/scripts/run_pylint.sh | 112 + sdks/python/scripts/run_pylint_2to3.sh | 101 + sdks/python/scripts/run_tox_cleanup.sh | 41 + sdks/python/setup.cfg | 3 + sdks/python/setup.py | 62 +- sdks/python/tox.ini | 146 +- settings.gradle | 198 +- 2372 files changed, 214639 insertions(+), 43141 deletions(-) -- To stop receiving notification emails like this one, please contact [email protected].
