This is an automated email from the ASF dual-hosted git repository. bhulette pushed a commit to branch website-revamp in repository https://gitbox.apache.org/repos/asf/beam.git
commit 0850259add2661e5d169dd939b18c270b5ffefcf Merge: 1f790fc d7655c1 Author: Brian Hulette <[email protected]> AuthorDate: Tue Nov 24 16:28:51 2020 -0800 Merge remote-tracking branch 'origin/master' into website-revamp .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/workflows/build_wheels.yml | 8 +- .github/workflows/java_tests.yml | 2 +- .github/workflows/python_tests.yml | 2 +- .test-infra/jenkins/CommonTestProperties.groovy | 12 +- ...Properties.groovy => JavaTestProperties.groovy} | 4 +- .test-infra/jenkins/LoadTestsBuilder.groovy | 15 +- .test-infra/jenkins/PythonTestProperties.groovy | 10 + .test-infra/jenkins/README.md | 6 +- .../jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 150 +++++ ...it_CrossLanguageValidatesRunner_Dataflow.groovy | 38 +- ...mmit_CrossLanguageValidatesRunner_Direct.groovy | 23 +- ...ommit_CrossLanguageValidatesRunner_Flink.groovy | 23 +- ...ommit_CrossLanguageValidatesRunner_Spark.groovy | 23 +- ...tCommit_Java_Examples_Dataflow_V2_Java11.groovy | 50 ++ ..._PreCommit_Java_Examples_Dataflow_Java11.groovy | 1 - .../jenkins/job_Publish_Docker_Snapshots.groovy | 8 +- .../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 55 ++ CHANGES.md | 9 +- README.md | 4 +- build.gradle | 28 +- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 20 +- .../apache/beam/gradle/GrpcVendoring_1_26_0.groovy | 2 +- learning/katas/go/README.md | 29 + .../src/main/scripts/build_release_candidate.sh | 6 +- release/src/main/scripts/publish_docker_images.sh | 2 +- .../core/construction/PTransformMatchers.java | 2 +- runners/flink/job-server/flink_job_server.gradle | 5 +- runners/google-cloud-dataflow-java/build.gradle | 78 ++- .../dataflow/DataflowPipelineTranslator.java | 14 +- .../beam/runners/dataflow/DataflowRunner.java | 29 +- .../runners/dataflow/GroupIntoBatchesOverride.java | 132 ++++- .../beam/runners/dataflow/util/PropertyNames.java | 1 + .../dataflow/DataflowPipelineTranslatorTest.java | 67 ++- .../beam/runners/dataflow/DataflowRunnerTest.java | 64 ++- .../dataflow/worker/WindmillTimerInternals.java | 13 +- .../dataflow/worker/WorkItemStatusClient.java | 2 +- .../worker/WindmillTimerInternalsTest.java | 4 +- runners/portability/java/build.gradle | 9 +- runners/spark/build.gradle | 66 ++- sdks/go/pkg/beam/core/metrics/metrics.go | 118 ++++ .../core/runtime/graphx/schema/logicaltypes.go | 121 ++++ .../pkg/beam/core/runtime/graphx/schema/schema.go | 115 ++-- .../beam/core/runtime/graphx/schema/schema_test.go | 55 +- sdks/go/pkg/beam/core/runtime/graphx/translate.go | 44 +- sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 210 +++++++ sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go | 218 +++++++ .../go/pkg/beam/core/runtime/harness/monitoring.go | 187 +----- .../beam/core/runtime/harness/monitoring_test.go | 33 +- sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go | 172 ++++++ .../beam/core/runtime/metricsx/metricsx_test.go | 166 ++++++ sdks/go/pkg/beam/core/runtime/metricsx/urns.go | 170 ++++++ sdks/go/pkg/beam/core/runtime/options.go | 1 + sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 69 ++- sdks/go/pkg/beam/core/runtime/xlangx/namespace.go | 2 +- .../pkg/beam/core/runtime/xlangx/namespace_test.go | 348 +++++------ sdks/go/pkg/beam/core/runtime/xlangx/translate.go | 228 -------- sdks/go/pkg/beam/doc_test.go | 2 +- sdks/go/pkg/beam/io/textio/sdf_test.go | 2 +- sdks/go/pkg/beam/options/jobopts/options.go | 3 + sdks/go/pkg/beam/pipeline.go | 6 + sdks/go/pkg/beam/runner.go | 6 +- sdks/go/pkg/beam/runners/dataflow/dataflow.go | 25 +- sdks/go/pkg/beam/runners/direct/direct.go | 17 +- sdks/go/pkg/beam/runners/dot/dot.go | 10 +- sdks/go/pkg/beam/runners/flink/flink.go | 3 +- sdks/go/pkg/beam/runners/session/session.go | 6 +- sdks/go/pkg/beam/runners/spark/spark.go | 3 +- .../beam/runners/universal/runnerlib/execute.go | 48 +- .../go/pkg/beam/runners/universal/runnerlib/job.go | 3 + sdks/go/pkg/beam/runners/universal/universal.go | 28 +- sdks/go/pkg/beam/runners/vet/vet.go | 8 +- sdks/go/pkg/beam/testing/ptest/ptest.go | 3 +- sdks/go/pkg/beam/x/beamx/run.go | 8 + sdks/go/pkg/beam/xlang.go | 59 +- sdks/go/test/load/build.gradle | 66 +++ sdks/go/test/load/pardo/pardo.go | 112 ++++ sdks/go/test/load/util.go | 227 ++++++++ sdks/java/container/build.gradle | 63 +- sdks/java/container/common.gradle | 104 ++++ .../java/container/java11/build.gradle | 11 +- .../java/container/java8/build.gradle | 11 +- .../org/apache/beam/sdk/io/CountingSource.java | 3 +- .../sdk/schemas/FieldValueTypeInformation.java | 32 +- .../apache/beam/sdk/schemas/JavaBeanSchema.java | 27 +- .../sdk/schemas/annotations/SchemaCaseFormat.java | 73 +++ .../sdk/schemas/annotations/SchemaFieldName.java | 11 +- .../beam/sdk/schemas/utils/JavaBeanUtils.java | 39 +- .../apache/beam/sdk/schemas/utils/POJOUtils.java | 5 +- .../apache/beam/sdk/testing/UsesTestStream.java | 2 +- .../java/org/apache/beam/sdk/transforms/DoFn.java | 4 +- .../beam/sdk/transforms/GroupIntoBatches.java | 70 ++- .../sdk/transforms/reflect/DoFnSignatures.java | 77 +-- .../splittabledofn/RestrictionTracker.java | 23 +- .../beam/sdk/util/common/ReflectHelpers.java | 60 ++ .../beam/sdk/schemas/AutoValueSchemaTest.java | 57 ++ .../beam/sdk/schemas/JavaBeanSchemaTest.java | 73 +++ .../beam/sdk/schemas/JavaFieldSchemaTest.java | 55 ++ .../beam/sdk/schemas/utils/SchemaTestUtils.java | 22 + .../beam/sdk/schemas/utils/TestJavaBeans.java | 129 +++++ .../apache/beam/sdk/schemas/utils/TestPOJOs.java | 56 ++ .../beam/sdk/transforms/GroupIntoBatchesTest.java | 82 +++ sdks/java/extensions/sorter/build.gradle | 46 +- sdks/java/extensions/sql/build.gradle | 44 +- .../impl/transform/BeamBuiltinAggregations.java | 143 ++--- .../provider/bigquery/BeamBigQuerySqlDialect.java | 11 + .../extensions/sql/BeamSqlDslAggregationTest.java | 263 ++++----- .../zetasql/SupportedZetaSqlBuiltinFunctions.java | 4 +- .../sql/zetasql/ZetaSQLQueryPlanner.java | 8 + .../translation/ArrayScanColumnRefToUncollect.java | 4 +- .../ArrayScanLiteralToUncollectConverter.java | 4 +- .../translation/ArrayScanToJoinConverter.java | 5 +- .../translation/SqlOperatorMappingTable.java | 1 + .../zetasql/unnest/BeamZetaSqlUncollectRel.java | 120 ++++ .../zetasql/unnest/BeamZetaSqlUncollectRule.java | 54 ++ .../sql/zetasql/unnest/BeamZetaSqlUnnestRel.java | 164 ++++++ .../sql/zetasql/unnest/BeamZetaSqlUnnestRule.java | 120 ++++ .../sql/zetasql/unnest/ZetaSqlUnnest.java | 140 +++++ .../sql/zetasql/unnest/package-info.java | 8 +- .../beam/sdk/extensions/sql/zetasql/TestInput.java | 66 ++- .../sql/zetasql/ZetaSqlDialectSpecTest.java | 137 ++++- .../extensions/sql/zetasql/ZetaSqlTestBase.java | 8 +- .../fn/harness/logging/BeamFnLoggingClient.java | 50 +- .../harness/logging/BeamFnLoggingClientTest.java | 50 +- .../expansion-service/build.gradle | 37 ++ .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 22 +- .../beam/sdk/io/gcp/spanner/MutationUtils.java | 320 +++++++++++ .../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 6 + .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 8 + .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 112 +++- .../io/gcp/spanner/SpannerTransformRegistrar.java | 374 ++++++++++++ .../beam/sdk/io/gcp/spanner/StructUtils.java | 387 +++++++++++++ .../beam/sdk/io/gcp/spanner/MutationUtilsTest.java | 285 +++++++++ .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 5 +- .../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 258 +++++++++ sdks/java/io/hadoop-common/build.gradle | 50 +- sdks/java/io/hadoop-file-system/build.gradle | 57 +- sdks/java/io/hadoop-format/build.gradle | 104 +++- .../io/hadoop/format/HadoopFormatIOElasticIT.java | 20 +- .../sdk/io/hadoop/format/HadoopFormatIOIT.java | 58 +- .../hadoop/format/HadoopFormatIOTestOptions.java | 6 + sdks/java/io/hbase/build.gradle | 2 +- sdks/java/io/hcatalog/build.gradle | 42 +- sdks/java/io/parquet/build.gradle | 43 +- sdks/java/maven-archetypes/examples/build.gradle | 1 + sdks/java/maven-archetypes/examples/sample.txt | 102 ++++ .../src/main/resources/archetype-resources/pom.xml | 8 +- .../beam/sdk/nexmark/NexmarkConfiguration.java | 2 +- .../apache/beam/sdk/nexmark/NexmarkLauncher.java | 4 +- .../apache/beam/sdk/nexmark/NexmarkOptions.java | 30 +- .../sdk/nexmark/queries/SessionSideInputJoin.java | 2 +- sdks/python/apache_beam/dataframe/expressions.py | 92 ++- sdks/python/apache_beam/dataframe/frame_base.py | 8 +- sdks/python/apache_beam/dataframe/frames.py | 65 ++- sdks/python/apache_beam/dataframe/frames_test.py | 91 +-- sdks/python/apache_beam/dataframe/io.py | 1 + .../apache_beam/dataframe/pandas_doctests_test.py | 3 +- sdks/python/apache_beam/dataframe/partitionings.py | 26 + sdks/python/apache_beam/dataframe/transforms.py | 62 +- .../apache_beam/dataframe/transforms_test.py | 17 +- .../apache_beam/examples/kafkataxi/README.md | 6 +- sdks/python/apache_beam/internal/metrics/metric.py | 90 +++ .../apache_beam/internal/metrics/metric_test.py | 41 ++ sdks/python/apache_beam/io/gcp/bigquery_tools.py | 36 +- .../gcp/resource_identifiers.py} | 34 +- sdks/python/apache_beam/io/gcp/spanner.py | 635 +++++++++++++++++++++ .../io/gcp/tests/xlang_spannerio_it_test.py | 339 +++++++++++ sdks/python/apache_beam/io/iobase.py | 12 +- sdks/python/apache_beam/io/mongodbio.py | 166 ++++-- sdks/python/apache_beam/io/mongodbio_it_test.py | 94 ++- sdks/python/apache_beam/io/mongodbio_test.py | 235 +++++++- sdks/python/apache_beam/io/parquetio.py | 8 + sdks/python/apache_beam/io/parquetio_test.py | 4 + sdks/python/apache_beam/metrics/cells.py | 22 +- sdks/python/apache_beam/metrics/execution.pxd | 4 +- sdks/python/apache_beam/metrics/execution.py | 71 ++- sdks/python/apache_beam/metrics/metric.py | 18 +- sdks/python/apache_beam/metrics/metric_test.py | 16 + sdks/python/apache_beam/metrics/metricbase.py | 46 +- .../python/apache_beam/metrics/monitoring_infos.py | 21 +- .../apache_beam/metrics/monitoring_infos_test.py | 35 ++ sdks/python/apache_beam/pipeline.py | 12 +- .../apache_beam/runners/dataflow/internal/names.py | 4 +- .../runners/interactive/background_caching_job.py | 11 +- .../interactive/background_caching_job_test.py | 42 +- .../runners/interactive/interactive_environment.py | 49 +- .../runners/interactive/interactive_runner.py | 4 +- .../runners/interactive/interactive_runner_test.py | 3 - .../runners/interactive/pipeline_fragment.py | 9 +- .../runners/interactive/pipeline_instrument.py | 29 +- .../runners/interactive/recording_manager.py | 20 +- .../runners/interactive/recording_manager_test.py | 61 +- .../runners/interactive/user_pipeline_tracker.py | 150 +++++ .../interactive/user_pipeline_tracker_test.py | 209 +++++++ .../runners/portability/flink_runner_test.py | 3 +- .../portability/fn_api_runner/fn_runner_test.py | 81 ++- .../portability/fn_api_runner/translations.py | 29 +- .../portability/fn_api_runner/translations_test.py | 39 ++ .../apache_beam/runners/worker/sdk_worker.py | 38 +- .../apache_beam/runners/worker/sdk_worker_test.py | 60 ++ sdks/python/apache_beam/transforms/core.py | 28 +- sdks/python/apache_beam/transforms/util.py | 49 +- sdks/python/apache_beam/transforms/util_test.py | 17 + sdks/python/apache_beam/utils/sharded_key.py | 2 +- sdks/python/apache_beam/utils/shared.py | 24 +- sdks/python/setup.py | 4 +- sdks/python/test-suites/portable/common.gradle | 6 +- sdks/python/test-suites/tox/common.gradle | 3 +- sdks/python/test-suites/tox/py38/build.gradle | 7 + sdks/python/tox.ini | 21 + settings.gradle | 4 + .../content/en/documentation/programming-guide.md | 365 +++++++++++- .../en/documentation/runtime/environments.md | 18 +- .../site/content/en/get-started/quickstart-java.md | 20 +- .../partials/section-menu/en/documentation.html | 10 + .../images/multi-language-pipelines-diagram.svg | 20 + 216 files changed, 10389 insertions(+), 2166 deletions(-)
