This is an automated email from the ASF dual-hosted git repository. bhulette pushed a commit to branch website-revamp in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8b22fb4b9aa8607313f1e39374a226bc226a349c Merge: 140a913 31716b0 Author: Brian Hulette <[email protected]> AuthorDate: Mon Dec 7 18:09:14 2020 -0800 Merge remote-tracking branch 'origin/master' into website-revamp .test-infra/jenkins/PythonTestProperties.groovy | 17 +- .../jenkins/job_LoadTests_Combine_Flink_Go.groovy | 136 ++++++++++ .../jenkins/job_LoadTests_GBK_Flink_Go.groovy | 220 ++++++++++++++++ .../jenkins/job_LoadTests_coGBK_Flink_Go.groovy | 182 +++++++++++++ .../job_PostCommit_PortableJar_Flink.groovy | 3 +- .../job_PostCommit_PortableJar_Spark.groovy | 2 +- ...Commit_Python_Chicago_Taxi_Example_Flink.groovy | 8 +- ...ommit_Python_ValidatesContainer_Dataflow.groovy | 9 +- .../dashboards/Post-Commits_status_dashboard.json | 2 +- CHANGES.md | 4 + build.gradle | 13 +- buildSrc/build.gradle | 4 +- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 23 +- dev-support/docker/Dockerfile | 117 +++++++++ dev-support/docker/bashcolors.sh | 91 +++++++ dev-support/docker/beam_env_checks.sh | 121 +++++++++ model/pipeline/src/main/proto/metrics.proto | 13 + .../DefaultExpansionServiceClientFactory.java | 7 +- .../core/construction/ExpansionServiceClient.java | 2 +- .../ExpansionServiceClientFactory.java | 2 +- .../beam/runners/core/construction/External.java | 2 +- .../core/construction/PTransformTranslation.java | 1 + .../beam/runners/core/SideInputHandlerTest.java | 118 +++++---- runners/direct-java/build.gradle | 22 +- runners/flink/job-server/flink_job_server.gradle | 66 ----- .../worker/windmill/GrpcWindmillServer.java | 14 +- .../test_flink_uber_jar.sh | 0 runners/portability/test_pipeline_jar.sh | 3 +- runners/spark/job-server/build.gradle | 32 --- sdks/go/test/load/build.gradle | 3 + sdks/go/test/load/cogbk/cogbk.go | 106 ++++++++ sdks/go/test/load/combine/combine.go | 87 +++++++ sdks/go/test/load/group_by_key/group_by_key.go | 89 +++++++ sdks/java/build-tools/beam-linkage-check.sh | 80 +++--- .../apache/beam/sdk/annotations/Experimental.java | 7 - .../main/java/org/apache/beam/sdk/io/AvroIO.java | 7 +- .../main/java/org/apache/beam/sdk/io/FileIO.java | 5 +- .../main/java/org/apache/beam/sdk/io/TextIO.java | 5 +- .../java/org/apache/beam/sdk/state/Timers.java | 2 +- .../apache/beam/sdk/transforms/Deduplicate.java | 3 - .../java/org/apache/beam/sdk/transforms/DoFn.java | 13 - .../beam/sdk/transforms/PeriodicSequence.java | 3 - .../java/org/apache/beam/sdk/transforms/Watch.java | 3 - .../splittabledofn/ByteKeyRangeTracker.java | 3 - .../splittabledofn/GrowableOffsetRangeTracker.java | 3 - .../splittabledofn/HasDefaultTracker.java | 4 - .../HasDefaultWatermarkEstimator.java | 3 - .../splittabledofn/ManualWatermarkEstimator.java | 3 - .../splittabledofn/OffsetRangeTracker.java | 3 - .../splittabledofn/RestrictionTracker.java | 3 - .../sdk/transforms/splittabledofn/SplitResult.java | 3 - .../TimestampObservingWatermarkEstimator.java | 3 - .../splittabledofn/WatermarkEstimator.java | 3 - .../splittabledofn/WatermarkEstimators.java | 3 - .../transforms/splittabledofn/package-info.java | 3 - sdks/java/extensions/ml/build.gradle | 33 ++- sdks/java/extensions/sql/build.gradle | 1 + .../sql/meta/provider/bigtable/BigtableTable.java | 211 +++++++++++++++ .../provider/bigtable/BigtableTableProvider.java | 85 ++++++ .../sql/meta/provider/bigtable/package-info.java | 9 +- .../BigtableTableCreationFailuresTest.java | 145 +++++++++++ .../provider/bigtable/BigtableTableFlatTest.java | 163 ++++++++++++ .../meta/provider/bigtable/BigtableTableTest.java | 94 +++++++ .../bigtable/BigtableTableWithRowsTest.java | 158 +++++++++++ .../apache/beam/fn/harness/ToStringFnRunner.java | 66 +++++ .../beam/fn/harness/ToStringFnRunnerTest.java | 72 +++++ sdks/java/io/amazon-web-services2/build.gradle | 2 +- .../beam/sdk/io/aws2/options/AwsOptions.java | 4 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 5 + .../io/gcp/bigtable/BeamRowToBigtableMutation.java | 123 +++++++++ .../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 15 ++ .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 22 ++ .../sdk/io/gcp/bigtable/BigtableRowToBeamRow.java | 176 +++++++++++++ .../io/gcp/bigtable/BigtableRowToBeamRowFlat.java | 108 ++++++++ .../io/gcp/bigtable/BigtableRowToBeamRowFn.java | 46 ++++ .../beam/sdk/io/gcp/bigtable/CellValueParser.java | 106 ++++++++ .../apache/beam/sdk/io/gcp/bigtable/RowUtils.java | 13 +- .../apache/beam/sdk/io/gcp/healthcare/DicomIO.java | 199 ++++++++++++++ .../sdk/io/gcp/healthcare/HealthcareApiClient.java | 12 + .../io/gcp/healthcare/HttpHealthcareApiClient.java | 79 ++++++ .../beam/sdk/io/gcp/healthcare/WebPathParser.java | 63 +++++ .../io/gcp/testing/BigtableEmulatorWrapper.java | 75 ++++++ .../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 154 +++++++++++ .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 9 + .../bigtable/BeamRowToBigtableMutationTest.java | 89 +++++++ .../gcp/bigtable/BigtableRowToBeamRowFlatTest.java | 65 +++++ .../io/gcp/bigtable/BigtableRowToBeamRowTest.java | 93 +++++++ .../sdk/io/gcp/bigtable/CellValueParserTest.java | 277 ++++++++++++++++++++ .../apache/beam/sdk/io/gcp/bigtable/TestUtils.java | 109 ++++++++ .../beam/sdk/io/gcp/healthcare/DicomIOReadIT.java | 93 +++++++ .../beam/sdk/io/gcp/healthcare/DicomIOTest.java | 60 +++++ .../sdk/io/gcp/healthcare/WebPathParserTest.java | 48 ++++ .../src/test/resources/DICOM/testDicomFile.dcm | Bin 0 -> 10366 bytes .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 1 - sdks/java/io/kafka/build.gradle | 90 +++++-- .../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 69 ++++- sdks/python/apache_beam/coders/coders.py | 16 ++ .../apache_beam/coders/coders_test_common.py | 17 ++ sdks/python/apache_beam/dataframe/io.py | 143 +++++++++- sdks/python/apache_beam/dataframe/io_test.py | 41 +++ .../apache_beam/examples/kafkataxi/README.md | 4 +- .../apache_beam/examples/snippets/snippets.py | 4 + sdks/python/apache_beam/io/gcp/bigquery.py | 273 +++++++++++++------ .../apache_beam/io/gcp/bigquery_read_internal.py | 289 +++++++++++++++++++++ .../apache_beam/io/gcp/bigquery_read_it_test.py | 103 ++++++++ sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 26 +- sdks/python/apache_beam/io/gcp/bigtableio.py | 7 +- sdks/python/apache_beam/io/gcp/pubsub.py | 114 ++++++++ sdks/python/apache_beam/io/gcp/pubsub_test.py | 123 +++++++++ sdks/python/apache_beam/io/iobase.py | 20 +- sdks/python/apache_beam/io/restriction_trackers.py | 17 ++ sdks/python/apache_beam/metrics/cells.pxd | 2 + sdks/python/apache_beam/metrics/cells.py | 15 +- sdks/python/apache_beam/metrics/cells_test.py | 25 ++ .../python/apache_beam/options/pipeline_options.py | 6 + sdks/python/apache_beam/options/value_provider.py | 71 ++++- .../apache_beam/options/value_provider_test.py | 40 +++ sdks/python/apache_beam/runners/common.py | 5 +- .../Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png | Bin 700548 -> 703677 bytes .../runners/portability/flink_runner_test.py | 2 +- .../portability/fn_api_runner/fn_runner_test.py | 115 +++++++- .../portability/fn_api_runner/translations.py | 8 +- .../portability/fn_api_runner/translations_test.py | 31 +++ .../portability/fn_api_runner/worker_handlers.py | 2 +- .../apache_beam/runners/portability/job_server.py | 3 +- .../runners/portability/sdk_container_builder.py | 30 ++- .../runners/portability/spark_runner_test.py | 6 +- .../apache_beam/runners/worker/bundle_processor.py | 17 ++ .../apache_beam/runners/worker/sdk_worker_test.py | 2 + sdks/python/apache_beam/transforms/core.py | 32 ++- .../apache_beam/typehints/sharded_key_type.py | 75 ++++++ .../apache_beam/typehints/sharded_key_type_test.py | 80 ++++++ sdks/python/apache_beam/utils/shared.py | 2 +- sdks/python/build-requirements.txt | 3 + sdks/python/setup.py | 2 +- sdks/python/test-suites/portable/common.gradle | 69 ++++- start-build-env.sh | 132 ++++++++++ .../www/site/content/en/community/powered-by.md | 2 + website/www/site/content/en/contribute/_index.md | 16 +- .../dsls/sql/extensions/create-external-table.md | 132 ++++++++++ .../en/documentation/io/developing-io-java.md | 3 + .../en/documentation/io/developing-io-overview.md | 80 +++--- .../en/documentation/io/developing-io-python.md | 3 + .../content/en/documentation/programming-guide.md | 2 +- .../en/documentation/sdks/python-dependencies.md | 12 +- .../site/content/en/get-started/quickstart-py.md | 4 +- .../content/en/get-started/wordcount-example.md | 2 +- 148 files changed, 6607 insertions(+), 549 deletions(-)
