This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a change to branch website-revamp
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 7d0d220 Merge remote-tracking branch 'origin/master' into
website-revamp
add fe109d3 Revert "Update Dataflow worker harness container image
version."
add e63d333 Merge pull request #13727 from
tysonjh/revert-13699-worker-jvm-bump
add 04cd839 [BEAM-10556] Remove rawtypes suppression from
WindmillStateCache
add 1317a10 Merge #13742: [BEAM-10556] Remove rawtypes suppression from
WindmillStateCache
add c4be62c Do not set empty specs in external transforms
add 0220ef9 Merge pull request #13741: [BEAM-11614] Do not set empty
specs in external transforms
add c6b79e3 [BEAM-11614] Clarify Beam model for zero subtransform case
(#13744)
add 4451c03 [BEAM-11531] Remove transform implementation (#13705)
add cc0b930 [BEAM-11531] Skip some DataFrame tests that are broken in
pandas 1.2.0 (#13704)
add e2e777a [BEAM-11357] Add annotations to PTransforms and enable them
in Go SDK (#13434)
add 23d404d [BEAM-11569] Enabling wheels to be built using Beam-based
submodules workflow (#13736)
add 6432274 [BEAM-11624] Add hash functions in beam sql (#13733)
add 21ffad3 [BEAM-11533] PubSub support Beam DATETIME (SQL TIMESTAMP)
type (#13731)
add 39ee3ef [BEAM-8202] Support ParquetTable Writer
add 55e106f [BEAM-8202] Make Avro SchemaIOProvider (and tests) consistent
with Parquet
add 17fbe8b Merge pull request #13714: [BEAM-8202] Support ParquetTable
Writer
add 1dd1513 Add gradle-command-action as submodule
add 17c7162 Merge pull request #13753 from
TobKed/gradle-command-action-as-submodule
add 8ea198a Enabled strict dependency
add ba7702a removed * depdendency
add 8d7eb82 enabled strict dependency check to false
add 7f8c721 Fixed checker-qual issue
add 7d172ad removed * dep
add f39bc2d Fixed requested changes
add 54902d8 autovalue_version variable added
add a68b9e7 resolved merge conflicts
add 52f831a Alphabetize dependency lists.
add f371a26 Merge pull request #13690 from
sonam-vend/feature/strict-dependency
add c57d71f Make DataCatalogTableProvider.createDataCatalogClient public
(#13752)
add 6675586 [BEAM-11637] Use accumulators properly in BitAnd. (#13745)
add 0f36a72 [BEAM-11622]Change the input type of FHIR Search to
KV<String, Map<String, Object>> (#13726)
add cff7fc6 [BEAM-11497] Avoid zerodivision error in custom BQ source
add 4bdafec Merge pull request #13734 from [BEAM-11497] Avoid
zerodivision error in custom BQ source
add c9004b2 feat: Add partition increase handling to PubsubLiteIO
add b109e21 fix: Ensure manually set partitions are not ignored.
add 3ca4360 fix: Format
add 1db9ec3 fix: Format
add 49c5ed0 fix: Remove guava usage.
add 6a70085 fix: docs
add 343a847 Merge pull request #13567 from [BEAM-10114] Add partition
increase handling to PubsubLiteIO
add aab4353 Fix translate.go to prevent panic (#13720)
add b50e951 [BEAM-11533] Support DATE, TIME, DATETIME in PubSub (#13751)
add 6763c21 [BEAM-11594] Upgrade gradle to version 6.8
add d0b6fdf Merge pull request #13716: [BEAM-11594] Upgrade gradle to
version 6.8
add d1de1ab [BEAM-11538] Add a Deque Encoder
add f437145 Merge pull request #13635: [BEAM-11538] Add a Deque Encoder
add f5c87dc [BEAM-11593] Move SparkStructuredStreamingRunnerRegistrar to
its own package
add 8054866 Merge pull request #13715: [BEAM-11593] Move
SparkStructuredStreamingRunnerRegistrar to its own package
add e748846 Fix typo in universal.go (#13766)
add 334020f [BEAM-9615] Add Schema Logical Type Provider support (#13760)
add 0f71eb2 [BEAM-11032] Use metric for Java BigQuery streaming insert
API latency logging
add 0f574ef fix checkstyle error, rename equalsMetricName
add cf77f03 Merge pull request #13491 from ihji/BEAM-11032
add 5a360a8 BEAM-10095: Add Runner and SDK links to Beam overview page
add cdc7226 Merge pull request #13774 from pcoet/BEAM-10095
add 9f4ed60 [BEAM-10961] Enabled strict dependency on Spark (#13668)
add 4f32816 Enabled strict dependency on Local Java
add b538b59 set enableStrictDependencies: true
add cd1f664 Merge pull request #13665 from sonam-vend/local-java-runner
add 7450ca3 Remove redundant & detatched package comment
add e7d8fdb Merge pull request #13772: Remove redundant & detatched
dataflowlib package comment
add 5cd3130 [BEAM-9615] Disable schema registration.
add e6fa3bd Merge pull request #13777: [BEAM-9615] Disable schema
registration.
add 53f2757 Fix broken link to signature for 2.21.0
add ff503e8 Merge pull request #13768 from pcoet/patch-2
add 2781342 [BEAM-11461] update tox and simplify tox.ini (#13692)
add 35a49d1 Kafka To Pub/Sub Template Blogpost (#13365)
add 8d6fd6a [BEAM-10961] Enabled strict dependency on Core Java runner
(#13654)
add c27b8d6 [BEAM-10961] Enabled strict dependency on Twister2 (#13669)
add b131cc88 [BEAM-10961] Enabled strict dependencies on
sdks-fn-execution and sdks-harness (#13687)
add 6628a27 Enabled strict dependency on core-construction runnber module
add 51a495f set enableStrictDependencies: true
add eed16a5 Merge pull request #13650 from
sonam-vend/runner-core-construction
add c5e6cc5 Enabled strict dependency on Extensions Java runner
add a9c6cb6 set enableStrictDependencies: true
add 0bfdf6c Merge pull request #13657 from
sonam-vend/runner-extension-java
add d1c8c24 [BEAM-10961] Enabled strict dependencies on java-job-service
(#13662)
add 7e87324 [BEAM-10961] Enabled Strict dependency on Fn-execution Runner
(#13661)
add a724602 [BEAM-11659]: Add DlqProvider, a way of handling failures in
a generic way. (#13773)
add e8eed0b [BEAM-11325] ReadFromKafkaDoFn should stop reading when
topic/partition is removed or marked as stopped.
add d2e1f69 Merge pull request #13710 from [BEAM-11325] ReadFromKafkaDoFn
should stop reading when topic/partition is removed or marked as stopped
add 61ea255 [BEAM-11092][Go SDK] Ack Harness monitoring requests (#13788)
add a9e31ba [BEAM-10961] Enabled strict dependencies on samza (#13667)
add 0f6af9b [BEAM-11200] Update Go SDK version on release. (#13789)
add 014c76c [BEAM-10890] Log error counts to debug BigQuery streaming
insert requests for Java SDK
add 171ba9d add API_REQUEST_LATENCIES, match by MetricName
add b44d928 fix encoding error in windows
add cd13f6e Merge pull request #12852 from ihji/BEAM-10890
add e8fcb74 Remove boilerplate notes from 2.27.0
add 24179c3 Merge pull request #13790 from Remove boilerplate notes from
2.27.0
add aa57b51 [BEAM-10402] Add nullability annotations to SplitResult.of
add 8194761 Merge pull request #13769 from [BEAM-10402] Add nullability
annotations to SplitResult.of
add 04ae01c Expose commit_offset_in_finalize and timestamp_policy to
ReadFromKafka.
add 8b759d1 Merge pull request #13779 from [BEAM-11677] Expose
commit_offset_in_finalize and timestamp_policy to ReadFromKafka
add 262273c [BEAM-10961] Enabled strict Dependency on Portability (#13666)
add 4141550 [BEAM-10961] Enabled strict dependencies on example module
(#13649)
add 5d58250 [BEAM-10961] enabled strict dependency on sdks-core and
sdks-expansion-service (#13685)
add cf402a1 [BEAM-10961] Enabled strict dependency on SDKS-Testing
(#13689)
add 0460daa [BEAM-10961] Enabled strict dependency on Jet Runner (#13663)
add da2f431 [BEAM-11570] Disable classloader check for [auto] mode.
(#13722)
add 52dcc36 [BEAM-10961] Enabled strict dependency on Direct Java runner
(#13656)
add 9e0920c [BEAM-11542] Add support for projecting GroupBy objects
(#13637)
add d7bee01 strict dependency for some sdks/java/io modules - trivial
changes (#13794)
add 9bc7820 [BEAM-9541] Jenkins tests use latest Flink version.
add 704bae9 Write latest Flink version in CommonTestProperties.
add 809a0b7 Merge pull request #13675 from ibzib/flink-jenkins
add 4861df4 Allow extension of SdkContainerImageBuilder (#13756)
add 62c304d [BEAM-11411] [BEAM-11410] Kafka to pub sub E2E test (#13636)
add a35d31f re-remove finalizedBy cleanup for docker container test
add 0dd6980 Merge pull request #13796 from [BEAM-11678] Fix Java
DataflowRunner test container cleanup
add a77818a feat: Allow Pub/Sub Lite Sink to support increasing partitions
add aa98bcf Merge pull request #13787 from [BEAM-10114] Allow Pub/Sub
Lite Sink to support increasing partitions
add c89799a Merge pull request #13786 from [BEAM-11686] Add
FhirSearchParameter class as input for FhirIO.Search.
add cec7f10 [BEAM-11665] Publish Beam GCP BOM (#13737)
add 5492d32 Merge pull request #1 from apache/master
add 2a4caf6 sync libraries-bom version
add 9bec672 Merge #13732: [BEAM-9444] sync Google libraries-bom version
add ae60a7b [BEAM-11581] Start ExecutionStateSampler in Java SDK harness
before dispatch loop.
add 390fb7e Merge pull request #13695 from [BEAM-11581] Start
ExecutionStateSampler in Java SDK harness before dispatch loop
add e1b42bf Merge pull request #12647 from [BEAM-10378] Deserializing
Azure Credentials
add 3d1f25a [BEAM-10112] Add more state and timer python examples to
website
add 89a5e3d Address comments
add 3a2a351 Render py
add 29b9542 Merge pull request #13591 from [BEAM-10112] Add more state
and timer python examples to website
add efc35c5 Revert "Merge pull request #12647 from [BEAM-10378]
Deserializing Azure Credentials"
add 15e2ad8 Merge pull request #13807 from pabloem/fix-azure
add efa5d80 Changing random to seconds since epoch to prevent BQ job name
colliss… (#13749)
add 3ddf941 Add capabilities for the
HarnessMonitoringInfosRequest/Response and set it on the python SDK environment
add 5ef873f Merge pull request #13801 from Add capabilities for the
HarnessMonitoringInfosRequest/Response
add dde38b6 Merge pull request #13792 from Metric updates for FhirIO: Fix
metrics + namings to be consistent.
add 3516a39 [BEAM-9369] bump mockito-core to 3.7.7
add 17e833e Merge pull request #13803: [BEAM-9369] bump mockito-core to
3.7.7
add 4a0f056 BEAM-4986: Bump to Apache Parquet 1.11.1
add 67989ca Merge pull request #13806: BEAM-4986: Bump Apache Parquet to
1.11.1
add 71b7d8b [BEAM-10925] Load SQL UDFs from jar.
add 25b8903 Move JavaUdfLoader from zetasql -> sql.
add ee4d14a Make JavaUdfDefinitions a subclass of JavaUdfLoader.
add 1fbf45c Don't set context classloader.
add 4f43726 Merge pull request #13629 from ibzib/udf-loader
add 5d130bb Revert "Revert "Merge pull request #12647 from [BEAM-10378]
Deserializing Azure Credentials""
add 3b74cbf Fixing checker framework checks
add 17992cc Merge pull request #13808 from Roll forward [BEAM-10378]
Deserializing Azure Credentials #12647 with fix
add 34ce707 [BEAM-9541] Push all docker images from RC instead of
hard-coding them.
add 867a298 Merge pull request #13798 from ibzib/container-pull
add f2c9b6e [BEAM-9541] Update Python SDK's Flink version list.
add 7c4a21a8 Merge pull request #13810 from ibzib/flink-python-version
add ea9e277 [BEAM-11272] Remove combiner label constructor arg (#13355)
add 4c81134 [BEAM-9541] Create a Gradle task to push all docker images,
and use it instead of bash when building the release candidate.
add f4f4611 Merge pull request #13811 from ibzib/flink-docker-push
add 9863b57 [BEAM-11691] Skip JavaUdfLoaderTest instead of failing when
jar path system properties aren't set. (#13814)
add 5c213dd Merge pull request #13757: [BEAM-11640] Linkage Checker
version upgrade to 1.5.4 (latest)
add dca92f7 [BEAM-11689] Use public.nexus.pentaho.org for pentaho
dependencies instead of Spring.
add 3d7c3a7 Merge pull request #13813 from ibzib/BEAM-11689
add ed5b117 [BEAM-10961] Enable strict dependency checking on
sdks/java/io/hadoop-format (#13799)
add fc14f0b [BEAM-11695] Remove translations.pack_combiners from default
optimizers for FnApiRunner and PortableRunner (#13816)
add cb4c2de [BEAM-11697] Upgrade Flink runner to Flink versions 1.12.1
and 1.11.3
add 4edb26f Merge pull request #13818 from
iemejia/BEAM-11697-update-flink-versions
add 1b6c8e3 Enabled strict dependency on flink runner
add 0144bdb Added new line
add e7dacc2 removed checker-qual
add 75bc373 Removed flink-clients dep
add 02e3666 set enableStrictDependencies:true
add 5d83602 made flink-clients runtimeOnly
add 248e990 Remove stray colon.
add 280107a Merge pull request #13658 from sonam-vend/flink-runner
add fc17382 Moving to 2.29.0-SNAPSHOT on master branch.
add a4e230b [BEAM-11531] Add pd.to_datetime, handle DeferredBase args in
_defer_to_pandas (#13706)
add 2975487 [BEAM-8725] bump snappy-java to 1.1.8.4
add f62887e Merge pull request #13820: [BEAM-8726] bump snappy-java to
1.1.8.4
add b0c8529 strict dependency checking on sdks/io/amazon-web-services*
add 9c749fe move amazon-web-services* changes to another PR
add a176ac9 undo previous erroneous commit
add 3824542 better use of variables
add 944365a Merge pull request #13793: [BEAM-10961] strict dependency
checking on sdks/io/amazon-web-services*
add 870cc22 [BEAM-11689] Add public.nexus.pentaho.org to offline
repositories.
add aab90f2 Merge pull request #13823 from ibzib/pentaho-offline
add 3b51aaa [BEAM-11695] Combiner packing in Dataflow (#13763)
add be2a2cf [BEAM-11165] Async ZetaSQL Calc
add 2cc0f8f Merge pull request #13759: [BEAM-11165] Async ZetaSQL Calc
add 8070497 Add BigDecimal support for SpannerIO
add 9a4505e Merge pull request #13762: [BEAM-11643] Add BigDecimal
support for SpannerIO
add a9d8612 [BEAM-11679] Override PubsubUnboundedSource transform for
dataflow runner v2
add 56cebda Populate PubsubMessage message id
add 02ec09c Enable tests
add 197d338 Exclude failing FhirIO ITs
add bc7930e Include runner v2 IT tests in java post commit
add ebc3bc3 Merge pull request #13797 from [BEAM-11679] Override
PubsubUnboundedSource transform for dataflow runner v2
add 0b85362 [BEAM-11699] Correct python environment directly for
fnApiJobServerSetup
add d289168 Merge pull request #13822 from y1chi/BEAM-11699
add f0a3f6d Merge pull request #13496 from [BEAM-11408] Integrate
BigQuery sink streaming inserts with GroupIntoBatches
add 01b7596 Add example snippets to read fromQuery using BQ Storage API.
add 393bbde Make the query example consistent with the previous one for
the table.
add 3c15bf3 Run spotelessApply.
add f24ebd3 Merge pull request #13083: [BEAM-7034] Add example snippet to
read fromQuery using BQ Storage API.
add b8c8bf3 [BEAM-10120] Add dynamic timer support to portable Flink.
add 73731ec Merge pull request #13783 from [BEAM-10120] Add dynamic timer
support to portable Flink
add 623a9c0 [BEAM-11076] Reuse TriggerProto translation in
StreamingGroupAlsoByWindowViaWindowSetFn (#13831)
add a0b3a75 [BEAM-11658] Match .snappy files into the given (de)compressor
add 96b4a58 Merge pull request #13821: [BEAM-11658] Match .snappy files
into the given (de)compressor
add 997680a Implemented BIT_XOR for Zetasql dialect
add 90ef75a replaced integer with Bigint type
add 1f2830c added int32 field in testcase
add 5062684 INT32 to INT64
add 4edb5a2 fixed spotless checks
add e6b3c62 SQL_Java11 PreCommit issue fixed
add adc0f2c removed * dependencies
add 4f428a5 removed * dependencies from zetasql test file
add e7dc079 removed * dependency from SQloperators.java
add 2a7c635 Fixed check style violations
add d5b520b fixed checkstyle violations from sqloperators table
add 7e2e7af fixed violations from beambuiltinagg file
add 7add8cf bit_Xor to support bytes as well
add b195374 deleted extra introduced class
add 25303ab added meaningful name for unit test
add 42dfc39 Merge pull request #13569 from sonam-vend/feature/bit_xor
add b060998 [BEAM-11415] Enabling new ValidatesRunner framework on
Jenkins. (#13684)
add 506e5cf Disable Gradle module metadata generation
add 4d50329 Merge pull request #13836: [BEAM-11709] Disable Gradle module
metadata generation for Beam BOM
add ffb0091 [BEAM-10925] Load user-provided jars in CalcFn.
add 989c317 [BEAM-10925] Fix code style
add 6f4c664 Wrap classloader creation in doPrivileged block.
add 96280f7 Merge pull request #13835 from ibzib/calcfn-jars
add b3c931a [BEAM-10925] Add Java UDFs to UserFunctionDefinitions holder.
add 2e0f0a4 Add javadoc comment to JavaScalarFunction::getJarPath.
add bb88778 Merge pull request #13842 from ibzib/java-udf-wrapper
add 477fc28 [BEAM-10925] Add optional jar path field to
ZetaSqlScalarFunctionImpl.
add e47f0e5 Merge pull request #13844 from ibzib/jar-path
add 60e8761 [BEAM-5214] Update java quickstart (#13833)
add 9d19510 [BEAM-11634] Give JobInvoker threads unique names.
add a44768f Merge pull request #13746 from ibzib/BEAM-11634
add 5127734 [BEAM-10925] Add rule to replace Calc with BeamCalcRel for
ZetaSQL UDFs.
add a497ff2 Merge pull request #13841 from ibzib/calc-rule
add 6b76f4b [website] Clarify JIRA priority for security issues (CVEs)
add bf5810a Merge pull request #13819: [website] Clarify JIRA priority
for security issues (CVEs)
add 7f209ef Merge commit 'bf5810a311737573827b8e8d24ad910b5d4e77fb' into
website-revamp
No new revisions were added by this update.
Summary of changes:
.github/actions/cancel-workflow-runs | 1 +
.github/actions/github-push-action | 1 +
.github/actions/gradle-command-action | 1 +
.github/workflows/build_wheels.yml | 7 +-
.github/workflows/cancel.yml | 7 +-
.github/workflows/java_tests.yml | 19 +-
.gitignore | 1 +
.gitmodules | 9 +
.test-infra/jenkins/CommonTestProperties.groovy | 6 +-
...ommit_CrossLanguageValidatesRunner_Flink.groovy | 3 +-
.../job_PostCommit_Java_Nexmark_Flink.groovy | 13 +-
...Java_PortableValidatesRunner_Flink_Batch.groovy | 3 +-
..._PortableValidatesRunner_Flink_Streaming.groovy | 5 +-
...ob_PostCommit_Java_ValidatesRunner_Flink.groovy | 3 +-
...Commit_Java_ValidatesRunner_Flink_Java11.groovy | 7 +-
.../jenkins/job_Publish_Docker_Snapshots.groovy | 3 +-
CHANGES.md | 17 +-
build.gradle | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 27 +-
.../org/apache/beam/gradle/Repositories.groovy | 5 +-
examples/java/build.gradle | 17 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 10 +-
.../kafkatopubsub/kafka/consumer/Utils.java | 8 +
.../options/KafkaToPubsubOptions.java | 6 +
...igQueryReadFromQueryWithBigQueryStorageAPI.java | 74 +
.../kafkatopubsub/KafkaToPubsubE2ETest.java | 129 +
examples/kotlin/build.gradle | 14 +-
gradle.properties | 4 +-
gradle/wrapper/gradle-wrapper.properties | 2 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 30 +-
model/pipeline/src/main/proto/metrics.proto | 28 +-
release/build.gradle | 1 +
.../groovy/GoogleCloudPlatformBomArchetype.groovy | 44 +
.../groovy/mobilegaming-java-dataflowbom.groovy | 61 +
.../src/main/scripts/build_release_candidate.sh | 48 +-
release/src/main/scripts/cut_release_branch.sh | 6 +-
release/src/main/scripts/publish_docker_images.sh | 87 +-
release/src/main/scripts/run_rc_validation.sh | 11 +
runners/core-construction-java/build.gradle | 5 +-
runners/core-java/build.gradle | 8 +-
.../beam/runners/core/metrics/HistogramCell.java | 96 +
.../beam/runners/core/metrics/LabeledMetrics.java | 14 +
.../runners/core/metrics/MetricsContainerImpl.java | 104 +-
.../core/metrics/MonitoringInfoConstants.java | 10 +
.../core/metrics/MonitoringInfoMetricName.java | 2 +-
runners/direct-java/build.gradle | 14 +-
runners/extensions-java/metrics/build.gradle | 4 +-
runners/flink/1.10/job-server/build.gradle | 4 +
runners/flink/1.11/build.gradle | 2 +-
runners/flink/1.11/job-server/build.gradle | 4 +
runners/flink/1.12/build.gradle | 2 +-
runners/flink/1.12/job-server/build.gradle | 4 +
runners/flink/1.8/job-server/build.gradle | 4 +
runners/flink/1.9/job-server/build.gradle | 4 +
runners/flink/flink_runner.gradle | 12 +-
runners/flink/job-server/flink_job_server.gradle | 1 -
.../runners/flink/FlinkExecutionEnvironments.java | 15 +-
.../apache/beam/runners/flink/FlinkJobInvoker.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 19 +-
.../streaming/ExecutableStageDoFnOperator.java | 7 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 4 +-
runners/google-cloud-dataflow-java/build.gradle | 25 +-
.../beam/runners/dataflow/DataflowRunner.java | 211 +-
.../beam/runners/dataflow/TestDataflowRunner.java | 6 +-
.../dataflow/worker/StreamingDataflowWorker.java | 4 +
.../StreamingGroupAlsoByWindowViaWindowSetFn.java | 6 +-
.../dataflow/worker/WindmillStateCache.java | 11 +-
runners/java-fn-execution/build.gradle | 7 +-
.../fnexecution/control/TimerReceiverFactory.java | 1 +
.../translation/PipelineTranslatorUtils.java | 4 +-
runners/java-job-service/build.gradle | 10 +-
runners/jet/build.gradle | 8 +-
runners/local-java/build.gradle | 3 +-
runners/portability/java/build.gradle | 11 +-
runners/samza/build.gradle | 21 +-
runners/samza/job-server/build.gradle | 2 +
runners/spark/build.gradle | 7 +-
runners/spark/job-server/build.gradle | 4 +-
.../apache/beam/runners/spark/SparkJobInvoker.java | 2 +-
.../beam/runners/spark/SparkRunnerRegistrar.java | 9 +-
.../SparkStructuredStreamingRunnerRegistrar.java} | 22 +-
.../runners/spark/SparkRunnerRegistrarTest.java | 10 +-
...arkStructuredStreamingRunnerRegistrarTest.java} | 28 +-
runners/twister2/build.gradle | 17 +-
sdks/go/pkg/beam/coder.go | 22 +
sdks/go/pkg/beam/core/core.go | 31 +
sdks/go/pkg/beam/core/graph/coder/iterable.go | 36 +
sdks/go/pkg/beam/core/graph/coder/registry_test.go | 1 -
sdks/go/pkg/beam/core/graph/coder/row.go | 64 +-
sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 50 +-
sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 40 +-
sdks/go/pkg/beam/core/graph/coder/row_test.go | 121 +-
sdks/go/pkg/beam/core/graph/fn.go | 19 +-
sdks/go/pkg/beam/core/runtime/genx/genx.go | 2 +
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 15 +-
.../core/runtime/graphx/schema/logicaltypes.go | 162 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 201 +-
.../beam/core/runtime/graphx/schema/schema_test.go | 121 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 9 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 10 +
sdks/go/pkg/beam/encoding.go | 131 +
sdks/go/pkg/beam/encoding_test.go | 127 +
sdks/go/pkg/beam/example_schema_test.go | 254 ++
sdks/go/pkg/beam/forward.go | 4 +
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 2241 +++++++++--------
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 2585 ++++++++++----------
sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go | 321 ++-
sdks/go/pkg/beam/pardo_test.go | 72 +
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 5 +-
.../beam/runners/dataflow/dataflowlib/metrics.go | 3 -
sdks/go/pkg/beam/runners/universal/universal.go | 4 +-
sdks/go/pkg/beam/schema.go | 72 +
sdks/go/pkg/beam/transforms/top/top.go | 73 +-
sdks/go/pkg/beam/transforms/top/top_test.go | 18 +-
sdks/go/test/integration/xlang/xlang_test.go | 41 +-
sdks/go/test/run_validatesrunner_tests.sh | 60 +-
sdks/java/bom/build.gradle | 9 +-
sdks/java/bom/common.gradle | 128 +
sdks/java/{io/common => bom/gcp}/build.gradle | 23 +-
sdks/java/build-tools/beam-linkage-check.sh | 18 +-
sdks/java/container/build.gradle | 5 +
.../container/license_scripts/dep_urls_java.yaml | 4 +-
sdks/java/core/build.gradle | 5 +-
.../org/apache/beam/sdk/coders/DequeCoder.java | 82 +
.../apache/beam/sdk/io/AvroSchemaIOProvider.java | 4 +-
.../org/apache/beam/sdk/io/CompressedSource.java | 19 +-
.../java/org/apache/beam/sdk/io/Compression.java | 21 +
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 8 +-
.../apache/beam/sdk/metrics/DelegatingCounter.java | 11 +-
...gatingCounter.java => DelegatingHistogram.java} | 40 +-
.../org/apache/beam/sdk/metrics/Histogram.java} | 21 +-
.../apache/beam/sdk/metrics/MetricsContainer.java | 9 +
.../beam/sdk/metrics/MetricsEnvironment.java | 19 +
.../org/apache/beam/sdk/metrics/MetricsLogger.java | 66 +
.../io/Failure.java} | 49 +-
.../org/apache/beam/sdk/schemas/io/GenericDlq.java | 66 +
.../beam/sdk/schemas/io/GenericDlqProvider.java | 31 +-
.../apache/beam/sdk/testing/CoderProperties.java | 29 +
.../beam/sdk/testing/TestPipelineOptions.java | 10 +
.../org/apache/beam/sdk/transforms/Create.java | 4 +
.../sdk/transforms/splittabledofn/SplitResult.java | 2 +-
.../util/{Histogram.java => HistogramData.java} | 33 +-
.../java/org/apache/beam/sdk/util/RowJson.java | 59 +-
.../beam/sdk/util/RowJsonValueExtractors.java | 52 +
.../org/apache/beam/sdk/coders/DequeCoderTest.java | 110 +
.../apache/beam/sdk/io/CompressedSourceTest.java | 462 ++--
.../apache/beam/sdk/schemas/io/GenericDlqTest.java | 61 +
.../beam/sdk/schemas/io/StoringDlqProvider.java | 72 +
.../apache/beam/sdk/util/HistogramDataTest.java | 152 ++
.../org/apache/beam/sdk/util/HistogramTest.java | 152 --
.../java/org/apache/beam/sdk/util/RowJsonTest.java | 41 +-
sdks/java/expansion-service/build.gradle | 3 +-
.../google-cloud-platform-core/build.gradle | 1 +
.../LatencyRecordingHttpRequestInitializer.java | 26 +-
...LatencyRecordingHttpRequestInitializerTest.java | 6 +-
sdks/java/extensions/sql/build.gradle | 14 +
.../datacatalog/DataCatalogTableProvider.java | 3 +-
.../sdk/extensions/sql/impl/JavaUdfLoader.java | 231 ++
.../extensions/sql/impl/ScalarFunctionImpl.java | 16 +-
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 42 +-
.../impl/transform/BeamBuiltinAggregations.java | 92 +-
.../sdk/extensions/sql/impl/JavaUdfLoaderTest.java | 90 +
.../meta/provider/avro/AvroTableProviderTest.java | 10 +-
.../ParquetTableProviderTest.java} | 25 +-
.../provider/parquet/ParquetTableReadTest.java | 93 -
.../sql/udf-test-provider}/build.gradle | 20 +-
.../extensions/sql/provider/UdfTestProvider.java | 33 +-
.../sdk/extensions/sql/provider/package-info.java} | 18 +-
...taSqlCalcRule.java => BeamJavaUdfCalcRule.java} | 15 +-
.../extensions/sql/zetasql/BeamZetaSqlCalcRel.java | 116 +-
.../sql/zetasql/BeamZetaSqlCalcRule.java | 2 +-
.../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 3 +
.../zetasql/SupportedZetaSqlBuiltinFunctions.java | 20 +-
.../sql/zetasql/ZetaSQLQueryPlanner.java | 41 +-
.../translation/SqlOperatorMappingTable.java | 1 +
.../sql/zetasql/translation/SqlOperators.java | 27 +-
.../translation/UserFunctionDefinitions.java | 21 +-
.../translation/ZetaSqlScalarFunctionImpl.java | 13 +-
.../sql/zetasql/ZetaSqlDialectSpecTest.java | 13 +
sdks/java/fn-execution/build.gradle | 3 +-
sdks/java/harness/build.gradle | 5 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 4 +
sdks/java/io/amazon-web-services/build.gradle | 4 +-
sdks/java/io/amazon-web-services2/build.gradle | 11 +-
sdks/java/io/amqp/build.gradle | 4 +-
.../DefaultBlobstoreClientBuilderFactory.java | 8 +-
.../beam/sdk/io/azure/options/AzureModule.java | 174 ++
.../sdk/io/azure/options/BlobstoreOptions.java | 24 +-
sdks/java/io/cassandra/build.gradle | 1 +
sdks/java/io/common/build.gradle | 2 +-
.../elasticsearch-tests-2/build.gradle | 1 +
.../elasticsearch-tests-5/build.gradle | 1 +
.../elasticsearch-tests-6/build.gradle | 1 +
.../elasticsearch-tests-7/build.gradle | 1 +
.../elasticsearch-tests-common/build.gradle | 1 +
.../sdk/io/gcp/bigquery/BatchedStreamingWrite.java | 403 +++
.../sdk/io/gcp/bigquery/BigQueryDlqProvider.java | 85 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 23 +
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 8 +-
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 -
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 72 +-
.../beam/sdk/io/gcp/bigquery/StreamingInserts.java | 29 +
.../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 211 --
.../sdk/io/gcp/bigquery/StreamingWriteTables.java | 168 +-
.../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 8 +-
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 184 +-
.../sdk/io/gcp/healthcare/FhirSearchParameter.java | 89 +
.../gcp/healthcare/FhirSearchParameterCoder.java | 61 +
.../sdk/io/gcp/healthcare/HealthcareApiClient.java | 2 +-
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 12 +-
.../beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java | 66 +
.../beam/sdk/io/gcp/pubsub/PubsubMessages.java | 7 +-
.../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 42 +-
.../beam/sdk/io/gcp/pubsublite/DlqProvider.java | 77 +
.../beam/sdk/io/gcp/pubsublite/Publishers.java | 24 +-
.../sdk/io/gcp/pubsublite/SubscribeTransform.java | 40 +-
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 30 +-
.../pubsublite/SubscriptionPartitionLoader.java | 102 +
.../sdk/io/gcp/spanner/MutationSizeEstimator.java | 23 +
.../sdk/io/gcp/testing/FakeBigQueryServices.java | 6 -
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 73 +-
.../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 78 +-
.../beam/sdk/io/gcp/healthcare/FhirIOTest.java | 39 +-
.../beam/sdk/io/gcp/pubsub/PubsubReadIT.java | 6 +-
.../SubscriptionPartitionLoaderTest.java | 84 +
.../io/gcp/spanner/MutationSizeEstimatorTest.java | 25 +
sdks/java/io/hadoop-common/build.gradle | 4 +-
sdks/java/io/hadoop-file-system/build.gradle | 4 +-
sdks/java/io/hadoop-format/build.gradle | 5 +-
sdks/java/io/hbase/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 65 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 43 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 6 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 344 +++
sdks/java/io/kudu/build.gradle | 3 +-
sdks/java/io/mongodb/build.gradle | 4 +-
sdks/java/io/parquet/build.gradle | 2 +-
.../sdk/io/parquet/ParquetSchemaIOProvider.java | 23 +-
sdks/java/io/rabbitmq/build.gradle | 4 +-
sdks/java/io/redis/build.gradle | 4 +-
sdks/java/javadoc/build.gradle | 4 +-
.../maven-archetypes/examples/generate-sources.sh | 2 +-
.../maven-archetypes/gcp-bom-examples/build.gradle | 81 +
.../java/maven-archetypes/gcp-bom-examples/pom.xml | 136 +
.../META-INF/maven/archetype-metadata.xml | 45 +
.../src/main/resources/archetype-resources/pom.xml | 460 ++++
sdks/java/testing/expansion-service/build.gradle | 7 +-
sdks/java/testing/jpms-tests/build.gradle | 2 +-
sdks/java/testing/kafka-service/build.gradle | 2 +-
sdks/java/testing/load-tests/build.gradle | 8 +-
sdks/java/testing/nexmark/build.gradle | 9 +-
sdks/java/testing/test-utils/build.gradle | 12 +-
sdks/python/apache_beam/dataframe/frames.py | 98 +-
sdks/python/apache_beam/dataframe/frames_test.py | 117 +-
.../apache_beam/dataframe/pandas_doctests_test.py | 36 +-
.../dataframe/pandas_top_level_functions.py | 35 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 8 +-
.../apache_beam/io/gcp/bigquery_read_internal.py | 3 +-
sdks/python/apache_beam/io/kafka.py | 34 +-
.../python/apache_beam/options/pipeline_options.py | 8 +-
.../runners/dataflow/dataflow_runner.py | 87 +-
.../runners/dataflow/dataflow_runner_test.py | 33 +
.../runners/dataflow/ptransform_overrides.py | 27 +
.../runners/portability/flink_runner_test.py | 8 +-
.../runners/portability/fn_api_runner/fn_runner.py | 1 -
.../portability/fn_api_runner/fn_runner_test.py | 52 +
.../portability/fn_api_runner/translations.py | 3 +-
.../runners/portability/portable_runner.py | 1 -
.../runners/portability/sdk_container_builder.py | 68 +-
.../portability/sdk_container_builder_test.py | 103 +
sdks/python/apache_beam/transforms/combiners.py | 10 +-
sdks/python/apache_beam/transforms/environments.py | 1 +
.../apache_beam/transforms/environments_test.py | 2 +
sdks/python/apache_beam/transforms/external.py | 3 +-
.../python/apache_beam/transforms/external_test.py | 30 +
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/build.gradle | 6 +
sdks/python/test-suites/direct/xlang/build.gradle | 4 +-
sdks/python/tox.ini | 92 +-
settings.gradle | 3 +
website/www/site/content/en/blog/beam-2.27.0.md | 1 +
.../content/en/blog/kafka-to-pubsub-example.md | 89 +
.../site/content/en/contribute/jira-priorities.md | 1 +
.../site/content/en/contribute/release-guide.md | 4 +-
.../documentation/io/built-in/google-bigquery.md | 2 +-
.../content/en/documentation/programming-guide.md | 150 +-
.../site/content/en/get-started/beam-overview.md | 24 +-
.../www/site/content/en/get-started/downloads.md | 2 +-
.../site/content/en/get-started/quickstart-java.md | 24 +-
website/www/site/data/authors.yml | 13 +
290 files changed, 11608 insertions(+), 4473 deletions(-)
create mode 160000 .github/actions/cancel-workflow-runs
create mode 160000 .github/actions/github-push-action
create mode 160000 .github/actions/gradle-command-action
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java
create mode 100644
examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java
create mode 100644
release/src/main/groovy/GoogleCloudPlatformBomArchetype.groovy
create mode 100644 release/src/main/groovy/mobilegaming-java-dataflowbom.groovy
create mode 100644
runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/HistogramCell.java
copy
runners/spark/src/main/java/org/apache/beam/runners/spark/{SparkRunnerRegistrar.java
=> structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java} (72%)
copy
runners/spark/src/test/java/org/apache/beam/runners/spark/{SparkRunnerRegistrarTest.java
=> structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java} (65%)
create mode 100644 sdks/go/pkg/beam/core/core.go
create mode 100644 sdks/go/pkg/beam/encoding_test.go
create mode 100644 sdks/go/pkg/beam/example_schema_test.go
create mode 100644 sdks/go/pkg/beam/schema.go
create mode 100644 sdks/java/bom/common.gradle
copy sdks/java/{io/common => bom/gcp}/build.gradle (65%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DequeCoder.java
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/{DelegatingCounter.java
=> DelegatingHistogram.java} (57%)
copy sdks/java/{io/common/build.gradle =>
core/src/main/java/org/apache/beam/sdk/metrics/Histogram.java} (54%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsLogger.java
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/{metrics/MetricsContainer.java
=> schemas/io/Failure.java} (50%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java
copy
runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/LabeledMetrics.java
=>
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlqProvider.java
(52%)
rename sdks/java/core/src/main/java/org/apache/beam/sdk/util/{Histogram.java
=> HistogramData.java} (87%)
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DequeCoderTest.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/GenericDlqTest.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/StoringDlqProvider.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramDataTest.java
delete mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramTest.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoaderTest.java
copy
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/{avro/AvroTableProviderTest.java
=> parquet/ParquetTableProviderTest.java} (79%)
delete mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/ParquetTableReadTest.java
copy sdks/java/{io/common => extensions/sql/udf-test-provider}/build.gradle
(59%)
copy
runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/LabeledMetrics.java
=>
sdks/java/extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/UdfTestProvider.java
(53%)
copy sdks/java/{io/common/build.gradle =>
extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/package-info.java}
(54%)
copy
sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/{BeamZetaSqlCalcRule.java
=> BeamJavaUdfCalcRule.java} (82%)
create mode 100644
sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzureModule.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchedStreamingWrite.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryDlqProvider.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirSearchParameter.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirSearchParameterCoder.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/DlqProvider.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoaderTest.java
create mode 100644
sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/build.gradle
create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/pom.xml
create mode 100644
sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/META-INF/maven/archetype-metadata.xml
create mode 100644
sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml
create mode 100644
sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py
create mode 100644 website/www/site/content/en/blog/kafka-to-pubsub-example.md
