This is an automated email from the ASF dual-hosted git repository.
amaliujia pushed a change to branch rw_beam_zetasql
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 46c9f24 fixup: create job_PreCommit_BeamSQL_ZetaSQL.groovy.
add 389f649 [BEAM-7755] adding support for nested rows and arrays in
BigQuery to Beam row conversion (#9089)
add 9f88443 Upgrade google-cloud-core and google-cloud-bigquery
dependencies
add 8117b58 Merge pull request #9162 from aaltay/gcc
add 0650b9b [BEAM-7641] Add xunit results to run_validatescontainer.sh
add 9a3adc7 Merge pull request #9171: [BEAM-7641] Add xunit results to
run_validatescontainer.sh
add 7095832 [BEAM-7608] Honor DATASTORE_EMULATOR_HOST env var
add 4eab256 Merge pull request #8921: [BEAM-7608] Honor
DATASTORE_EMULATOR_HOST env var
add 6cf3b11 [BEAM-6611] BigQuery file loads in Streaming for Python SDK
(#8871)
add 4c95d57 [BEAM-7662] Create Python Combine load test job on Flink
add bf19854 Merge pull request #9115: [BEAM-7662] Create Python Combine
load test job on Flink
add 1cabb1a Set job name in Jet Runner
add 85bb004 Merge pull request #9177: [BEAM-7305] Set job name in Jet
Runner
add 45ef1f2 [BEAM-7830]: make spark default duration consistent with
PipelineResult
add d1e3e90 fixup! [BEAM-7830]: make spark default duration consistent
with PipelineResult
add 4e40b53 Merge pull request #9170: [BEAM-7830] Make spark default
duration consistent with PipelineResult
add 3d087be [BEAM-7366] Restore Spotless check for Flink Runner
add 8f6247e Merge pull request #9176: [BEAM-7366] Restore Spotless check
for Flink Runner
add abe5a7e [BEAM-7783] BeamIOSourceRel using custom BeamStatistics
add e9e8731 Merge pull request #9161 from riazela/BeamTableStatistics
add 206bf5f [BEAM-7807] Forbid TestNG imports on Beam classes
add 707f10b [BEAM-7807] Update Beam tests to not import TestNG related
classes
add 0fcb9c6 Merge pull request #9141 from iemejia/BEAM-7807-ban-testng
add f9e0969 [BEAM-7814] Make BigqueryMatcher wait for results. (#9154)
add e813673 [BEAM-7623] Add support to select MAP with Row as values in
Beam SQL
add c80ca6b Merge pull request #9181 from bmv126/map_with_row_as_value
add f75c2fb Provision datasources and dashboards from config.
add 83bb3dd [BEAM-7700] Java transform catalog (#9129)
add 3ee9da9 [BEAM-7574] Use Map-based window combining
add 6c1d115 [BEAM-7574] use Merging and NonMerging accumulating strategies
add 945a606 [BEAM-7574] unify Combine.perKey and Combine.globally
add 7be5dff [BEAM-7574] Explode windows to key when needed
add 1691935 [BEAM-7574] Optimize GC performance, finalize implementation
add fb83467 Merge pull request #8946: [BEAM-7574] fix Combine performance
for SparkRunner
add bfdb169 Add arbitrary value matchers to MetricsResultMatchers.
add d088904 [BEAM-6883] Improve long-running spark streaming test.
add c787b5e Merge pull request #9178: [BEAM-6883] Improve long-running
spark streaming test.
add c8164a4 Nitpicky grammar fix in programming guide
add ccaf3fe Merge pull request #9180 Nitpicky grammar fix in programming
guide
add 0e60dfa Merge pull request #9183: [BEAM-7675] Unify Gradle test
configuration across Py2 and Py3 (#9183)
add c31a2b1 [BEAM-7837] Add adapter class for API incompatible changes in
Spark 3
add a60d550 Merge pull request #9191: [BEAM-7837] Add adapter class for
API incompatible changes in Spark 3
add c4c75bd [BEAM-6202] Change retry policy for dataflow's get_job (#9182)
add 80a99ed Sets a default environment for Dataflow runner
add 0f2c9dd Merge pull request #9165: [BEAM-7824] Sets a default
environment for Dataflow runner
add 30989de [BEAM-6877] trivial_inference: make remaining tests pass
(#8893)
add e378f13 [BEAM-7820] Add hot key detection to Batch Dataflow Worker
(#9186)
add 68acce8 [BEAM-2264] Reuse GCP credentials in GCS calls.
add 786ea64 Merge pull request #9060: [BEAM-2264] Reuse GCP credentials
in GCS calls.
add 5f36ea1 Change retry policy for dataflow's list_messages
add 2a18da4 Merge pull request #9196 from aaltay/m404
add 5fb6609 [BEAM-7728] [SQL] Support ParquetTable (#9054)
add 5a340c7 Fixed type on singleton side inputs sample
add 60fab9b Merge pull request #9195 from davidcavazos/filter-sample-fix
add e8fad15 [BEAM-3645] Add multiplexing for python FnApiRunner (#8979)
add 43a6430 [BEAM-7840] Provide MapTuple and FlatMapTuple for Python 3
users. (#9168)
add 841610f [BEAM-4948] Switches dataflow runner to use a container image
with updated Guava dependency.
add 380d5c9 Merge pull request #9205 [BEAM-4948] Switches Dataflow to
container with updated Guava dependency.
add 6cc59ac [BEAM-6783] byte[] breaks in BeamSQL codegen
add afeb935 Merge pull request #9169: [BEAM-6783] byte[] breaks in
BeamSQL codegen
add 172b563 [BEAM-7808] Add method to convert avro field to beam field
add 866b8e7 Merge pull request #9142 from bmv126/Avro_field_to_beam_field
add f8b3d3e Add "--workerHarnessContainerImage=" to dataflow java11
legacy worker test target
add 4684cb0 Merge pull request #9203 from boyuanzz/fix
add 7c6241f Moving to 2.16.0-SNAPSHOT on master branch.
add e641e27 Update design-documents.md
add e4e4590 Merge pull request #9207 from riazela/patch-2
add 7472f81 [BEAM-7389] Add code examples for Filter page (#9184)
add 06021ac Add git scm extension to wipe workspace before cloning project
add 2793a72 Merge pull request #9200: [BEAM-7788] Add git scm extension
to wipe workspace before cloning project
add 199618b Use dateutil for more flexible parsing of RFC-3339 / ISO-8601
formatted strings
add 4805f49 Pin python-dateutil to current major version
add 2dc8609 Merge pull request #9147: Use dateutil for more flexible
parsing of RFC-3339 formatted strings
add 0285f29 [BEAM-7751] Add a CoGroupByKey load testing Jenkins job
add b9f8590 Merge pull request #9087: [BEAM-7751] Add a CoGroupByKey load
testing Jenkins job
add 1a4a4f0 Don't map transforms to pubsub subscriptions unless neccessary
add 0226799 Merge pull request #9146 from
ostrokach/bugfix/pubsub_reader_global_scope
add 7ea4547 Make SDK worker resilient to bad logging services. (#9214)
add 11f9ca5 [BEAM-7844] Custom MetadataHandler for NodeStats
(RowRateWindow)
add e5366eb Merge pull request #9185 from riazela/RowRateWindow
add b8cd3ee Update website for v2.14.0
add 13c4bb6 Merge pull request #9157 from akedin/website-update-v214
add c6eb09c [BEAM-7813] FileIO.write() doesn't fail when writing to
invalid bucket
add 850e846 Merge pull request #9175: [BEAM-7813] FileIO.write() doesn't
fail when writing to invalid bucket
add 7c51e13 [BEAM-6611] Move limitations comment to TriggerCopyJobs
add 2129c31 Merge pull request #9216 from ttanay/bq-copy-comment
add a40a278 Added 2.14.0 blog post draft
add bdf3ea7 fixup
add a3ee812 fixup
add 3a7d1db Add known issues
add cab46a0 fixup
add 22ad0c2 Updated the download page anchor
add 1d2de66 Add ApproximateUnique link
add a866d6c Merge pull request #9201 from aaltay/bl214
add dd79f79 [SQL] Use reflection to instantiate planner.
add 20bb131 Merge pull request #9221 from apilloud/sql-reflection
add 48326ba Fix date on 2.14.0 download links
add b14bbd4 Merge pull request #9226 from akedin/fix-214-date
add 851a4a5 Merge pull request #8929: [BEAM-7620] Make
run_rc_validation.sh non-interactive
add c1aec62 [BEAM-7876] create_svg from graphviz returns bytes and need
to be converted to string in order for the interactive runner to work on Python3
add fa45914 Merge pull request #9220 from davidyan74/BEAM-7876
add 96d947a [BEAM-7828] Fixes Key type conversion from/to client entity
in Python Datastore IO (#9174)
add 98b2bd9 [BEAM-7664] Add all GBK Flink Python test cases to job.
add d89517f [BEAM-7664] Changed worker amount, added scaling cluster and
renamed jobs
add dace04f [BEAM-7664] Fix typo in Combine Flink test
add 544682f Merge pull request #9106 from
kkucharc/BEAM-7664-add-more-gbk-flink-test-cases
add c41cffd [BEAM-7878] Refine Spark runner dependencies
add 6ab9ae2 Merge pull request #9225: [BEAM-7878] Refine Spark runner
dependencies
add 851339e [BEAM-7869] Add load testing jobs to README.md
add 95bd48c Merge pull request #9230 from
mwalenia/BEAM-7869-load-tests-readme
add 3ca9149 [BEAM-7844] Implementing NodeStat Estimations for all the
nodes
add 7d56a23 Merge pull request #9198 from riazela/RowRateWindowEstimation
add 90d6ba1 [BEAM-7814] Fix BigqueryMatcher._matches when called more
than once.
add 31c8e87 Merge pull request #9224 from udim/bq-matcher-fix
add 1031fdf Fix minor typos (#9192)
add 76424e7 Update Python dependencies page for 2.14.0
add c6c3bce Merge pull request #9055 from rosetn/patch-4
add 30661c1 [BEAM-7079] Add Chicago Taxi Example runner script with
Python scripts
add 846db7d [BEAM-7079] Add Chicago Taxi Example gradle task and Jenkins
job
add dd4283f Merge pull request #8939 from
mwalenia/BEAM-7079-chicago-taxi-example
add 2c8bf9f [BEAM-7868] optionally skip hidden pipeline options
add 47cf7ee Merge pull request #9211 from angoenka/hidden_options
add 7a0bc8d [BEAM-7577] Allow ValueProviders in Datastore Query filters
(#8950)
add d349553 Retry Datastore writes on [Errno 32] Broken pipe
add fef16e9 Merge pull request #8346: [BEAM-7476] Retry Datastore writes
on [Errno 32] Broken pipe
add 149153b [BEAM-7060] Introduce Python3-only test modules (#9223)
add b5e4175 [BEAM-7877] Change the log level when deleting unknown
temoprary files in FileBasedSink
add dbcd265 Merge pull request #9227 from ihji/BEAM-7877
add 2c1932a Update design-documents.md
add 6976651 [BEAM-7889] Update RC validation guide due to
run_rc_validation.sh change
add cca9e6d Merge pull request #9241: [BEAM-7889] Update RC validation
guide due to run_rc_validation.sh change
add 522190f [BEAM-7060] Fix
:sdks:python:apache_beam:testing:load_tests:run breakage
add ea08174 Merge pull request #9239: [BEAM-7060] Fix
:sdks:python:apache_beam:testing:load_tests:run breakage
add c5c7076 Fix some copy paste errors
add cb27549 Merge pull request #9194 from TheNeuralBit/coder-cleanup
add 01f3606 [Java] remove unneeded junit dependency.
add 917a613 Merge pull request #9208 from
amaliujia/rw-remove-unneeded-junit
add fc9e0aa [BEAM-7894] Upgrade AWS KPL to version 0.13.1
add e70eb15 Merge pull request #9249: [BEAM-7894] Upgrade AWS KPL to
version 0.13.1
add 9cf6f0d [BEAM-7898] Remove default implementation of getRowCount and
change the name to getTableStatistics
add 0d911b8 Merge pull request #9254 from riazela/TablesStatEstimation
add ec8b65a [BEAM-7389] Add helper conversion samples and simplified tests
add 913f065 Merge pull request #9252 from
davidcavazos/element-wise-with-timestamps
add 08d0146 [BEAM-7607] Per user request, making maxFilesPerBundle public
(#9160)
add 2387cbc [BEAM-7880] Upgrade Jackson databind to version 2.9.9.3
add 8a33913 Merge pull request #9229: [BEAM-7880] Upgrade Jackson
databind to version 2.9.9.3
add 00eef79 [BEAM-7721] Add a new module with BigQuery IO Read
performance test.
add 752d163 [BEAM-7721] Add cleanup to test and change the way of
reporting metrics
add 997bc70 [BEAM-7721] Refactor metric gathering
add 49644d3 Merge pull request #9041: [BEAM-7721] Add a new module with
BigQuery IO Read performance test
add 6252b66 Update Dataflow container images used by unreleased (dev) SDK.
add bd54ac5 Merge pull request #9243 from tvalentyn/patch-59
add a6a57df Increase default chunk size for gRPC commit and get data
streams. The initial choice of chunk size was arbitrary, and there is evidence
from testing that larger chunks improve performance.
add 7618f93 [BEAM-7901] Increase gRPC stream chunk sizes - fix test
add 6159162 [BEAM-7901] Increase default chunk size for gRPC commit and
get data streams.
add e1b2022 Adapt Jet Runner page to runner being released now
add a5c8ae0 Merge pull request #9245: [BEAM-7305] Adapt Jet Runner page
to runner being released now
add b47bc95 [BEAM-7777] Wiring up BeamCostModel
add 5fb50c7 [BEAM-7777] Implementing beamComputeSelfCost for all the rel
nodes
add a694fda Merge pull request #9217 from riazela/BeamCostModel
add 40936ba BEAM-7018: Added regex transform on Python SDK.
add 0a2ddc0 Merge pull request #8859 from mszb/BEAM-7018
add 3d75784 [BEAM-7845] Install Python SDK using pip instead of
setuptools.
add f1996da Merge pull request #9274 from tvalentyn/patch-60
add ee0d83d Update Python 2.7.x restriction
add dda2061 Merge pull request #9134 from rosetn/patch-5
add 24e9ced [BEAM-7833] warn user when --region flag is not explicitly
set (#9173)
add 164f249 Update env variable name in sync script
add b45bc6f [BEAM-6683] add createCrossLanguageValidatesRunner task
add 9678149 Merge pull request #8174: [BEAM-6683] add
createCrossLanguageValidatesRunner task
add 6fa94c9 [BEAM-7862] Moving FakeBigQueryServices to published
artifacts (#9206)
add 7ea9825 [BEAM-7899] Fix Python Dataflow VR tests by specify
sdk_location
add ff0f308 Merge pull request #9269: [BEAM-7899] Fix Python Dataflow VR
tests by specifying sdk_location
add bc2c6ff [BEAM-7678] Fixes bug in output element_type generation in Kv
PipelineVisitor (#9238)
add 1a3eed5 [BEAM-7389] Add code examples for MapTuple and FlatMapTuple
add 5e6ec5d5 Merge pull request #9276 from davidcavazos/map-flatmap-tuple
add 41d6dd9 [BEAM-7915] show cross-language validate runner Flink badge
on github PR template
add d9b43fc Merge pull request #9282 from ihji/BEAM-7915
add 124e6b6 Update python-pipeline-dependencies.md
add 28a4057 Merge pull request #9291 from rosetn/patch-8
add 1d20042 [BEAM-7860] Python Datastore: fix key sort order
add 0325c36 Merge pull request #9240: [BEAM-7860] Python Datastore: fix
key sort order
add eb8a29c Update python-sdk.md
add 2230cc9 Merge pull request #9290 from rosetn/patch-7
add 5504aa7 [BEAM-7918] adding nested row implementation for unnest and
uncollect (#9288)
add f792e2e Add helper functions for reading and writing to PubSub
directly from Python (#9212)
add 1bc2dd8 [BEAM-7060] Use typing in type decorators of core.py
add 67c8e9c Fully qualify use of internal typehints.
add 212596e Merge pull request #9179 [BEAM-7060] Use typing in type
decorators of core.py
add c9e5ea8 [BEAM -7741] Implement SetState for Python SDK (#9090)
add 57d8fd3 [BEAM-7912] Optimize GroupIntoBatches for batch Dataflow
pipelines.
add 7848ead [BEAM-7912] Optimize GroupIntoBatches for batch Dataflow
pipelines.
add f1dc92f [BEAM-7613] HadoopFileSystem can work with more than one
cluster.
add 25ac4ef [BEAM-7613] HadoopFileSystem can work with more than one
cluster.
add 0604e04 [BEAM-7924] Failure in Python 2 postcommit:
crossLanguagePythonJavaFlink
add 497bc77 Merge pull request #9292: [BEAM-7924] Failure in Python 2
postcommit: crossLanguagePythonJavaFlink
add 30b1ff0 [BEAM-7776] Create kubernetes.sh script to use kubectl with
desired namespace and kubeconfig file
add 22b97be [BEAM-7776] Stop using PerfkitBenchmarker in MongoDBIOIT job
add e1d4403 [BEAM-7776] Move repeatable kubernetes jenkins steps to
Kubernetes.groovy
add 0ecd166 Merge pull request #9116: [BEAM-7776] Stop Using Perfkit in
IOIT
add 13450c4 [BEAM-6907] Reuse Python tarball in integration tests
add 1545120 Add missing dependency and source copy in tox test
add b3c2915 Build sdk tarball before running installGcpTest task
add b69c81a [BEAM-6907] Reuse Python tarball in tox & dataflow
integration tests
add 936110f [BEAM-7820] Add basic hot key detection logging in Worker.
(#9270)
add ca050b9 use vendor-bytebuddy in sdks-java-core
add 3e97543 [BEAM-5822] Use vendored bytebuddy in sdks-java-core
add 459e270 Add assertArrayCountEqual, which checks if two containers
have the same elements (#9235)
add c164fbf Move DirectRunner specific classes into direct runner package.
add fffe83a Merge pull request #9297 Move DirectRunner specific classes
into direct runner package.
add 9d131d4 [BEAM-7896] Implementing RateEstimation for KafkaTable with
Unit and Integration Tests
add cd2ab9e Merge pull request #9298 from riazela/KafkaRateEstimation2
add 6454f72 [BEAM-7874], [BEAM-7873] Distributed FnApiRunner bugfixs
(#9218)
add ce3d121 [BEAM-7940] Fix beam_Release_Python_NightlySnapshot
add 59464cf Merge pull request #9307: [BEAM-7940] Fix
beam_Release_Python_NightlySnapshot
add dfd46d8 [BEAM-7846] add test for BEAM-7689
add a2b57e3 Merge pull request #9228 from ihji/BEAM-7846
add 9146b9c [BEAM-4775] Ensure each JobInvocation test is run in isolation
add f392558 [BEAM-4775] Refactor SamzaRunner - separate pipeline result
creation
add c81dd52 [BEAM-4775] Introduce PortablePipelineResult interface
add fc6cf80 [BEAM-4775] Use PortablePipelineResult in all portable runners
add e3ac1c1 [BEAM-4775] Allow reading portableMetrics from JobInvocation
using grpc calls
add ec647ee [BEAM-4775] Stash handle to pipeline result to query metrics
later
add 79fcd3b [BEAM-4775] Read portable metrics from grpc response in
Python SDK
add 5705e1f [BEAM-4775] Return portable metrics from Flink runner
add 2cfa48b Merge pull request #9020: [BEAM-4775] Support returning
metrics from job service
add 83bd4f1 [BEAM-7772] Remove Pkb from file-based IOIT jobs
add 716b9aa [BEAM-7772] Refactor: improve naming in the file
add f458576 [BEAM-7772] Refactor: autoformat for consistency (intelij
rules applied)
add 1668956 Merge pull request #9287: [BEAM-7776] Remove Pkb from
file-based IOIT jobs
add 6943270 upgrade calcite to 1.20.0
add 4501f60 api changes in calcite 1.20.0
add 26d5642 Merge pull request #9311: [SQL] Upgrade to Calcite 1.20.0
add 8e989a3 [BEAM-5148] Optionally load bson (#9308)
add babf86b [BEAM-5822] Remove non vendored byte-buddy dependency from
sdks/java/core
add 6c65527 [BEAM-5822] Remove non vendored byte-buddy dependency from
sdks/java/core
add 5adf530 Merge branch 'master' into rw_beam_zetasql
No new revisions were added by this update.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 1 +
.test-infra/jenkins/CommonJobProperties.groovy | 2 +-
.test-infra/jenkins/Infrastructure.groovy | 9 +
.test-infra/jenkins/Kubernetes.groovy | 108 +++
.test-infra/jenkins/README.md | 23 +
.../jenkins/job_LoadTests_CoGBK_Java.groovy | 209 ++++++
.../job_LoadTests_Combine_Flink_Python.groovy | 186 +++++
.../jenkins/job_LoadTests_GBK_Flink_Python.groovy | 155 +++-
.../job_PerformanceTests_FileBasedIO_IT.groovy | 189 +++--
.../job_PerformanceTests_MongoDBIO_IT.groovy | 83 +-
.../jenkins/job_PerformanceTests_Python.groovy | 17 +-
...mmit_CrossLanguageValidatesRunner_Flink.groovy} | 15 +-
...mit_Python_Chicago_Taxi_Example_Dataflow.groovy | 58 ++
.../job_PostCommit_Python_MongoDBIO_IT.groovy | 2 +-
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 4 +-
..._PostCommit_Python_ValidatesRunner_Spark.groovy | 2 +-
...b_PreCommit_Python_ValidatesRunner_Flink.groovy | 2 +-
.test-infra/kubernetes/kubernetes.sh | 93 +++
.test-infra/metrics/beamgrafana-deploy.yaml | 30 +-
.../metrics/build_and_publish_containers.sh | 57 ++
.../metrics/dashboards/pre-commit_tests.json | 241 ------
.test-infra/metrics/docker-compose.yml | 22 +-
.../metrics/grafana/Dockerfile | 15 +-
.../dashboards/Post-Commits_status_dashboard.json | 2 +-
.../{ => grafana}/dashboards/code_velocity.json | 71 +-
.../dashboards/post-commit_tests.json | 65 +-
.../dashboards/pre-commit_tests.json} | 295 ++++----
.../dashboards/source_data_freshness.json | 37 +-
.../dashboards/stability_critical_jobs_status.json | 8 +-
.../grafana/provisioning/dashboards/all.yaml | 22 +-
.../provisioning/datasources/beampostgresql.yaml | 25 +-
.test-infra/metrics/postgres/init.sql | 18 +
.test-infra/metrics/sync/github/sync.py | 4 +-
.test-infra/metrics/sync/jenkins/syncjenkins.py | 11 +-
build.gradle | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 203 +++--
examples/java/README.md | 2 +-
examples/kotlin/README.md | 2 +-
gradle.properties | 2 +-
.../Core Transforms/Map/FlatMapElements/task.html | 2 +-
.../java/Core Transforms/Map/MapElements/task.html | 2 +-
ownership/JAVA_DEPENDENCY_OWNERS.yaml | 15 -
release/src/main/scripts/run_rc_validation.sh | 540 ++++++-------
release/src/main/scripts/script.config | 135 ++++
runners/core-construction-java/build.gradle | 21 -
.../graph/GreedyPCollectionFusers.java | 4 +-
.../runners/core/construction/ExternalTest.java | 47 +-
.../expansion/TestExpansionService.java | 58 --
runners/core-java/build.gradle | 1 -
runners/flink/flink_runner.gradle | 11 +
runners/flink/job-server/flink_job_server.gradle | 30 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 40 +-
.../runners/flink/FlinkPortableRunnerResult.java | 53 ++
runners/google-cloud-dataflow-java/build.gradle | 23 +-
.../beam/runners/dataflow/DataflowRunner.java | 77 ++
.../dataflow/options/DataflowPipelineOptions.java | 4 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 63 +-
.../worker/DataflowWorkProgressUpdater.java | 19 +-
.../beam/runners/dataflow/worker/HotKeyLogger.java | 77 ++
.../dataflow/worker/StreamingDataflowWorker.java | 23 +-
.../util/common/worker/WorkProgressUpdater.java | 2 +-
.../worker/windmill/GrpcWindmillServer.java | 8 +-
.../worker/DataflowWorkProgressUpdaterTest.java | 24 +-
.../worker/DataflowWorkUnitClientTest.java | 1 +
.../runners/dataflow/worker/HotKeyLoggerTest.java | 104 +++
.../worker/StreamingDataflowWorkerTest.java | 15 +-
.../worker/windmill/GrpcWindmillServerTest.java | 2 +-
.../jobsubmission/InMemoryJobService.java | 24 +
.../fnexecution/jobsubmission/JobInvocation.java | 33 +-
...lineRunner.java => PortablePipelineResult.java} | 15 +-
.../jobsubmission/PortablePipelineRunner.java | 3 +-
.../jobsubmission/JobInvocationTest.java | 62 +-
.../org/apache/beam/runners/jet/JetRunner.java | 7 +
.../beam/runners/samza/SamzaPipelineRunner.java | 4 +-
.../runners/samza/SamzaPortablePipelineResult.java | 48 ++
.../org/apache/beam/runners/samza/SamzaRunner.java | 16 +-
runners/spark/build.gradle | 12 +-
.../beam/runners/spark/SparkPipelineResult.java | 27 +-
.../beam/runners/spark/SparkPipelineRunner.java | 6 +-
.../spark/coders/SparkRunnerKryoRegistrator.java | 2 -
.../spark/translation/GroupCombineFunctions.java | 110 +--
.../GroupNonMergingWindowsFunctions.java | 202 ++---
.../spark/translation/SparkAbstractCombineFn.java | 124 ---
.../SparkBatchPortablePipelineTranslator.java | 4 +-
.../runners/spark/translation/SparkCombineFn.java | 833 +++++++++++++++++++++
.../spark/translation/SparkGlobalCombineFn.java | 261 -------
.../spark/translation/SparkKeyedCombineFn.java | 278 -------
.../spark/translation/TransformTranslator.java | 56 +-
.../spark/translation/TranslationUtils.java | 12 +-
.../streaming/StreamingTransformTranslator.java | 9 +-
.../beam/runners/spark/util/SparkCompat.java | 65 ++
.../beam/runners/spark/io/NumShardsTest.java | 2 +-
.../GroupNonMergingWindowsFunctionsTest.java | 124 ++-
.../spark/translation/SparkCombineFnTest.java | 298 ++++++++
.../streaming/StreamingSourceMetricsTest.java | 29 +-
sdks/go/pkg/beam/pardo.go | 2 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 8 +-
.../src/main/resources/beam/checkstyle.xml | 16 +
sdks/java/core/build.gradle | 4 +-
.../apache/beam/sdk/coders/BigEndianLongCoder.java | 2 +-
.../beam/sdk/coders/BigEndianShortCoder.java | 2 +-
.../org/apache/beam/sdk/coders/FloatCoder.java | 2 +-
.../apache/beam/sdk/coders/RowCoderGenerator.java | 44 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 2 +-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 5 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 2 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 3 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 4 +-
.../beam/sdk/options/PipelineOptionsReflector.java | 11 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 2 +-
.../org/apache/beam/sdk/options/ValueProvider.java | 1 +
.../beam/sdk/schemas/utils/AutoValueUtils.java | 40 +-
.../beam/sdk/schemas/utils/AvroByteBuddyUtils.java | 22 +-
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 24 +-
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 54 +-
.../beam/sdk/schemas/utils/ConvertHelpers.java | 24 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 30 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 42 +-
.../org/apache/beam/sdk/transforms/Combine.java | 5 +
.../beam/sdk/transforms/GroupIntoBatches.java | 5 +
.../java/org/apache/beam/sdk/transforms/ParDo.java | 4 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 68 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 32 +-
.../reflect/StableInvokerNamingStrategy.java | 4 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 2 +-
.../beam/sdk/transforms/windowing/Sessions.java | 6 +
.../org/apache/beam/sdk/util/WindowedValue.java | 55 +-
.../org/apache/beam/sdk/io/FileBasedSinkTest.java | 16 +
.../java/org/apache/beam/sdk/io/SimpleSink.java | 4 +
.../beam/sdk/metrics/MetricResultsMatchers.java | 158 ++--
.../sdk/options/PipelineOptionsReflectorTest.java | 20 +-
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 32 +
.../beam/sdk/transforms/ParDoLifecycleTest.java | 2 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 20 +
sdks/java/extensions/sql/build.gradle | 8 +-
.../sql/meta/provider/hcatalog/HCatalogTable.java | 7 +
.../extensions/sql/src/main/codegen/config.fmpp | 4 +
.../beam/sdk/extensions/sql/BeamSqlTable.java | 9 +-
.../sdk/extensions/sql/impl/BeamCalciteTable.java | 5 +-
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 13 +-
.../extensions/sql/impl/BeamTableStatistics.java | 2 +-
.../extensions/sql/impl/CalciteQueryPlanner.java | 33 +-
.../sql/impl/parser/SqlCreateExternalTable.java | 4 +-
.../extensions/sql/impl/planner/BeamCostModel.java | 253 +++++++
.../sdk/extensions/sql/impl/planner/NodeStats.java | 86 +++
.../sql/impl/planner/NodeStatsMetadata.java | 55 ++
.../sql/impl/planner/RelMdNodeStats.java | 84 +++
.../sql/impl/rel/BeamAggregationRel.java | 67 +-
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 256 +++++--
.../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 14 +
.../extensions/sql/impl/rel/BeamIOSourceRel.java | 38 +-
.../extensions/sql/impl/rel/BeamIntersectRel.java | 33 +
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 33 +-
.../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 26 +
.../sdk/extensions/sql/impl/rel/BeamRelNode.java | 26 +
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 20 +
.../extensions/sql/impl/rel/BeamSqlRelUtils.java | 19 +
.../extensions/sql/impl/rel/BeamUncollectRel.java | 25 +-
.../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 28 +
.../sdk/extensions/sql/impl/rel/BeamUnnestRel.java | 39 +-
.../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 15 +
.../extensions/sql/impl/rule/BeamUnnestRule.java | 4 +-
.../sql/impl/schema/BeamPCollectionTable.java | 7 +
.../sql/meta/provider/bigquery/BigQueryTable.java | 6 +-
.../sql/meta/provider/kafka/BeamKafkaTable.java | 152 +++-
.../parquet/GenericRecordReadConverter.java | 64 ++
.../sql/meta/provider/parquet/ParquetTable.java | 67 ++
.../provider/parquet/ParquetTableProvider.java | 52 ++
.../sql/meta/provider/parquet/package-info.java | 11 +-
.../meta/provider/pubsub/PubsubIOJsonTable.java | 7 +
.../provider/seqgen/GenerateSequenceTable.java | 7 +
.../sql/meta/provider/test/TestBoundedTable.java | 7 +
.../sql/meta/provider/test/TestTableProvider.java | 2 +-
.../sql/meta/provider/test/TestUnboundedTable.java | 14 +
.../sql/meta/provider/text/TextTable.java | 4 +-
.../sdk/extensions/sql/BeamComplexTypeTest.java | 129 ++++
.../sql/impl/planner/BeamCostModelTest.java | 103 +++
.../sql/impl/planner/CalciteQueryPlannerTest.java | 73 ++
.../extensions/sql/impl/planner/NodeStatsTest.java | 94 +++
.../sdk/extensions/sql/impl/rel/BaseRelTest.java | 6 +-
.../sql/impl/rel/BeamAggregationRelTest.java | 152 ++++
.../extensions/sql/impl/rel/BeamCalcRelTest.java | 163 ++++
.../sql/impl/rel/BeamEnumerableConverterTest.java | 6 +
.../sql/impl/rel/BeamIOSourceRelTest.java | 163 ++++
.../sql/impl/rel/BeamIntersectRelTest.java | 27 +
.../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 75 ++
.../rel/BeamJoinRelUnboundedVsBoundedTest.java | 50 +-
.../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 50 +-
.../extensions/sql/impl/rel/BeamMinusRelTest.java | 99 ++-
.../extensions/sql/impl/rel/BeamSortRelTest.java | 39 +-
.../sql/impl/rel/BeamUncollectRelTest.java | 103 +++
.../extensions/sql/impl/rel/BeamUnionRelTest.java | 53 ++
.../extensions/sql/impl/rel/BeamUnnestRelTest.java | 71 ++
.../extensions/sql/impl/rel/BeamValuesRelTest.java | 24 +
.../sql/impl/rule/JoinReorderingTest.java | 6 +-
.../meta/provider/bigquery/BigQueryRowCountIT.java | 6 +-
.../meta/provider/bigquery/BigQueryTestTable.java | 4 +-
.../meta/provider/kafka/BeamKafkaCSVTableTest.java | 118 ++-
.../sql/meta/provider/kafka/KafkaCSVTableIT.java | 292 ++++++++
.../sql/meta/provider/kafka/KafkaCSVTestTable.java | 197 +++++
.../sql/meta/provider/kafka/KafkaTestRecord.java | 26 +-
.../provider/parquet/GenericRecordToRowTest.java | 79 ++
.../provider/parquet/ParquetTableReadTest.java | 90 +++
.../sql/src/test/resources/users.parquet | Bin 0 -> 615 bytes
sdks/java/io/bigquery-io-perf-tests/build.gradle | 40 +
.../BigQueryIOReadPerformanceIT.java | 203 +++++
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 6 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 13 +-
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 45 +-
.../sdk/io/gcp/testing}/FakeBigQueryServices.java | 14 +-
.../sdk/io/gcp/testing}/FakeDatasetService.java | 5 +-
.../beam/sdk/io/gcp/testing}/FakeJobService.java | 7 +-
.../beam/sdk/io/gcp/testing}/TableContainer.java | 2 +-
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryHelpersTest.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 3 +
.../gcp/bigquery/BigQueryIOStorageQueryTest.java | 7 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 3 +
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 79 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 12 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +-
.../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 84 ++-
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 49 +-
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 12 +-
sdks/java/io/kinesis/build.gradle | 2 +-
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 1 +
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 1 +
sdks/java/testing/expansion-service/build.gradle | 52 ++
.../beam/sdk/expansion/TestExpansionService.java | 101 +++
.../apache/beam/sdk/nexmark/NexmarkUtilsTest.java | 10 +-
sdks/python/apache_beam/coders/avro_coder.py | 99 ---
sdks/python/apache_beam/coders/avro_coder_test.py | 71 --
.../{version.py => coders/avro_record.py} | 21 +-
sdks/python/apache_beam/coders/coder_impl.py | 23 +
sdks/python/apache_beam/coders/coders.py | 41 +-
sdks/python/apache_beam/coders/coders_test.py | 43 ++
.../apache_beam/coders/coders_test_common.py | 1 +
.../apache_beam/examples/snippets/snippets.py | 2 +-
.../snippets/transforms/element_wise/filter.py | 2 +-
.../snippets/transforms/element_wise/flat_map.py | 27 +
.../transforms/element_wise/flat_map_test.py | 71 +-
.../snippets/transforms/element_wise/map.py | 23 +
.../snippets/transforms/element_wise/map_test.py | 71 +-
.../transforms/element_wise/with_timestamps.py | 22 +
.../element_wise/with_timestamps_test.py | 110 +--
.../python/apache_beam/examples/wordcount_xlang.py | 2 +-
sdks/python/apache_beam/internal/gcp/auth.py | 149 ++--
.../apache_beam/io/external/generate_sequence.py | 5 +-
.../io/external/generate_sequence_test.py | 64 ++
.../io/external/xlang_parquetio_test.py | 88 +++
sdks/python/apache_beam/io/filebasedsink_test.py | 8 +
sdks/python/apache_beam/io/gcp/bigquery.py | 39 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 129 +++-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 53 ++
sdks/python/apache_beam/io/gcp/bigquery_test.py | 7 +
.../apache_beam/io/gcp/datastore/v1/helper.py | 3 +-
.../io/gcp/datastore/v1/query_splitter_test.py | 11 +-
.../io/gcp/datastore/v1new/datastoreio.py | 11 +-
.../io/gcp/datastore/v1new/datastoreio_test.py | 16 +
.../apache_beam/io/gcp/datastore/v1new/helper.py | 6 +-
.../io/gcp/datastore/v1new/query_splitter.py | 63 +-
.../io/gcp/datastore/v1new/query_splitter_test.py | 51 +-
.../apache_beam/io/gcp/datastore/v1new/types.py | 61 +-
.../io/gcp/datastore/v1new/types_test.py | 39 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 33 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 2 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 14 -
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 62 +-
.../io/gcp/tests/bigquery_matcher_test.py | 13 +-
sdks/python/apache_beam/io/gcp/tests/utils.py | 57 +-
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 200 ++++-
sdks/python/apache_beam/io/mongodbio.py | 18 +-
.../python/apache_beam/options/pipeline_options.py | 18 +-
sdks/python/apache_beam/pipeline.py | 6 +-
sdks/python/apache_beam/pipeline_test.py | 83 ++
.../runners/dataflow/dataflow_runner.py | 15 +-
.../runners/dataflow/internal/apiclient.py | 22 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
.../apache_beam/runners/direct/direct_userstate.py | 134 +++-
.../runners/direct/transform_evaluator.py | 66 +-
.../interactive/display/pipeline_graph_renderer.py | 2 +-
.../python/apache_beam/runners/pipeline_context.py | 3 +
.../runners/portability/expansion_service_test.py | 50 +-
.../runners/portability/fn_api_runner.py | 476 +++++++-----
.../runners/portability/fn_api_runner_test.py | 38 +-
.../runners/portability/local_job_service.py | 24 +-
.../runners/portability/portable_runner.py | 13 +-
.../apache_beam/runners/worker/bundle_processor.py | 74 +-
.../apache_beam/runners/worker/data_plane.py | 33 +-
.../apache_beam/runners/worker/data_plane_test.py | 15 +-
.../apache_beam/runners/worker/log_handler.py | 26 +-
.../apache_beam/runners/worker/sdk_worker.py | 5 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 30 +-
.../chicago_taxi}/__init__.py | 0
.../testing/benchmarks/chicago_taxi/preprocess.py | 258 +++++++
.../benchmarks/chicago_taxi/process_tfma.py | 191 +++++
.../benchmarks/chicago_taxi/requirements.txt | 25 +
.../testing/benchmarks/chicago_taxi/run_chicago.sh | 192 +++++
.../testing/benchmarks/chicago_taxi/setup.py | 41 +
.../chicago_taxi/tfdv_analyze_and_validate.py | 225 ++++++
.../chicago_taxi/trainer}/__init__.py | 0
.../benchmarks/chicago_taxi/trainer/model.py | 164 ++++
.../benchmarks/chicago_taxi/trainer/task.py | 189 +++++
.../benchmarks/chicago_taxi/trainer/taxi.py | 186 +++++
.../python/apache_beam/testing/extra_assertions.py | 64 ++
.../apache_beam/testing/extra_assertions_test.py | 71 ++
.../apache_beam/testing/load_tests/build.gradle | 2 +-
sdks/python/apache_beam/testing/test_pipeline.py | 3 +
sdks/python/apache_beam/transforms/combiners.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 224 +++++-
.../python/apache_beam/transforms/external_test.py | 72 +-
sdks/python/apache_beam/transforms/ptransform.py | 2 +-
.../apache_beam/transforms/ptransform_test.py | 32 -
sdks/python/apache_beam/transforms/trigger.py | 16 +
sdks/python/apache_beam/transforms/userstate.py | 110 +--
.../apache_beam/transforms/userstate_test.py | 156 +++-
sdks/python/apache_beam/transforms/util.py | 222 ++++++
sdks/python/apache_beam/transforms/util_test.py | 280 +++++++
sdks/python/apache_beam/typehints/opcodes.py | 36 +
.../apache_beam/typehints/trivial_inference.py | 27 +-
.../typehints/trivial_inference_test.py | 33 +
.../typehints/trivial_inference_test_py3.py | 50 ++
sdks/python/apache_beam/utils/retry.py | 9 +
sdks/python/apache_beam/utils/timestamp.py | 30 +-
sdks/python/apache_beam/utils/timestamp_test.py | 9 +
sdks/python/apache_beam/version.py | 2 +-
sdks/python/build.gradle | 417 +----------
sdks/python/container/build.gradle | 2 +-
sdks/python/container/py3/build.gradle | 2 +-
sdks/python/container/run_validatescontainer.sh | 4 +
sdks/python/scripts/generate_pydoc.sh | 3 +
sdks/python/scripts/run_expansion_services.sh | 136 ++++
sdks/python/scripts/run_integration_test.sh | 9 +-
sdks/python/scripts/run_pylint.sh | 32 +-
sdks/python/setup.py | 5 +-
sdks/python/test-suites/dataflow/build.gradle | 53 --
.../dataflow/{py37 => py2}/build.gradle | 65 +-
sdks/python/test-suites/dataflow/py35/build.gradle | 17 +-
sdks/python/test-suites/dataflow/py36/build.gradle | 17 +-
sdks/python/test-suites/dataflow/py37/build.gradle | 21 +-
sdks/python/test-suites/direct/py2/build.gradle | 114 +++
sdks/python/test-suites/portable/py2/build.gradle | 235 ++++++
sdks/python/test-suites/portable/py35/build.gradle | 2 +-
sdks/python/test-suites/tox/py2/build.gradle | 59 ++
sdks/python/tox.ini | 6 +-
settings.gradle | 7 +-
website/_config.yml | 2 +-
website/src/.htaccess | 2 +-
website/src/_data/authors.yml | 3 +
.../src/_includes/section-menu/documentation.html | 67 +-
website/src/_posts/2019-07-31-beam-2.14.0.md | 106 +++
website/src/contribute/design-documents.md | 3 +-
website/src/contribute/release-guide.md | 7 +-
website/src/contribute/runner-guide.md | 26 +-
website/src/documentation/io/built-in.md | 4 +-
.../patterns/file-processing-patterns.md | 2 +-
website/src/documentation/programming-guide.md | 2 +-
website/src/documentation/runners/jet.md | 41 +-
.../src/documentation/sdks/python-dependencies.md | 39 +
.../sdks/python-pipeline-dependencies.md | 2 +-
.../java/aggregation/approximatequantiles.md | 43 ++
.../java/aggregation/approximateunique.md | 40 +
.../transforms/java/aggregation/cogroupbykey.md | 73 ++
.../transforms/java/aggregation/combine.md | 82 ++
.../java/aggregation/combinewithcontext.md | 37 +
.../transforms/java/aggregation/count.md | 50 ++
.../transforms/java/aggregation/distinct.md | 43 ++
.../transforms/java/aggregation/groupbykey.md | 50 ++
.../java/aggregation/groupintobatches.md | 42 ++
.../transforms/java/aggregation/latest.md | 52 ++
.../transforms/java/aggregation/max.md | 56 ++
.../transforms/java/aggregation/mean.md | 58 ++
.../transforms/java/aggregation/min.md | 42 ++
.../transforms/java/aggregation/sample.md | 40 +
.../transforms/java/aggregation/sum.md | 51 ++
.../transforms/java/aggregation/top.md | 39 +
.../transforms/java/element-wise/filter.md | 62 ++
.../java/element-wise/flatmapelements.md | 40 +
.../transforms/java/element-wise/keys.md | 43 ++
.../transforms/java/element-wise/kvswap.md | 43 ++
.../transforms/java/element-wise/mapelements.md | 63 ++
.../transforms/java/element-wise/pardo.md | 152 ++++
.../transforms/java/element-wise/partition.md | 62 ++
.../transforms/java/element-wise/regex.md | 36 +
.../transforms/java/element-wise/reify.md | 39 +
.../transforms/java/element-wise/tostring.md | 37 +
.../transforms/java/element-wise/values.md | 44 ++
.../transforms/java/element-wise/withkeys.md | 55 ++
.../transforms/java/element-wise/withtimestamps.md | 36 +
website/src/documentation/transforms/java/index.md | 81 ++
.../documentation/transforms/java/other/create.md | 40 +
.../documentation/transforms/java/other/flatten.md | 67 ++
.../documentation/transforms/java/other/passert.md | 61 ++
.../documentation/transforms/java/other/view.md | 37 +
.../documentation/transforms/java/other/window.md | 40 +
.../transforms/python/element-wise/filter.md | 171 ++++-
.../src/documentation/transforms/python/index.md | 2 +-
.../transforms/python/other/reshuffle.md | 2 +-
website/src/get-started/downloads.md | 7 +
website/src/get-started/quickstart-py.md | 2 +-
website/src/roadmap/python-sdk.md | 2 +-
404 files changed, 16817 insertions(+), 4433 deletions(-)
create mode 100644 .test-infra/jenkins/Kubernetes.groovy
create mode 100644 .test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy
create mode 100644
.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy
copy .test-infra/jenkins/{job_PostCommit_Python_MongoDBIO_IT.groovy =>
job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy} (69%)
create mode 100644
.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Dataflow.groovy
create mode 100755 .test-infra/kubernetes/kubernetes.sh
create mode 100755 .test-infra/metrics/build_and_publish_containers.sh
delete mode 100644 .test-infra/metrics/dashboards/pre-commit_tests.json
copy gradle.properties => .test-infra/metrics/grafana/Dockerfile (80%)
rename .test-infra/metrics/{ =>
grafana}/dashboards/Post-Commits_status_dashboard.json (99%)
rename .test-infra/metrics/{ => grafana}/dashboards/code_velocity.json (96%)
rename .test-infra/metrics/{ => grafana}/dashboards/post-commit_tests.json
(95%)
copy .test-infra/metrics/{dashboards/stability_critical_jobs_status.json =>
grafana/dashboards/pre-commit_tests.json} (51%)
rename .test-infra/metrics/{ => grafana}/dashboards/source_data_freshness.json
(89%)
rename .test-infra/metrics/{ =>
grafana}/dashboards/stability_critical_jobs_status.json (95%)
copy gradle.properties =>
.test-infra/metrics/grafana/provisioning/dashboards/all.yaml (79%)
copy gradle.properties =>
.test-infra/metrics/grafana/provisioning/datasources/beampostgresql.yaml (76%)
create mode 100644 .test-infra/metrics/postgres/init.sql
create mode 100755 release/src/main/scripts/script.config
delete mode 100644
runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/expansion/TestExpansionService.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableRunnerResult.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/HotKeyLogger.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/HotKeyLoggerTest.java
copy
runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/{PortablePipelineRunner.java
=> PortablePipelineResult.java} (64%)
create mode 100644
runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPortablePipelineResult.java
delete mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkCombineFn.java
delete mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
delete mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkCompat.java
create mode 100644
runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkCombineFnTest.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamCostModel.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/NodeStats.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/NodeStatsMetadata.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/RelMdNodeStats.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/GenericRecordReadConverter.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/ParquetTable.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/ParquetTableProvider.java
copy
runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineRunner.java
=>
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/package-info.java
(66%)
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamCostModelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/planner/CalciteQueryPlannerTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/planner/NodeStatsTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUncollectRelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRelTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaCSVTableIT.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaCSVTestTable.java
copy
runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineRunner.java
=>
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTestRecord.java
(55%)
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/GenericRecordToRowTest.java
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/parquet/ParquetTableReadTest.java
create mode 100644 sdks/java/extensions/sql/src/test/resources/users.parquet
create mode 100644 sdks/java/io/bigquery-io-perf-tests/build.gradle
create mode 100644
sdks/java/io/bigquery-io-perf-tests/src/test/java/org/apache/beam/sdk/bigqueryioperftests/BigQueryIOReadPerformanceIT.java
rename
sdks/java/io/google-cloud-platform/src/{test/java/org/apache/beam/sdk/io/gcp/bigquery
=> main/java/org/apache/beam/sdk/io/gcp/testing}/FakeBigQueryServices.java
(87%)
rename
sdks/java/io/google-cloud-platform/src/{test/java/org/apache/beam/sdk/io/gcp/bigquery
=> main/java/org/apache/beam/sdk/io/gcp/testing}/FakeDatasetService.java (98%)
rename
sdks/java/io/google-cloud-platform/src/{test/java/org/apache/beam/sdk/io/gcp/bigquery
=> main/java/org/apache/beam/sdk/io/gcp/testing}/FakeJobService.java (98%)
rename
sdks/java/io/google-cloud-platform/src/{test/java/org/apache/beam/sdk/io/gcp/bigquery
=> main/java/org/apache/beam/sdk/io/gcp/testing}/TableContainer.java (97%)
create mode 100644 sdks/java/testing/expansion-service/build.gradle
create mode 100644
sdks/java/testing/expansion-service/src/test/java/org/apache/beam/sdk/expansion/TestExpansionService.java
delete mode 100644 sdks/python/apache_beam/coders/avro_coder.py
delete mode 100644 sdks/python/apache_beam/coders/avro_coder_test.py
copy sdks/python/apache_beam/{version.py => coders/avro_record.py} (65%)
create mode 100644
sdks/python/apache_beam/io/external/generate_sequence_test.py
create mode 100644 sdks/python/apache_beam/io/external/xlang_parquetio_test.py
copy sdks/python/apache_beam/testing/{load_tests =>
benchmarks/chicago_taxi}/__init__.py (100%)
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/preprocess.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/process_tfma.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/requirements.txt
create mode 100755
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/run_chicago.sh
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/setup.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/tfdv_analyze_and_validate.py
copy sdks/python/apache_beam/testing/{load_tests =>
benchmarks/chicago_taxi/trainer}/__init__.py (100%)
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/trainer/model.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/trainer/task.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/chicago_taxi/trainer/taxi.py
create mode 100644 sdks/python/apache_beam/testing/extra_assertions.py
create mode 100644 sdks/python/apache_beam/testing/extra_assertions_test.py
create mode 100644
sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
create mode 100755 sdks/python/scripts/run_expansion_services.sh
delete mode 100644 sdks/python/test-suites/dataflow/build.gradle
copy sdks/python/test-suites/dataflow/{py37 => py2}/build.gradle (63%)
create mode 100644 sdks/python/test-suites/direct/py2/build.gradle
create mode 100644 sdks/python/test-suites/portable/py2/build.gradle
create mode 100644 sdks/python/test-suites/tox/py2/build.gradle
create mode 100644 website/src/_posts/2019-07-31-beam-2.14.0.md
create mode 100644
website/src/documentation/transforms/java/aggregation/approximatequantiles.md
create mode 100644
website/src/documentation/transforms/java/aggregation/approximateunique.md
create mode 100644
website/src/documentation/transforms/java/aggregation/cogroupbykey.md
create mode 100644
website/src/documentation/transforms/java/aggregation/combine.md
create mode 100644
website/src/documentation/transforms/java/aggregation/combinewithcontext.md
create mode 100644
website/src/documentation/transforms/java/aggregation/count.md
create mode 100644
website/src/documentation/transforms/java/aggregation/distinct.md
create mode 100644
website/src/documentation/transforms/java/aggregation/groupbykey.md
create mode 100644
website/src/documentation/transforms/java/aggregation/groupintobatches.md
create mode 100644
website/src/documentation/transforms/java/aggregation/latest.md
create mode 100644 website/src/documentation/transforms/java/aggregation/max.md
create mode 100644
website/src/documentation/transforms/java/aggregation/mean.md
create mode 100644 website/src/documentation/transforms/java/aggregation/min.md
create mode 100644
website/src/documentation/transforms/java/aggregation/sample.md
create mode 100644 website/src/documentation/transforms/java/aggregation/sum.md
create mode 100644 website/src/documentation/transforms/java/aggregation/top.md
create mode 100644
website/src/documentation/transforms/java/element-wise/filter.md
create mode 100644
website/src/documentation/transforms/java/element-wise/flatmapelements.md
create mode 100644
website/src/documentation/transforms/java/element-wise/keys.md
create mode 100644
website/src/documentation/transforms/java/element-wise/kvswap.md
create mode 100644
website/src/documentation/transforms/java/element-wise/mapelements.md
create mode 100644
website/src/documentation/transforms/java/element-wise/pardo.md
create mode 100644
website/src/documentation/transforms/java/element-wise/partition.md
create mode 100644
website/src/documentation/transforms/java/element-wise/regex.md
create mode 100644
website/src/documentation/transforms/java/element-wise/reify.md
create mode 100644
website/src/documentation/transforms/java/element-wise/tostring.md
create mode 100644
website/src/documentation/transforms/java/element-wise/values.md
create mode 100644
website/src/documentation/transforms/java/element-wise/withkeys.md
create mode 100644
website/src/documentation/transforms/java/element-wise/withtimestamps.md
create mode 100644 website/src/documentation/transforms/java/index.md
create mode 100644 website/src/documentation/transforms/java/other/create.md
create mode 100644 website/src/documentation/transforms/java/other/flatten.md
create mode 100644 website/src/documentation/transforms/java/other/passert.md
create mode 100644 website/src/documentation/transforms/java/other/view.md
create mode 100644 website/src/documentation/transforms/java/other/window.md
