[beam] branch master updated: [BEAM-14503] Add support for Flink 1.15 (#17739)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 24decb50cf3 [BEAM-14503] Add support for Flink 1.15 (#17739)
24decb50cf3 is described below
commit 24decb50cf3e14e8cce4dd9d82b8963bf57a1805
Author: Julien Tournay
AuthorDate: Wed May 25 19:59:04 2022 +0200
[BEAM-14503] Add support for Flink 1.15 (#17739)
* Flink 1.15 runner
* Ignore Flink 1.11
---
gradle.properties | 2 +-
.../streaming/ProcessingTimeCallbackCompat.java| 22 +++
.../beam/runners/flink/MiniClusterCompat.java | 29 ++
runners/flink/1.15/build.gradle| 34 +
.../flink/1.15/job-server-container/build.gradle | 26 +
runners/flink/1.15/job-server/build.gradle | 31 +++
.../streaming/ProcessingTimeCallbackCompat.java| 22 +++
.../beam/runners/flink/MiniClusterCompat.java | 30 +++
runners/flink/flink_runner.gradle | 44 --
.../flink/FlinkStreamingTransformTranslators.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 3 +-
.../streaming/io/UnboundedSourceWrapper.java | 4 +-
.../flink/FlinkRequiresStableInputTest.java| 2 +-
.../beam/runners/flink/FlinkSavepointTest.java | 2 +-
.../python/apache_beam/options/pipeline_options.py | 2 +-
settings.gradle.kts| 4 ++
16 files changed, 240 insertions(+), 21 deletions(-)
diff --git a/gradle.properties b/gradle.properties
index f880631cd68..d1528eb2cd0 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -37,5 +37,5 @@ javaVersion=1.8
docker_image_default_repo_root=apache
docker_image_default_repo_prefix=beam_
-flink_versions=1.12,1.13,1.14
+flink_versions=1.12,1.13,1.14,1.15
diff --git
a/runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ProcessingTimeCallbackCompat.java
b/runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ProcessingTimeCallbackCompat.java
new file mode 100644
index 000..a494fec01dd
--- /dev/null
+++
b/runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ProcessingTimeCallbackCompat.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.translation.wrappers.streaming;
+
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+
+public interface ProcessingTimeCallbackCompat extends ProcessingTimeCallback {}
diff --git
a/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/MiniClusterCompat.java
b/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/MiniClusterCompat.java
new file mode 100644
index 000..1bbcd0159b1
--- /dev/null
+++
b/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/MiniClusterCompat.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.minicluster.MiniCluster;
+
+public cl
[beam] branch master updated: Nit: correct description for precommit cron jobs.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 973752b07ad Nit: correct description for precommit cron jobs.
new f7be5ae5a47 Merge pull request #17276 from ibzib/job-description
973752b07ad is described below
commit 973752b07ad95847375b1731289d08c73ed20047
Author: Kyle Weaver
AuthorDate: Mon Apr 4 16:39:10 2022 -0700
Nit: correct description for precommit cron jobs.
The default setting runs each job every 6 hours.
---
.test-infra/jenkins/PrecommitJobBuilder.groovy | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.test-infra/jenkins/PrecommitJobBuilder.groovy
b/.test-infra/jenkins/PrecommitJobBuilder.groovy
index df2ef22d193..5a0df8eae89 100644
--- a/.test-infra/jenkins/PrecommitJobBuilder.groovy
+++ b/.test-infra/jenkins/PrecommitJobBuilder.groovy
@@ -57,11 +57,11 @@ class PrecommitJobBuilder {
definePhraseJob additionalCustomization
}
- /** Create a pre-commit job which runs on a daily schedule. */
+ /** Create a pre-commit job which runs on a regular schedule. */
private void defineCronJob(Closure additionalCustomization) {
def job = createBaseJob 'Cron'
job.with {
- description buildDescription('on a daily schedule.')
+ description buildDescription('on a regular schedule.')
commonJobProperties.setAutoJob delegate
}
job.with additionalCustomization
[beam] branch master updated (372978533e7 -> 48bdce3d764)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 372978533e7 fixes golint and deprecated issues in recent Go SDK import (#17304) new 96f775fe4d1 [BEAM-14262] Update plugins for Dockerized Jenkins. new f1c4003301a Add ansicolor and ws-cleanup plugins. new 48bdce3d764 Merge pull request #17303 from ibzib/update-plugins The 35202 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .test-infra/dockerized-jenkins/plugins.txt | 121 + 1 file changed, 55 insertions(+), 66 deletions(-)
[beam] branch master updated: Fix dependency issue causing failures
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new ad93728d609 Fix dependency issue causing failures new 0c2cefffd5f Merge pull request #17283 from damccorm/users/damccorm/reportGenerator ad93728d609 is described below commit ad93728d6099dcf641f32247eccc80e955d05e06 Author: Danny McCormick AuthorDate: Tue Apr 5 13:07:11 2022 -0400 Fix dependency issue causing failures --- .test-infra/jenkins/metrics_report/requirements.txt | 4 1 file changed, 4 insertions(+) diff --git a/.test-infra/jenkins/metrics_report/requirements.txt b/.test-infra/jenkins/metrics_report/requirements.txt index 8a9efe73b32..4716e65ddb9 100644 --- a/.test-infra/jenkins/metrics_report/requirements.txt +++ b/.test-infra/jenkins/metrics_report/requirements.txt @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +# Markupsafe breaking change broke Jinja and some other libs +# Pinning it to a version which works even though we are not using explicitly +# https://github.com/aws/aws-sam-cli/issues/3661 +markupsafe==2.0.1 influxdb==5.3.0 Jinja2==2.11.3 prettytable==0.7.2
[beam] branch master updated: [BEAM-9649] Add region option to Mongo Dataflow test.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 17a41790dc2 [BEAM-9649] Add region option to Mongo Dataflow test.
new 29dfe41037e Merge pull request #17284 from ibzib/BEAM-9649
17a41790dc2 is described below
commit 17a41790dc2a41cb78ef886d4177204c7841a469
Author: Kyle Weaver
AuthorDate: Tue Apr 5 10:43:51 2022 -0700
[BEAM-9649] Add region option to Mongo Dataflow test.
---
.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy | 1 +
1 file changed, 1 insertion(+)
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
index 78c285e67a2..7d4d711a8e9 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
@@ -39,6 +39,7 @@ job(jobName) {
Map pipelineOptions = [
temp_location: 'gs://temp-storage-for-perf-tests/loadtests',
project : 'apache-beam-testing',
+region : 'us-central1',
mongo_uri: "mongodb://\$${mongoHostName}:27017",
num_documents: '100',
batch_size : '1',
[beam] branch master updated: Add self-descriptive message for expected errors.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e4f37d22812 Add self-descriptive message for expected errors.
new 2583cd466f7 Merge pull request #17254 from ibzib/red-herring
e4f37d22812 is described below
commit e4f37d22812c416b20ff9df182a8ef0d7945edc9
Author: Kyle Weaver
AuthorDate: Fri Apr 1 16:42:56 2022 -0700
Add self-descriptive message for expected errors.
Ideally we would not log these in the first place, but this is an easy hack.
---
.../apache_beam/runners/portability/fn_api_runner/fn_runner_test.py | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py
b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py
index 12a784b8410..06b649591cd 100644
---
a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py
+++
b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py
@@ -889,7 +889,8 @@ class FnApiRunnerTest(unittest.TestCase):
with self.create_pipeline() as p:
def raise_error(x):
- raise RuntimeError('x')
+ raise RuntimeError(
+ 'This error is expected and does not indicate a test failure.')
# pylint: disable=expression-not-assigned
(
@@ -912,7 +913,8 @@ class FnApiRunnerTest(unittest.TestCase):
return third(x)
def third(x):
- raise ValueError('x')
+ raise ValueError(
+ 'This error is expected and does not indicate a test failure.')
try:
with self.create_pipeline() as p:
[beam] branch master updated: [BEAM-14225] load balance jenkins jobs
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d856f28 [BEAM-14225] load balance jenkins jobs
new 3d4454c Merge pull request #17238 from ibzib/BEAM-14225
d856f28 is described below
commit d856f280e10d744a9e61a9fb2c5422470d638103
Author: Kyle Weaver
AuthorDate: Thu Mar 31 16:49:15 2022 -0700
[BEAM-14225] load balance jenkins jobs
---
.test-infra/jenkins/CommonJobProperties.groovy | 2 +-
.test-infra/jenkins/CronJobBuilder.groovy| 2 +-
.test-infra/jenkins/PostcommitJobBuilder.groovy | 2 +-
.test-infra/jenkins/job_00_seed.groovy | 4 ++--
.test-infra/jenkins/job_Dependency_Check.groovy | 3 +--
.test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy | 2 +-
.test-infra/jenkins/job_Metrics_Report.groovy| 3 +--
.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Java.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy | 4 ++--
.test-infra/jenkins/job_PerformanceTests_HadoopFormat.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy| 2 +-
.test-infra/jenkins/job_PerformanceTests_JDBC.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_Python.groovy | 2 +-
.test-infra/jenkins/job_PerformanceTests_SQLIO_Java.groovy | 2 +-
.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy | 2 +-
.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy | 2 +-
.test-infra/jenkins/job_Publish_Docker_Snapshots.groovy | 2 +-
.test-infra/jenkins/job_Release_NightlySnapshot.groovy | 2 +-
.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy| 2 +-
21 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/.test-infra/jenkins/CommonJobProperties.groovy
b/.test-infra/jenkins/CommonJobProperties.groovy
index b0dc45f..a715e0e 100644
--- a/.test-infra/jenkins/CommonJobProperties.groovy
+++ b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -219,7 +219,7 @@ class CommonJobProperties {
// Sets common config for jobs which run on a schedule; optionally on push
static void setAutoJob(context,
- String buildSchedule = '0 */6 * * *',
+ String buildSchedule = 'H H/6 * * *',
notifyAddress = 'bui...@beam.apache.org',
triggerOnCommit = false,
emailIndividuals = false) {
diff --git a/.test-infra/jenkins/CronJobBuilder.groovy
b/.test-infra/jenkins/CronJobBuilder.groovy
index fc4be04..100cc55 100644
--- a/.test-infra/jenkins/CronJobBuilder.groovy
+++ b/.test-infra/jenkins/CronJobBuilder.groovy
@@ -37,7 +37,7 @@ class CronJobBuilder {
* @param cronPattern Defines when the job should be fired. Default: "every
6th hour".
* @param jobDefinition Closure for the job.
*/
- static void cronJob(nameBase, cronPattern = '0 */6 * * *', scope,
jobDefinition = {}) {
+ static void cronJob(nameBase, cronPattern = 'H H/6 * * *', scope,
jobDefinition = {}) {
CronJobBuilder builder = new CronJobBuilder(scope, jobDefinition)
builder.defineAutoPostCommitJob(nameBase, cronPattern)
}
diff --git a/.test-infra/jenkins/PostcommitJobBuilder.groovy
b/.test-infra/jenkins/PostcommitJobBuilder.groovy
index ca1a70c..70d3759 100644
--- a/.test-infra/jenkins/PostcommitJobBuilder.groovy
+++ b/.test-infra/jenkins/PostcommitJobBuilder.groovy
@@ -58,7 +58,7 @@ class PostcommitJobBuilder {
void defineAutoPostCommitJob(name) {
def autoBuilds = scope.job(name) {
- commonJobProperties.setAutoJob delegate, '0 */6 * * *',
'bui...@beam.apache.org', true, true
+ commonJobProperties.setAutoJob delegate, 'H H/6 * * *',
'bui...@beam.apache.org', true, true
}
autoBuilds.with(jobDefinition)
diff --git a/.test-infra/jenkins/job_00_seed.groovy
b/.test-infra/jenkins/job_00_seed.groovy
index 820f3c4..2ffac70 100644
--- a/.test-infra/jenkins/job_00_seed.groovy
+++ b/.test-infra/jenkins/job_00_seed.groovy
@@ -73,8 +73,8 @@ job('beam_SeedJob') {
}
triggers {
-// Run once per day
-cron('H */6 * * *')
+// Run every six hours
+cron('H H/6 * * *')
githubPullRequest {
admins(['asfbot'])
diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy
b/.test-infra/jenkins/job_Dependency_Check.groovy
index f97e0bb..b34723e 100644
--- a/.test-infra/jenkins/job_Dependency_Check.groovy
+++ b/.test-infra/jenkins/job_Dependency_Check.groovy
@@ -33,10 +33,9 @@ job('beam_Dependency_Check') {
false
)
- // This is a job that r
[beam] branch master updated (9d706e3 -> 8af69d7)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 9d706e3 [BEAM-10976] Bundle finalization: E2E support (#17045) new 973282d [BEAM-14094]Fix null pointer exception in HllCountInitFn new 58a8658 [BEAM-14094]Fix null pointer exception in HllCountInitFn new 8af69d7 Merge pull request #17081 from brachi-wernick/BEAM-14094-fix-HllCountInitFn-npe The 35128 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../beam/sdk/extensions/zetasketch/HllCountInitFn.java | 16 1 file changed, 12 insertions(+), 4 deletions(-)
[beam] branch master updated (6cddf29 -> 141fb79)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 6cddf29 Minor: Make IOTypeHints a real NamedTuple (#17174)
new 12530a3 [BEAM-14139] Drop support for Flink 1.11.
new 868a773 [BEAM-14139] Remove obsolete reference to Flink 1.11.
new e63bc2d [BEAM-14139] Update list of supported Flink versions.
new 11d6aad [BEAM-14139] Update CHANGES.md
new 141fb79 Merge pull request #17145 from ibzib/no-flink11
The 35071 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 31
gradle.properties | 2 +-
runners/flink/1.11/build.gradle| 33
.../flink/1.11/job-server-container/build.gradle | 26 --
runners/flink/1.11/job-server/build.gradle | 31
.../streaming/AbstractStreamOperatorCompat.java| 61 --
.../beam/runners/flink/FlinkRunnerTestCompat.java | 47 ---
.../runners/flink/SourceTransformationCompat.java | 28 ---
runners/flink/1.12/build.gradle| 4 +-
.../beam/runners/flink/RemoteMiniClusterImpl.java | 0
.../runners/flink/metrics/MetricGroupWrapper.java | 0
.../runners/flink/streaming/StreamSources.java | 22 +-
runners/flink/1.13/build.gradle| 4 +-
.../runners/flink/streaming/StreamSources.java | 92 --
runners/flink/1.14/build.gradle| 4 +-
.../runners/flink/streaming/StreamSources.java | 22 +-
.../apache/beam/runners/flink/FlinkRunnerTest.java | 41 ++
.../beam/runners/flink/FlinkSavepointTest.java | 4 -
.../FlinkStreamingTransformTranslatorsTest.java| 13 +--
.../python/apache_beam/options/pipeline_options.py | 2 +-
settings.gradle.kts| 4 -
21 files changed, 73 insertions(+), 398 deletions(-)
delete mode 100644 runners/flink/1.11/build.gradle
delete mode 100644 runners/flink/1.11/job-server-container/build.gradle
delete mode 100644 runners/flink/1.11/job-server/build.gradle
delete mode 100644
runners/flink/1.11/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/AbstractStreamOperatorCompat.java
delete mode 100644
runners/flink/1.11/src/test/java/org/apache/beam/runners/flink/FlinkRunnerTestCompat.java
delete mode 100644
runners/flink/1.11/src/test/java/org/apache/beam/runners/flink/SourceTransformationCompat.java
rename runners/flink/{1.11 =>
1.12}/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java
(100%)
rename runners/flink/{1.11 =>
1.12}/src/test/java/org/apache/beam/runners/flink/metrics/MetricGroupWrapper.java
(100%)
delete mode 100644
runners/flink/1.13/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java
[beam] branch master updated (b9846fb -> dae7215)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from b9846fb Add ability to handle streaming input to AvroSchemaIOProvider (#17126) add dae7215 [BEAM-12898] Flink Load Tests failure- UncheckedExecutionException - leaking vms (#17046) No new revisions were added by this update. Summary of changes: .test-infra/dataproc/flink_cluster.sh | 7 --- .test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy | 4 ++-- .test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy | 4 ++-- .test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy | 4 ++-- .test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 4 ++-- .test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy | 8 .test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy | 4 ++-- .test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy | 4 ++-- 8 files changed, 20 insertions(+), 19 deletions(-)
[beam] branch master updated: [BEAM-14124] Add display data to BQ storage reads.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ce71caf [BEAM-14124] Add display data to BQ storage reads.
new 57c8647 Merge pull request #17115 from ibzib/bq-display-data
ce71caf is described below
commit ce71cafc4801474ba7791a194200e4599b0b9e33
Author: Kyle Weaver
AuthorDate: Thu Mar 17 13:12:35 2022 -0700
[BEAM-14124] Add display data to BQ storage reads.
Add display data for "Selected fields" and "Projection pushdown
applied". I also want to add one for "Number of fields pushed down", but
that will be a little more involved so I'll do it in a separate PR.
---
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 26 ++-
.../gcp/bigquery/BigQueryStorageTableSource.java | 52 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 5 +++
3 files changed, 70 insertions(+), 13 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 9786d90..c510db7 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -590,6 +590,7 @@ public class BigQueryIO {
.setMethod(TypedRead.Method.DEFAULT)
.setUseAvroLogicalTypes(false)
.setFormat(DataFormat.AVRO)
+.setProjectionPushdownApplied(false)
.build();
}
@@ -805,6 +806,8 @@ public class BigQueryIO {
abstract Builder setFromBeamRowFn(FromBeamRowFunction fromRowFn);
abstract Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes);
+
+ abstract Builder setProjectionPushdownApplied(boolean
projectionPushdownApplied);
}
abstract @Nullable ValueProvider getJsonTableRef();
@@ -853,6 +856,8 @@ public class BigQueryIO {
abstract Boolean getUseAvroLogicalTypes();
+abstract boolean getProjectionPushdownApplied();
+
/**
* An enumeration type for the priority of a query.
*
@@ -1229,7 +1234,8 @@ public class BigQueryIO {
getRowRestriction(),
getParseFn(),
outputCoder,
-getBigQueryServices(;
+getBigQueryServices(),
+getProjectionPushdownApplied(;
}
checkArgument(
@@ -1430,6 +1436,10 @@ public class BigQueryIO {
DisplayData.item("table",
BigQueryHelpers.displayTable(getTableProvider()))
.withLabel("Table"))
.addIfNotNull(DisplayData.item("query",
getQuery()).withLabel("Query"))
+ .addIfNotDefault(
+ DisplayData.item("projectionPushdownApplied",
getProjectionPushdownApplied())
+ .withLabel("Projection Pushdown Applied"),
+ false)
.addIfNotNull(
DisplayData.item("flattenResults", getFlattenResults())
.withLabel("Flatten Query Results"))
@@ -1438,6 +1448,13 @@ public class BigQueryIO {
.withLabel("Use Legacy SQL Dialect"))
.addIfNotDefault(
DisplayData.item("validation",
getValidate()).withLabel("Validation Enabled"), true);
+
+ ValueProvider> selectedFieldsProvider = getSelectedFields();
+ if (selectedFieldsProvider != null &&
selectedFieldsProvider.isAccessible()) {
+builder.add(
+DisplayData.item("selectedFields", String.join(", ",
selectedFieldsProvider.get()))
+.withLabel("Selected Fields"));
+ }
}
/** Ensures that methods of the from() / fromQuery() family are called at
most once. */
@@ -1623,6 +1640,11 @@ public class BigQueryIO {
return toBuilder().setUseAvroLogicalTypes(true).build();
}
+@VisibleForTesting
+TypedRead withProjectionPushdownApplied() {
+ return toBuilder().setProjectionPushdownApplied(true).build();
+}
+
@Override
public boolean supportsProjectionPushdown() {
// We can't do projection pushdown when a query is set. The query may
project certain fields
@@ -1643,7 +1665,7 @@ public class BigQueryIO {
outputFields.keySet());
ImmutableList fields =
ImmutableList.copyOf(fieldAccessDescriptor.fieldNamesAccessed());
- return withSelectedFields(fields);
+ return withSelectedFields(fields).withProjectionPushdownApplied();
}
}
diff --git
a/sdks/java/io/google-clou
[beam] branch master updated: Don't print in task configuration.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a8f405b Don't print in task configuration.
new 83131fb Merge pull request #17139 from ibzib/no-print
a8f405b is described below
commit a8f405bc382a2f5fece186f806a81f0108491578
Author: Kyle Weaver
AuthorDate: Mon Mar 21 09:30:43 2022 -0700
Don't print in task configuration.
This will print every time any Gradle command is run, which is way too
spammy.
---
playground/frontend/build.gradle | 1 -
1 file changed, 1 deletion(-)
diff --git a/playground/frontend/build.gradle b/playground/frontend/build.gradle
index 5a4eeef..48a8cf3 100644
--- a/playground/frontend/build.gradle
+++ b/playground/frontend/build.gradle
@@ -141,7 +141,6 @@ docker {
project.docker_image_default_repo_root)
files "./build/"
tags containerImageTags()
- println("container:" + name)
buildArgs(['FLUTTER_VERSION':
project.rootProject.hasProperty(["flutter-version"]) ?
project.rootProject["flutter-version"] :
"2.5.2-stable" ])
[beam] branch master updated: [BEAM-14123] Fix typo in hdfsIntegrationTest task name.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d699086 [BEAM-14123] Fix typo in hdfsIntegrationTest task name.
new b36a557 Merge pull request #17114 from ibzib/hdfs-typo
d699086 is described below
commit d699086b6d1a20210eae0cbb4867bddd95dd0aa1
Author: Kyle Weaver
AuthorDate: Thu Mar 17 16:09:14 2022 -0700
[BEAM-14123] Fix typo in hdfsIntegrationTest task name.
---
sdks/python/test-suites/direct/common.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sdks/python/test-suites/direct/common.gradle
b/sdks/python/test-suites/direct/common.gradle
index 4f1eac7..15df969 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -177,7 +177,7 @@ tasks.register("directRunnerIT") {
}
}
-tasks.register("hdfsIntegrationTest ") {
+tasks.register("hdfsIntegrationTest") {
doLast {
exec {
executable 'sh'
[beam] branch master updated (0510cec -> 3e472e2)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 0510cec [BEAM-14038] Auto-startup for Python expansion service. (#17035) new b88bed5 [BEAM-12976] Test a whole pipeline using projection pushdown in BQ IO. new eb5fbd3 [BEAM-12976] Enable projection pushdown for Java pipelines on Dataflow, Flink, and Spark. new 3e472e2 Merge pull request #17107 from ibzib/enable-optimizer The 34922 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGES.md | 1 + .../org/apache/beam/runners/flink/FlinkRunner.java | 6 +++ .../beam/runners/dataflow/DataflowRunner.java | 8 ++- .../org/apache/beam/runners/spark/SparkRunner.java | 6 +++ .../io/gcp/bigquery/BigQueryIOStorageReadIT.java | 59 +- 5 files changed, 78 insertions(+), 2 deletions(-)
[beam] branch master updated (bdc3162 -> 4136fb4)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from bdc3162 [BEAM-13925] Turn pr bot on for go prs (#16984) add 4136fb4 [BEAM-13964] Bump kotlin to 1.6.x (#16882) No new revisions were added by this update. Summary of changes: examples/kotlin/build.gradle | 4 ++-- .../apache/beam/examples/kotlin/common/WriteOneFilePerWindow.kt | 5 ++--- .../org/apache/beam/examples/kotlin/cookbook/DistinctExample.kt | 2 -- .../java/org/apache/beam/examples/kotlin/snippets/Snippets.kt | 8 4 files changed, 8 insertions(+), 11 deletions(-)
[beam] branch master updated (b0899a4 -> 0fe57b1)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from b0899a4 Fix BoundedQueueExecutor and StreamingDataflowWorker to
actually limit memory from windmill (#16901)
add 0fe57b1 [BEAM-1857] Add Neo4jIO (#15916)
No new revisions were added by this update.
Summary of changes:
build.gradle.kts |1 +
{.test-infra => sdks/java/io/neo4j}/OWNERS |3 +-
.../io/{contextualtextio => neo4j}/build.gradle| 22 +-
.../java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java | 1221
.../apache/beam/sdk/io/neo4j}/package-info.java|4 +-
.../org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java| 275 +
.../org/apache/beam/sdk/io/neo4j/Neo4jIOTest.java | 117 ++
.../apache/beam/sdk/io/neo4j/Neo4jTestUtil.java| 70 ++
settings.gradle.kts|1 +
9 files changed, 1699 insertions(+), 15 deletions(-)
copy {.test-infra => sdks/java/io/neo4j}/OWNERS (83%)
copy sdks/java/io/{contextualtextio => neo4j}/build.gradle (63%)
create mode 100644
sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java
copy
sdks/java/io/{amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis
=> neo4j/src/main/java/org/apache/beam/sdk/io/neo4j}/package-info.java (90%)
create mode 100644
sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java
create mode 100644
sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOTest.java
create mode 100644
sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jTestUtil.java
[beam] branch master updated (43cd356 -> 60ea50f)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 43cd356 Palo Alto case study (#16915) new e303fad [BEAM-13796] projection pushdown in BQ IO new faf8c0f [BEAM-13796] Move test to ReadTest class and correct javadoc for QueryTest. new 185c2e3 [BEAM-13796] Pushdown is not supported on TypedRead#fromQuery. new 60ea50f Merge pull request #16720 from ibzib/BEAM-13796 The 34715 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 30 ++- .../gcp/bigquery/BigQueryIOStorageQueryTest.java | 5 +- .../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 60 ++ 3 files changed, 92 insertions(+), 3 deletions(-)
[beam] branch master updated (95542f6 -> 4e5a2b4)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 95542f6 Merge pull request #16736: Update Java LTS roadmap info on
website for Java 17
new 26c4056 [BEAM-13106] Support Flink 1.14.
new d38c117 [BEAM-13106] Reuse executor instead of shutting it down
mid-test.
new 47486e4 [BEAM-13106] Prevent infinite wait in Flink savepoint test.
new 99a0907 [BEAM-13106] A couple additional fixes to FlinkSavepointTest.
new 4e5a2b4 Merge pull request #16679 from ibzib/BEAM-13106
The 34681 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 2 +
gradle.properties | 2 +-
.../streaming/AbstractStreamOperatorCompat.java| 31 ++
.../runners/flink/metrics/MetricGroupWrapper.java} | 15 -
.../runners/flink/streaming/StreamSources.java | 27 +++-
.../streaming/AbstractStreamOperatorCompat.java| 31 ++
.../runners/flink/streaming/StreamSources.java | 27 +++-
.../runners/flink/streaming/StreamSources.java | 27 +++-
runners/flink/{1.13 => 1.14}/build.gradle | 9 +--
.../job-server-container/build.gradle | 0
.../flink/{1.11 => 1.14}/job-server/build.gradle | 2 +-
.../streaming/AbstractStreamOperatorCompat.java| 28 +
.../beam/runners/flink/RemoteMiniClusterImpl.java} | 20 --
.../runners/flink/metrics/MetricGroupWrapper.java} | 15 -
.../runners/flink/streaming/StreamSources.java | 15 +++--
runners/flink/flink_runner.gradle | 13 +++-
.../flink/FlinkPipelineExecutionEnvironment.java | 7 ++-
.../wrappers/streaming/DoFnOperator.java | 71 +-
.../streaming/ExecutableStageDoFnOperator.java | 8 +--
.../beam/runners/flink/FlinkSavepointTest.java | 70 +
.../flink/metrics/FlinkMetricContainerTest.java| 3 +-
.../flink/streaming/BoundedSourceRestoreTest.java | 28 +
.../wrappers/streaming/DoFnOperatorTest.java | 4 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 2 +-
.../streaming/io/UnboundedSourceWrapperTest.java | 43 -
.../python/apache_beam/options/pipeline_options.py | 2 +-
settings.gradle.kts| 4 ++
27 files changed, 343 insertions(+), 163 deletions(-)
copy
runners/flink/{src/main/java/org/apache/beam/runners/flink/metrics/package-info.java
=>
1.11/src/test/java/org/apache/beam/runners/flink/metrics/MetricGroupWrapper.java}
(67%)
copy runners/flink/{1.13 => 1.14}/build.gradle (84%)
copy runners/flink/{1.13 => 1.14}/job-server-container/build.gradle (100%)
copy runners/flink/{1.11 => 1.14}/job-server/build.gradle (95%)
copy runners/flink/{1.12 =>
1.14}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/AbstractStreamOperatorCompat.java
(75%)
copy
runners/flink/{src/test/java/org/apache/beam/runners/flink/RemoteMiniCluster.java
=>
1.14/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java}
(66%)
copy
runners/flink/{src/main/java/org/apache/beam/runners/flink/metrics/package-info.java
=>
1.14/src/test/java/org/apache/beam/runners/flink/metrics/MetricGroupWrapper.java}
(63%)
copy runners/flink/{1.12 =>
1.14}/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java
(84%)
[beam] branch master updated: Update Dataflow Python dev container images.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 0881676 Update Dataflow Python dev container images. new d37d030 Merge pull request #16782 from ibzib/py0208 0881676 is described below commit 0881676b11ea52cfe69b699b45f9519d33f14995 Author: Kyle Weaver AuthorDate: Tue Feb 8 11:21:46 2022 -0800 Update Dataflow Python dev container images. --- sdks/python/apache_beam/runners/dataflow/internal/names.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index be6a2c0..e99048b 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -36,10 +36,10 @@ SERIALIZED_SOURCE_KEY = 'serialized_source' # Update this version to the next version whenever there is a change that will # require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20220126' +BEAM_CONTAINER_VERSION = 'beam-master-20220208' # Update this version to the next version whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20220126' +BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20220208' DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'gcr.io/cloud-dataflow/v1beta3'
[beam] branch master updated: [BEAM-12976] Log projection pushdown optimizations.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 87f10bb [BEAM-12976] Log projection pushdown optimizations.
new 6595861 Merge pull request #16724 from ibzib/log-opt
87f10bb is described below
commit 87f10bbe7e342730c1113a294387d2e22150755e
Author: Kyle Weaver
AuthorDate: Thu Feb 3 16:13:18 2022 -0800
[BEAM-12976] Log projection pushdown optimizations.
---
.../core/construction/graph/ProjectionPushdownOptimizer.java | 10 ++
1 file changed, 10 insertions(+)
diff --git
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java
index 9abb17f..4b6669a 100644
---
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java
+++
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java
@@ -37,9 +37,12 @@ import org.apache.beam.sdk.values.TaggedPValue;
import org.apache.beam.sdk.values.TupleTag;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** See {@link #optimize(Pipeline)}. */
public class ProjectionPushdownOptimizer {
+ private static final Logger LOG =
LoggerFactory.getLogger(ProjectionPushdownOptimizer.class);
/**
* Performs all known projection pushdown optimizations in-place on a
Pipeline.
@@ -78,6 +81,13 @@ public class ProjectionPushdownOptimizer {
// fields.
for (Entry>, Map,
FieldAccessDescriptor>>
entry : taggedFieldAccess.entrySet()) {
+ for (Entry, FieldAccessDescriptor> outputFields :
entry.getValue().entrySet()) {
+LOG.info(
+"Optimizing transform {}: output {} will contain reduced field set
{}",
+entry.getKey(),
+outputFields.getKey(),
+outputFields.getValue().fieldNamesAccessed());
+ }
PTransformMatcher matcher = application -> application.getTransform() ==
entry.getKey();
PushdownOverrideFactory overrideFactory =
new PushdownOverrideFactory<>(entry.getValue());
[beam] branch master updated: Make num-stages counter into an internal counter.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7d3af58 Make num-stages counter into an internal counter.
new c7a7eec Merge pull request #16638 from robertwb/internal-counters
7d3af58 is described below
commit 7d3af58d08c2b4bbe46b76ed76621fc7ea3303f6
Author: Robert Bradshaw
AuthorDate: Thu Jan 27 14:52:05 2022 -0800
Make num-stages counter into an internal counter.
This unbreaks tests that (arguably to brittly) reject extra counters.
---
sdks/python/apache_beam/dataframe/transforms_test.py | 2 +-
.../apache_beam/runners/portability/fn_api_runner/fn_runner.py | 7 +--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/sdks/python/apache_beam/dataframe/transforms_test.py
b/sdks/python/apache_beam/dataframe/transforms_test.py
index c4aa6869..cd76f07 100644
--- a/sdks/python/apache_beam/dataframe/transforms_test.py
+++ b/sdks/python/apache_beam/dataframe/transforms_test.py
@@ -353,7 +353,7 @@ class TransformTest(unittest.TestCase):
class FusionTest(unittest.TestCase):
@staticmethod
def fused_stages(p):
-return p.result.metrics().query(
+return p.result.monitoring_metrics().query(
metrics.MetricsFilter().with_name(
fn_runner.FnApiRunner.NUM_FUSED_STAGES_COUNTER)
)['counters'][0].result
diff --git
a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
index 9b08427..9a3e100 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
@@ -358,8 +358,11 @@ class FnApiRunner(runner.PipelineRunner):
stage_context.components.environments, self._provision_info)
pipeline_metrics = MetricsContainer('')
pipeline_metrics.get_counter(
-MetricName(str(type(self)),
- self.NUM_FUSED_STAGES_COUNTER)).update(len(stages))
+MetricName(
+str(type(self)),
+self.NUM_FUSED_STAGES_COUNTER,
+urn='internal:' + self.NUM_FUSED_STAGES_COUNTER)).update(
+len(stages))
monitoring_infos_by_stage = {}
runner_execution_context = execution.FnApiRunnerExecutionContext(
[beam] branch master updated (a4fcd93 -> 51d7e63)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from a4fcd93 Merge pull request #16505 from [BEAM-13527] [Playground] Pipeline options dialog new bdee4d0 [BEAM-13751] Don't block on gcloud when attempting to get default GCP region. new 03a556a [BEAM-13751] Parameterize wait timeout so test doesn't waste 2s. new 07e903b [BEAM-13751] Add comment explaining sleep. new 51d7e63 Merge pull request #16639 from ibzib/BEAM-13751 The 34370 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../dataflow/options/DefaultGcpRegionFactory.java | 21 +++-- .../options/DataflowPipelineOptionsTest.java | 92 -- 2 files changed, 101 insertions(+), 12 deletions(-)
[beam] branch master updated (a589b10 -> cf79c26)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from a589b10 Merge pull request #16604: [BEAM-13714] Get rid of unnecessary logs for BigQuery streaming writes add cf79c26 [BEAM-12976] Implement Java projection pushdown optimizer. (#16513) No new revisions were added by this update. Summary of changes: .../graph/PCollectionOutputTagVisitor.java | 91 + .../graph/ProjectionProducerVisitor.java | 6 + .../graph/ProjectionPushdownOptimizer.java | 136 .../graph/ProjectionPushdownOptimizerTest.java | 366 + 4 files changed, 599 insertions(+) create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PCollectionOutputTagVisitor.java create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizerTest.java
[beam] branch master updated: [BEAM-13577] Beam Select's uniquifyNames function loses nullability of Complex types while inferring schema
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 586739d [BEAM-13577] Beam Select's uniquifyNames function loses
nullability of Complex types while inferring schema
new ab6374e Merge pull request #16380 from
talatuyarer/BEAM-13577-select-uniquifyNames-nullablity
586739d is described below
commit 586739d854b5f6503f03469bee278de0bbd72ce9
Author: Talat Uyarer
AuthorDate: Thu Jan 13 21:21:04 2022 -0800
[BEAM-13577] Beam Select's uniquifyNames function loses nullability of
Complex types while inferring schema
---
.../apache/beam/sdk/schemas/transforms/Select.java | 10 ++--
.../sdk/extensions/sql/BeamSqlDslArrayTest.java| 53 +++---
.../rel/BeamCoGBKJoinRelBoundedVsBoundedTest.java | 32 -
3 files changed, 84 insertions(+), 11 deletions(-)
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java
index 597aa62..763b7cf 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java
@@ -226,12 +226,16 @@ public class Select {
.withNullable(fieldType.getNullable())
.withMetadata(fieldType.getAllMetadata());
case ARRAY:
-return
FieldType.array(uniquifyNames(fieldType.getCollectionElementType()));
+return
FieldType.array(uniquifyNames(fieldType.getCollectionElementType()))
+.withNullable(fieldType.getNullable());
case ITERABLE:
-return
FieldType.iterable(uniquifyNames(fieldType.getCollectionElementType()));
+return
FieldType.iterable(uniquifyNames(fieldType.getCollectionElementType()))
+.withNullable(fieldType.getNullable());
case MAP:
return FieldType.map(
-uniquifyNames(fieldType.getMapKeyType()),
uniquifyNames(fieldType.getMapValueType()));
+uniquifyNames(fieldType.getMapKeyType()),
+uniquifyNames(fieldType.getMapValueType()))
+.withNullable(fieldType.getNullable());
default:
return fieldType;
}
diff --git
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslArrayTest.java
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslArrayTest.java
index f072467..65d6d72 100644
---
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslArrayTest.java
+++
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslArrayTest.java
@@ -19,6 +19,7 @@ package org.apache.beam.sdk.extensions.sql;
import java.util.Arrays;
import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.Schema.FieldType;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
@@ -37,7 +38,8 @@ public class BeamSqlDslArrayTest {
private static final Schema INPUT_SCHEMA =
Schema.builder()
.addInt32Field("f_int")
- .addArrayField("f_stringArr", Schema.FieldType.STRING)
+ .addStringField("f_string")
+ .addNullableField("f_stringArr",
FieldType.array(Schema.FieldType.STRING))
.build();
@Rule public final TestPipeline pipeline = TestPipeline.create();
@@ -91,6 +93,41 @@ public class BeamSqlDslArrayTest {
}
@Test
+ public void testProjectArrayFieldWithCoGBKJoin() {
+PCollection input = pCollectionOf2Elements();
+
+Schema resultType =
+Schema.builder()
+.addNullableField("f_stringArr",
FieldType.array(Schema.FieldType.STRING))
+.build();
+
+// When we use longer enough IN clause, Calcite calculate physical plan
with BeamCoGBKJoin.
+// This SQL push Calcite to use BeamCoGBKJoin for simple select statement.
+PCollection result =
+input.apply(
+"sqlQuery",
+SqlTransform.query(
+"SELECT f_stringArr FROM PCOLLECTION WHERE f_string IN ('A',
'B', "
++
"'ABCAABAAAGAG','ABCAABAAAGCB','ABCAABAAAGCJ','ABCAABAAAGEB','ABCAABAAAGEK',"
++
"'ABCAABAAAGFB','ABCAABAAAGFG','ABCAABABAGBJ','ABCAABABBKIF','ABCAABABCAIK',"
++
"'ABCAABAEJAAF','ABCAABAEJAED','ABCAABAEJAEE','ABCAABAEJAEF','ABCIABAAAGGJ',"
++
"'ABCIABAAAGKB','ABCIABAAAJBC','ABCIABAAAJCD','ABCIABAAAJEK','ABCIABAAAJFE',"
++
"'ABCIABAAAJGE','ABCIABAAAJGF','ABCIABAAAJGG','ABCIABAAAJJK','ABCIABAABAGK',"
+
[beam] branch master updated (91143f6 -> 77e924f)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 91143f6 Update walkthrough.md (#16512) add 77e924f [BEAM-11808][BEAM-9879] Support aggregate functions with two arguments (#16200) No new revisions were added by this update. Summary of changes: .../sdk/extensions/sql/impl/udaf/StringAgg.java| 59 +++- .../zetasql/SupportedZetaSqlBuiltinFunctions.java | 6 +- .../translation/AggregateScanConverter.java| 62 + .../zetasql/translation/ExpressionConverter.java | 2 +- .../translation/SqlNullIfOperatorRewriter.java | 4 +- .../translation/SqlOperatorMappingTable.java | 150 - .../sql/zetasql/translation/SqlOperators.java | 55 +++- .../sql/zetasql/ZetaSqlDialectSpecTest.java| 62 + 8 files changed, 293 insertions(+), 107 deletions(-)
[beam] branch master updated: [BEAM-12464] Change ProtoSchemaTranslator beam schema creation to match the order for protobufs containing Oneof fields (#14974)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5ab52a3 [BEAM-12464] Change ProtoSchemaTranslator beam schema
creation to match the order for protobufs containing Oneof fields (#14974)
5ab52a3 is described below
commit 5ab52a3f4cfe2680098186763550b5f8ad30319c
Author: Reuben van Ammers
AuthorDate: Fri Jan 14 06:13:00 2022 +1100
[BEAM-12464] Change ProtoSchemaTranslator beam schema creation to match the
order for protobufs containing Oneof fields (#14974)
* ProtoSchemaTranslator now orders oneof fields in the resultant beam
schema in accordance with their location in the protobuf definition
* add reverse order protobuf
* add noncontiguous oneof and some renaming
* Comments and variable renaming
* add reversed row tests
* add noncontiguous tests
* remove redundant null check
* minor test comment update
* update
* add reversedonof test
* add noncontiguous oneof test
Co-authored-by: Reuben van Ammers
---
.../extensions/protobuf/ProtoSchemaTranslator.java | 26 -
.../protobuf/ProtoDynamicMessageSchemaTest.java| 86 ++
.../protobuf/ProtoMessageSchemaTest.java | 46
.../protobuf/ProtoSchemaTranslatorTest.java| 14 +++
.../sdk/extensions/protobuf/TestProtoSchemas.java | 125 -
.../src/test/proto/proto3_schema_messages.proto| 28 +
6 files changed, 314 insertions(+), 11 deletions(-)
diff --git
a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java
b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java
index 91eb1bd7..ef46b59 100644
---
a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java
+++
b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java
@@ -156,13 +156,18 @@ class ProtoSchemaTranslator {
}
static Schema getSchema(Descriptors.Descriptor descriptor) {
-Set oneOfFields = Sets.newHashSet();
+/* OneOfComponentFields refers to the field number in the protobuf where
the component subfields
+ * are. This is needed to prevent double inclusion of the component
fields.*/
+Set oneOfComponentFields = Sets.newHashSet();
+/* OneOfFieldLocation stores the field number of the first field in the
OneOf. Using this, we can use the location
+of the first field in the OneOf as the location of the entire OneOf.*/
+Map oneOfFieldLocation = Maps.newHashMap();
List fields =
Lists.newArrayListWithCapacity(descriptor.getFields().size());
for (OneofDescriptor oneofDescriptor : descriptor.getOneofs()) {
List subFields =
Lists.newArrayListWithCapacity(oneofDescriptor.getFieldCount());
Map enumIds = Maps.newHashMap();
for (FieldDescriptor fieldDescriptor : oneofDescriptor.getFields()) {
-oneOfFields.add(fieldDescriptor.getNumber());
+oneOfComponentFields.add(fieldDescriptor.getNumber());
// Store proto field number in a field option.
FieldType fieldType = beamFieldTypeFromProtoField(fieldDescriptor);
subFields.add(
@@ -172,17 +177,26 @@ class ProtoSchemaTranslator {
enumIds.putIfAbsent(fieldDescriptor.getName(),
fieldDescriptor.getNumber()) == null);
}
FieldType oneOfType = FieldType.logicalType(OneOfType.create(subFields,
enumIds));
- fields.add(Field.of(oneofDescriptor.getName(), oneOfType));
+ oneOfFieldLocation.put(
+ oneofDescriptor.getFields().get(0).getNumber(),
+ Field.of(oneofDescriptor.getName(), oneOfType));
}
for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields())
{
- if (!oneOfFields.contains(fieldDescriptor.getNumber())) {
+ int fieldDescriptorNumber = fieldDescriptor.getNumber();
+ if (!oneOfComponentFields.contains(fieldDescriptorNumber)) {
// Store proto field number in metadata.
FieldType fieldType = beamFieldTypeFromProtoField(fieldDescriptor);
fields.add(
-withFieldNumber(
-Field.of(fieldDescriptor.getName(), fieldType),
fieldDescriptor.getNumber())
+withFieldNumber(Field.of(fieldDescriptor.getName(), fieldType),
fieldDescriptorNumber)
.withOptions(getFieldOptions(fieldDescriptor)));
+/* Note that descriptor.getFields() returns an iterator in the order
of the fields in the .proto file, not
+ * in field number order. Therefore we can safely insert the
OneOfField at the field of its first component.*/
+ } else {
+Field oneOfField = oneOfFieldLocation.get
[beam] branch master updated (6b68536 -> 4c9d4f0)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 6b68536 Add Nexmark Query 14 (#16337) new 83e9218 [BEAM-13569] Change Spark dependencies to implementation. new f0289c0 remove redundant dependency new e35eea8 remove redundant testImplementation dependencies new 4c9d4f0 Merge pull request #16396 from ibzib/BEAM-13569 The 34111 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: runners/spark/job-server/spark_job_server.gradle | 1 - runners/spark/spark_runner.gradle| 14 +- 2 files changed, 5 insertions(+), 10 deletions(-)
[beam] branch master updated: [BEAM-13581] Remove previous job name for Flink PVR precommit.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 79fe544 [BEAM-13581] Remove previous job name for Flink PVR precommit.
new ea09119 Merge pull request #16394 from ibzib/seed
79fe544 is described below
commit 79fe544072e8885aa738baa488fb590cb5aa301b
Author: Kyle Weaver
AuthorDate: Wed Dec 29 15:33:01 2021 -0800
[BEAM-13581] Remove previous job name for Flink PVR precommit.
For some reason Jenkins is having problems renaming it.
---
.../job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy| 1 -
1 file changed, 1 deletion(-)
diff --git
a/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
b/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
index 8efb098..22aed70 100644
---
a/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
+++
b/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
@@ -32,7 +32,6 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
],
)
builder.build {
- previousNames('beam_PostCommit_Java_PVR_Flink_Batch')
// Publish all test results to Jenkins.
publishers {
archiveJunit('**/build/test-results/**/*.xml')
[beam] branch master updated: [BEAM-13402] Simplify PubsubLiteSink
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7dbbd09 [BEAM-13402] Simplify PubsubLiteSink
new b33bebc Merge pull request #16215 from
dpcollins-google/publish-no-desync
7dbbd09 is described below
commit 7dbbd097585017c7a655afc041435123745086dc
Author: Daniel Collins
AuthorDate: Sun Dec 12 23:54:02 2021 -0500
[BEAM-13402] Simplify PubsubLiteSink
This also makes it not afflicted by futures never terminating by avoiding
indefinite waits
---
.../io/gcp/pubsublite/internal/PublisherCache.java | 5 +
.../gcp/pubsublite/internal/PublisherOrError.java | 49 --
.../io/gcp/pubsublite/internal/PubsubLiteSink.java | 107 +
.../pubsublite/internal/PubsubLiteSinkTest.java| 46 -
4 files changed, 29 insertions(+), 178 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherCache.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherCache.java
index 76b69b3..ac85ba9 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherCache.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherCache.java
@@ -27,9 +27,13 @@ import
com.google.errorprone.annotations.concurrent.GuardedBy;
import java.util.HashMap;
import org.apache.beam.sdk.io.gcp.pubsublite.PublisherOptions;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** A map of working publishers by PublisherOptions. */
class PublisherCache implements AutoCloseable {
+ private final Logger logger = LoggerFactory.getLogger(PublisherCache.class);
+
@GuardedBy("this")
private final HashMap>
livePublishers =
new HashMap<>();
@@ -49,6 +53,7 @@ class PublisherCache implements AutoCloseable {
new Listener() {
@Override
public void failed(State s, Throwable t) {
+logger.warn("Publisher failed.", t);
evict(options);
}
},
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherOrError.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherOrError.java
deleted file mode 100644
index 7eb1c66..000
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherOrError.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.gcp.pubsublite.internal;
-
-import com.google.auto.value.AutoOneOf;
-import com.google.cloud.pubsublite.MessageMetadata;
-import com.google.cloud.pubsublite.internal.CheckedApiException;
-import com.google.cloud.pubsublite.internal.Publisher;
-
-/** A helper representing either a Publisher or an error. */
-@AutoOneOf(PublisherOrError.Kind.class)
-@SuppressWarnings({
- "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
-})
-abstract class PublisherOrError {
- enum Kind {
-PUBLISHER,
-ERROR
- }
-
- abstract Kind getKind();
-
- abstract Publisher publisher();
-
- abstract CheckedApiException error();
-
- static PublisherOrError ofPublisher(Publisher p) {
-return AutoOneOf_PublisherOrError.publisher(p);
- }
-
- static PublisherOrError ofError(CheckedApiException e) {
-return AutoOneOf_PublisherOrError.error(e);
- }
-}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteSink.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteSink.java
index 4b666d2..f370919 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apa
[beam] branch master updated: [BEAM-13402] Version bump Pub/Sub Lite and implement changes to ensure client and channel reuse (#16358)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d7ccd0f [BEAM-13402] Version bump Pub/Sub Lite and implement changes
to ensure client and channel reuse (#16358)
d7ccd0f is described below
commit d7ccd0fe39958c4cd33fd1da850f8f13b2874495
Author: dpcollins-google <40498610+dpcollins-goo...@users.noreply.github.com>
AuthorDate: Tue Dec 28 15:59:56 2021 -0500
[BEAM-13402] Version bump Pub/Sub Lite and implement changes to ensure
client and channel reuse (#16358)
* [BEAM-13402] Version bump Pub/Sub Lite and implement changes to ensure
client (and channel) reuse
* revert TopicBacklogReader changes- ManagedBacklogReader already handles
this
* reformat
* revert TopicBacklogReaderSettings changes
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../pubsublite/internal/InitialOffsetReader.java | 5 +-
.../internal/InitialOffsetReaderImpl.java | 13 +--
.../internal/PerSubscriptionPartitionSdf.java | 7 +-
.../pubsublite/internal/PublisherAssembler.java| 57 +++
.../pubsublite/internal/SubscribeTransform.java| 11 ++-
.../pubsublite/internal/SubscriberAssembler.java | 105 +
.../SubscriptionPartitionProcessorImpl.java| 2 +-
.../internal/TopicBacklogReaderImpl.java | 22 +
.../internal/TopicBacklogReaderSettings.java | 45 ++---
10 files changed, 136 insertions(+), 133 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 76d7b47..a6c53d1 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -452,7 +452,7 @@ class BeamModulePlugin implements Plugin {
def errorprone_version = "2.3.4"
def google_clients_version = "1.32.1"
def google_cloud_bigdataoss_version = "2.2.4"
-def google_cloud_pubsublite_version = "1.4.6"
+def google_cloud_pubsublite_version = "1.4.7"
def google_code_gson_version = "2.8.9"
def google_oauth_clients_version = "1.32.1"
// Try to keep grpc_version consistent with gRPC version in
google_cloud_platform_libraries_bom
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java
index 402a37b..292d3e1 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java
@@ -20,9 +20,6 @@ package org.apache.beam.sdk.io.gcp.pubsublite.internal;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.pubsublite.Offset;
-interface InitialOffsetReader extends AutoCloseable {
+interface InitialOffsetReader {
Offset read() throws ApiException;
-
- @Override
- void close();
}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReaderImpl.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReaderImpl.java
index 5972b13..ddee9b7 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReaderImpl.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReaderImpl.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.gcp.pubsublite.internal;
import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical;
+import static java.util.concurrent.TimeUnit.MINUTES;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.pubsublite.Offset;
@@ -31,8 +32,9 @@ class InitialOffsetReaderImpl implements InitialOffsetReader {
private final SubscriptionPath subscription;
private final Partition partition;
- InitialOffsetReaderImpl(CursorClient client, SubscriptionPath subscription,
Partition partition) {
-this.client = client;
+ InitialOffsetReaderImpl(
+ CursorClient unownedCursorClient, SubscriptionPath subscription,
Partition partition) {
+this.client = unownedCursorClient;
this.subscription = subscription;
this.partition = partition;
}
@@ -40,15 +42,10 @@ class InitialOffsetReaderImpl implements
InitialOffsetReader {
@Override
public Offset read() throws ApiException {
[beam] branch master updated: [BEAM-13571] Fix ClassNotFound exception in Flink tests
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0712b0a [BEAM-13571] Fix ClassNotFound exception in Flink tests
new 9ac567a Merge pull request #16375 from ibzib/flink-class1
0712b0a is described below
commit 0712b0a12c4885d5af9b78b7e4bd3bfa626defc1
Author: Kyle Weaver
AuthorDate: Tue Dec 28 09:15:37 2021 -0800
[BEAM-13571] Fix ClassNotFound exception in Flink tests
---
runners/flink/job-server/flink_job_server.gradle | 1 +
1 file changed, 1 insertion(+)
diff --git a/runners/flink/job-server/flink_job_server.gradle
b/runners/flink/job-server/flink_job_server.gradle
index 07ce308..1dcb837 100644
--- a/runners/flink/job-server/flink_job_server.gradle
+++ b/runners/flink/job-server/flink_job_server.gradle
@@ -80,6 +80,7 @@ dependencies {
implementation project(flinkRunnerProject)
permitUnusedDeclared project(flinkRunnerProject) // BEAM-11761
runtimeOnly group: "org.slf4j", name: "jcl-over-slf4j", version:
dependencies.create(project.library.java.slf4j_api).getVersion()
+ validatesPortableRunner project(path: flinkRunnerProject)
validatesPortableRunner project(path: flinkRunnerProject, configuration:
"testRuntimeMigration")
validatesPortableRunner project(path: ":sdks:java:core", configuration:
"shadowTest")
validatesPortableRunner project(path: ":runners:core-java", configuration:
"testRuntimeMigration")
[beam] branch master updated (9e07ab0 -> 8b50ba2)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 9e07ab0 Merge pull request #16376 from ibzib/BEAM-13572
add 8b50ba2 [BEAM-13052] Add Pub/Sub Lite xlang transforms in python
(#15727)
No new revisions were added by this update.
Summary of changes:
.../internal/ExternalTransformConfig.java | 108 ++
.../internal/ExternalTransformRegistrarImpl.java} | 21 ++--
.../gcp/pubsublite/internal/ProtoFromBytes.java} | 45
.../io/gcp/pubsublite/internal/ProtoToBytes.java | 33 +++---
.../{tools => io/gcp/pubsublite}/__init__.py | 8 +-
.../apache_beam/io/gcp/pubsublite/external.py | 126 +
.../apache_beam/io/gcp/pubsublite/proto_api.py | 112 ++
sdks/python/setup.py | 1 +
8 files changed, 400 insertions(+), 54 deletions(-)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformConfig.java
copy
sdks/java/io/{snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeTransformRegistrar.java
=>
google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java}
(70%)
copy
sdks/java/{testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/RowToCsv.java
=>
io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ProtoFromBytes.java}
(51%)
copy learning/katas/java/Triggers/Early Triggers/Early
Triggers/src/org/apache/beam/learning/katas/triggers/earlytriggers/GenerateEvent.java
=>
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ProtoToBytes.java
(60%)
copy sdks/python/apache_beam/{tools => io/gcp/pubsublite}/__init__.py (86%)
create mode 100644 sdks/python/apache_beam/io/gcp/pubsublite/external.py
create mode 100644 sdks/python/apache_beam/io/gcp/pubsublite/proto_api.py
[beam] branch master updated: [BEAM-13498] [BEAM-13573] exclude new tests on Flink
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2d458bc [BEAM-13498] [BEAM-13573] exclude new tests on Flink
new 9e07ab0 Merge pull request #16376 from ibzib/BEAM-13572
2d458bc is described below
commit 2d458bc75aa2940531cb78578a9bf7a938f7b75a
Author: Kyle Weaver
AuthorDate: Tue Dec 28 12:29:20 2021 -0500
[BEAM-13498] [BEAM-13573] exclude new tests on Flink
---
runners/flink/flink_runner.gradle | 4
1 file changed, 4 insertions(+)
diff --git a/runners/flink/flink_runner.gradle
b/runners/flink/flink_runner.gradle
index 19204a5..c224852 100644
--- a/runners/flink/flink_runner.gradle
+++ b/runners/flink/flink_runner.gradle
@@ -237,6 +237,8 @@ def createValidatesRunnerTask(Map m) {
excludeCategories
'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
excludeCategories
'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+excludeCategories
'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery'
+excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderInBundle'
}
filter {
// https://issues.apache.org/jira/browse/BEAM-12039
@@ -245,6 +247,8 @@ def createValidatesRunnerTask(Map m) {
excludeTestsMatching
'org.apache.beam.sdk.testing.TestStreamTest.testFirstElementLate'
// https://issues.apache.org/jira/browse/BEAM-12038
excludeTestsMatching
'org.apache.beam.sdk.testing.TestStreamTest.testLateDataAccumulating'
+// TODO(BEAM-13573)
+excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoTest$TimestampTests.testOnWindowTimestampSkew'
}
}
}
[beam] branch master updated (f2f1bdf -> acff5d1)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from f2f1bdf Exclude UsesOnWindowExpiration by category from Dataflow v2 streaming (#16346) add acff5d1 [BEAM-13052] Increment pubsub python version and fix breakages. (#16126) No new revisions were added by this update. Summary of changes: .../examples/complete/game/game_stats_it_test.py | 7 +++-- .../examples/complete/game/leader_board_it_test.py | 7 +++-- .../streaming_wordcount_debugging_it_test.py | 14 + .../examples/streaming_wordcount_it_test.py| 14 + sdks/python/apache_beam/io/gcp/bigquery_test.py| 7 +++-- sdks/python/apache_beam/io/gcp/pubsub.py | 15 +- .../apache_beam/io/gcp/pubsub_integration_test.py | 21 +- .../apache_beam/io/gcp/pubsub_io_perf_test.py | 12 sdks/python/apache_beam/io/gcp/pubsub_test.py | 33 +- .../apache_beam/io/gcp/tests/pubsub_matcher.py | 5 ++-- sdks/python/apache_beam/io/gcp/tests/utils.py | 2 +- sdks/python/apache_beam/io/gcp/tests/utils_test.py | 8 -- ...low_exercise_streaming_metrics_pipeline_test.py | 14 + .../runners/direct/transform_evaluator.py | 15 +- sdks/python/apache_beam/testing/test_utils.py | 16 +++ sdks/python/apache_beam/testing/test_utils_test.py | 4 +-- sdks/python/apache_beam/utils/timestamp.py | 2 +- sdks/python/setup.py | 2 +- 18 files changed, 107 insertions(+), 91 deletions(-)
[beam] branch master updated: [BEAM-13521] flink pvr batch precommit
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 1096956 [BEAM-13521] flink pvr batch precommit
new 5b06fa3 Merge pull request #16349 from ibzib/flink-pvr
1096956 is described below
commit 1096956c2ed9e379dc914d961d7b3e6131ed59ca
Author: Kyle Weaver
AuthorDate: Thu Dec 23 11:53:31 2021 -0800
[BEAM-13521] flink pvr batch precommit
---
...Java_PortableValidatesRunner_Flink_Batch.groovy | 44 --
...Java_PortableValidatesRunner_Flink_Batch.groovy | 40
2 files changed, 40 insertions(+), 44 deletions(-)
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
deleted file mode 100644
index 9857320..000
---
a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Batch',
-'Run Java Flink PortableValidatesRunner Batch', 'Java Flink
PortableValidatesRunner Batch Tests', this) {
- description('Runs the Java PortableValidatesRunner suite on the Flink
runner.')
-
- // Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
- // Publish all test results to Jenkins
- publishers {
-archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Gradle goals for this job.
- steps {
-gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
-
tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesPortableRunnerBatch")
- commonJobProperties.setGradleSwitches(delegate)
-}
- }
-}
diff --git
a/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
b/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
new file mode 100644
index 000..8efb098
--- /dev/null
+++
b/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonTestProperties
+import PrecommitJobBuilder
+
+// This job runs the suite of ValidatesRunner tests against the Flink runner.
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+scope: this,
+nameBase: 'Java_PVR_Flink_Batch',
+gradleTask:
":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesPortableRunnerBatch",
+timeoutMins: 240,
+triggerPathPatterns: [
+ '^sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/.*$',
+ '^runners/flink/.*$',
+ '^runners/java-fn-execution/.*$',
+],
+)
+builder.build {
+ previousNames('beam_PostCommit_Java_PVR_Flink_Batch')
+ // Publish all test results to Jenkins.
+ publishers {
+archiveJunit('**/build/test-results/**/*.xml')
+ }
+}
[beam] branch master updated: [BEAM-13430] Upgrade beam to gradle 7.3.2 (#16319)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4774ac7 [BEAM-13430] Upgrade beam to gradle 7.3.2 (#16319)
4774ac7 is described below
commit 4774ac713f427fefb38114f661516faef26d8207
Author: dpcollins-google <40498610+dpcollins-goo...@users.noreply.github.com>
AuthorDate: Thu Dec 23 12:32:17 2021 -0500
[BEAM-13430] Upgrade beam to gradle 7.3.2 (#16319)
* [BEAM-13430] Upgrade beam to gradle 7.3.2
* [BEAM-13430] Clean up tests that override the time of the JVM
* [BEAM-13430] Remove shadow override
---
buildSrc/build.gradle.kts | 15 +-
.../jarlibs/shadow-6.1.0-log4jfix-SNAPSHOT.jar | Bin 5628292 -> 0 bytes
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 11 +-
gradle/wrapper/gradle-wrapper.jar | Bin 59203 -> 59536 bytes
gradle/wrapper/gradle-wrapper.properties | 2 +-
gradlew| 257 -
ownership/JAVA_DEPENDENCY_OWNERS.yaml | 5 -
runners/flink/job-server/flink_job_server.gradle | 1 +
runners/samza/job-server/build.gradle | 1 +
runners/spark/job-server/spark_job_server.gradle | 5 +-
runners/spark/spark_runner.gradle | 2 +-
.../SubscriptionPartitionProcessorImplTest.java| 4 +
12 files changed, 172 insertions(+), 131 deletions(-)
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
index 3339eba..0f3b42d 100644
--- a/buildSrc/build.gradle.kts
+++ b/buildSrc/build.gradle.kts
@@ -31,31 +31,28 @@ repositories {
url = uri("https://repo.spring.io/plugins-release/;)
content { includeGroup("io.spring.gradle") }
}
- flatDir { dirs("jarlibs") }
}
// Dependencies on other plugins used when this plugin is invoked
dependencies {
implementation(gradleApi())
implementation(localGroovy())
-
implementation("com.github.jengelman.gradle.plugins:shadow:6.1.0-log4jfix-SNAPSHOT")
-
implementation("gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.5.0")
+ implementation("gradle.plugin.com.github.johnrengelman:shadow:7.1.1")
+ implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.3")
- runtimeOnly("net.ltgt.gradle:gradle-apt-plugin:0.21")
// Enable a Java annotation processor
runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13")
// Enable proto code generation
runtimeOnly("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0")
// Enable Avro code generation
runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1")
// Enable a code formatting plugin
- runtimeOnly("gradle.plugin.com.github.blindpirate:gogradle:0.11.4")
// Enable Go code compilation
runtimeOnly("gradle.plugin.com.palantir.gradle.docker:gradle-docker:0.22.0")
// Enable building Docker containers
runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5")
// Adds a 'taskTree' task to print task dependency
tree
-
runtimeOnly("com.github.jengelman.gradle.plugins:shadow:6.1.0-log4jfix-SNAPSHOT")
// Enable shading Java dependencies
+ runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1")
// Enable shading Java dependencies
runtimeOnly("ca.coglinc:javacc-gradle-plugin:2.4.0")
// Enable the JavaCC parser generator
runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5")
runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0")
// Enable creating an offline repository
runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:1.2.1")
// Enable errorprone Java static analysis
- runtimeOnly("org.ajoberstar.grgit:grgit-gradle:4.0.2")
// Enable website git publish to asf-site branch
- runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.13.2")
// Enable docker compose tasks
- runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.4.3")
// Enable dep analysis
+ runtimeOnly("org.ajoberstar.grgit:grgit-gradle:4.1.1")
[beam] branch master updated: [BEAM-13498] Sickbay portable Flink testProcessElementSkew.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new c03b4b5e [BEAM-13498] Sickbay portable Flink testProcessElementSkew. new c4dd8f4 Merge pull request #16295 from ibzib/BEAM-13498 c03b4b5e is described below commit c03b4b5ef7d70f00082cbc42c64e281993fab2f8 Author: Kyle Weaver AuthorDate: Mon Dec 20 15:46:01 2021 -0800 [BEAM-13498] Sickbay portable Flink testProcessElementSkew. --- runners/flink/job-server/flink_job_server.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index f00f244..41407e8 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -192,6 +192,8 @@ def portableValidatesRunnerTask(String name, Boolean streaming, Boolean checkpoi excludeTestsMatching 'org.apache.beam.sdk.testing.TestStreamTest.testLateDataAccumulating' // TODO(BEAM-12710) excludeTestsMatching 'org.apache.beam.sdk.testing.TestStreamTest.testFirstElementLate' + // TODO(BEAM-12498) + excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$TimestampTests.testProcessElementSkew' }, ) }
[beam] branch master updated: [BEAM-13496] Upgrade Flink runner to include log4j patches.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4e4fb27 [BEAM-13496] Upgrade Flink runner to include log4j patches.
new 05cbe24 Merge pull request #16293 from ibzib/flink-upgrade
4e4fb27 is described below
commit 4e4fb278905a931b88c73277620b35f1f0f7f6db
Author: Kyle Weaver
AuthorDate: Mon Dec 20 15:12:06 2021 -0800
[BEAM-13496] Upgrade Flink runner to include log4j patches.
---
runners/flink/1.11/build.gradle | 2 +-
runners/flink/1.12/build.gradle | 2 +-
runners/flink/1.13/build.gradle | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/runners/flink/1.11/build.gradle b/runners/flink/1.11/build.gradle
index c3736ed..81cc0a0 100644
--- a/runners/flink/1.11/build.gradle
+++ b/runners/flink/1.11/build.gradle
@@ -20,7 +20,7 @@ def basePath = '..'
/* All properties required for loading the Flink build script */
project.ext {
// Set the version of all Flink-related dependencies here.
- flink_version = '1.11.4'
+ flink_version = '1.11.6'
// Version specific code overrides.
main_source_overrides = ['./src/main/java']
test_source_overrides = ['./src/test/java']
diff --git a/runners/flink/1.12/build.gradle b/runners/flink/1.12/build.gradle
index 31878af..ed62e0f 100644
--- a/runners/flink/1.12/build.gradle
+++ b/runners/flink/1.12/build.gradle
@@ -20,7 +20,7 @@ def basePath = '..'
/* All properties required for loading the Flink build script */
project.ext {
// Set the version of all Flink-related dependencies here.
- flink_version = '1.12.5'
+ flink_version = '1.12.7'
// Version specific code overrides.
main_source_overrides = ["${basePath}/1.11/src/main/java", './src/main/java']
test_source_overrides = ["${basePath}/1.11/src/test/java", './src/test/java']
diff --git a/runners/flink/1.13/build.gradle b/runners/flink/1.13/build.gradle
index d6d04eb..d29d2f9 100644
--- a/runners/flink/1.13/build.gradle
+++ b/runners/flink/1.13/build.gradle
@@ -20,7 +20,7 @@ def basePath = '..'
/* All properties required for loading the Flink build script */
project.ext {
// Set the version of all Flink-related dependencies here.
- flink_version = '1.13.2'
+ flink_version = '1.13.5'
// Version specific code overrides.
main_source_overrides = ["${basePath}/1.11/src/main/java",
"${basePath}/1.12/src/main/java", './src/main/java']
test_source_overrides = ["${basePath}/1.11/src/test/java",
"${basePath}/1.12/src/test/java", './src/test/java']
[beam] branch master updated: [BEAM-13497] Correct class name in Flink tests.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 930c885 [BEAM-13497] Correct class name in Flink tests. new 73b1ef4 Merge pull request #16294 from ibzib/BEAM-13497 930c885 is described below commit 930c88587673d015d7431bb5e8ff9cbf2902c672 Author: Kyle Weaver AuthorDate: Mon Dec 20 15:34:27 2021 -0800 [BEAM-13497] Correct class name in Flink tests. --- runners/flink/job-server/flink_job_server.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index f00f244..0859887 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -179,7 +179,7 @@ def portableValidatesRunnerTask(String name, Boolean streaming, Boolean checkpoi excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections' excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream' excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery' - excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderInStage' + excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderInBundle' } } },
[beam] branch master updated (a4cca44 -> 4f2bbff)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from a4cca44 [BEAM-13015] Add a state backed iterable that can be mutated under certain circumstances. (#16252) add 4f2bbff [BEAM-13388] Update Cloud DLP after breaking changes. (#16236) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/ml/gcp/cloud_dlp.py | 22 ++-- sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py | 33 +++- 2 files changed, 35 insertions(+), 20 deletions(-)
[beam] branch master updated (da11b84 -> d41ae47)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from da11b84 [BEAM-13454] Fix and test dataframe read_fwf. (#16064) add d41ae47 [BEAM-12976] Pipeline visitor to discover pushdown opportunities. (#16176) No new revisions were added by this update. Summary of changes: .../graph/ProjectionProducerVisitor.java | 83 +++ .../graph/ProjectionProducerVisitorTest.java | 274 + 2 files changed, 357 insertions(+) create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java
[beam] branch master updated: Include name of missing tag in error message.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e056d73 Include name of missing tag in error message.
new 25e6008 Merge pull request #16175 from ibzib/tag-name
e056d73 is described below
commit e056d73ca8d8f0bab6d0439bfc5f498081319976
Author: Kyle Weaver
AuthorDate: Wed Dec 8 14:16:33 2021 -0800
Include name of missing tag in error message.
---
.../src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index fef7a01..178644c 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -221,7 +221,8 @@ public class PCollectionTuple implements PInput, POutput {
@SuppressWarnings("unchecked")
PCollection pcollection = (PCollection) pcollectionMap.get(tag);
if (pcollection == null) {
- throw new IllegalArgumentException("TupleTag not found in this
PCollectionTuple tuple");
+ throw new IllegalArgumentException(
+ String.format("TupleTag %s not found in this PCollectionTuple
tuple", tag));
}
return pcollection;
}
[beam] branch master updated (f98a3b0 -> daadaa4)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from f98a3b0 [BEAM-12565] Dataframe compare implementation (#16027) add daadaa4 [BEAM-12976] Use a map to pass all pushdown requests at once. (#16189) No new revisions were added by this update. Summary of changes: .../java/org/apache/beam/sdk/schemas/ProjectionProducer.java | 10 ++ .../sql/meta/provider/SchemaIOTableProviderWrapper.java | 5 - .../sql/meta/provider/TestSchemaIOTableProviderWrapper.java | 12 +--- 3 files changed, 19 insertions(+), 8 deletions(-)
[beam] branch master updated: Clarify instructions on how to get contributor list.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 91950d6 Clarify instructions on how to get contributor list. new dfb5726 Merge pull request #16133 from tvalentyn/tags_command 91950d6 is described below commit 91950d60925dfb88e32869b5194313a589c15242 Author: Valentyn Tymofieiev AuthorDate: Fri Dec 3 11:57:16 2021 -0800 Clarify instructions on how to get contributor list. --- website/www/site/content/en/contribute/release-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 8a8b607..9896640 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -680,7 +680,7 @@ See [beam-2.31.0.md](https://github.com/apache/beam/commit/a32a75ed0657c122c6625 - Copy the changes for the current release from `CHANGES.md` to the blog post and edit as necessary. - Be sure to add yourself to [authors.yml](https://github.com/apache/beam/blob/master/website/www/site/data/authors.yml) if necessary. -__Tip__: Use git log to find contributors to the releases. (e.g: `git log --pretty='%aN' ^v2.10.0 v2.11.0 | sort | uniq`). +__Tip__: Use git log to find contributors to the releases. (e.g: `git fetch origin --tags; git log --pretty='%aN' ^v2.10.0 v2.11.0-RC1 | sort | uniq`). Make sure to clean it up, as there may be duplicate or incorrect user names. __NOTE__: Make sure to include any breaking changes, even to `@Experimental` features,
[beam] branch master updated (aa60f1b -> 580f588)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from aa60f1b Don't pin a particular version of Tensorflow. (#16102) add 580f588 [BEAM-12733] Fix failing integration tests for Java Recommendation AI (#15375) No new revisions were added by this update. Summary of changes: .../ml/RecommendationAICatalogItemIT.java | 34 ++ 1 file changed, 28 insertions(+), 6 deletions(-)
[beam] branch master updated: [BEAM-13220] Update release instructions.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8d75633 [BEAM-13220] Update release instructions.
new 5fc55f3 Merge pull request #16065 from ibzib/BEAM-13220
8d75633 is described below
commit 8d75633d0c082b1151405cf364c9370bfc95f596
Author: Kyle Weaver
AuthorDate: Wed Nov 24 15:24:04 2021 -0800
[BEAM-13220] Update release instructions.
The biggest change here is to run Java wordcount and mobile gaming examples
on Jenkins instead of the bash script, since getting the script to work
properly took too long.
---
.../main/scripts/publish_github_release_notes.sh | 2 +-
release/src/main/scripts/run_rc_validation.sh | 128 +
release/src/main/scripts/script.config | 2 -
.../site/content/en/contribute/release-guide.md| 20 ++--
4 files changed, 14 insertions(+), 138 deletions(-)
diff --git a/release/src/main/scripts/publish_github_release_notes.sh
b/release/src/main/scripts/publish_github_release_notes.sh
index 7f58648..09fec15 100755
--- a/release/src/main/scripts/publish_github_release_notes.sh
+++ b/release/src/main/scripts/publish_github_release_notes.sh
@@ -33,7 +33,7 @@ RELEASE_NOTES=$(
)
# Escape notes' content to work with JSON
-ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python -c 'import json,sys;
print(json.dumps(sys.stdin.read()))')
+ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python3 -c 'import json,sys;
print(json.dumps(sys.stdin.read()))')
# Build JSON for the API request
REQUEST_JSON="$(cat <<-EOF
diff --git a/release/src/main/scripts/run_rc_validation.sh
b/release/src/main/scripts/run_rc_validation.sh
index baff09b..28ba2eb 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -207,133 +207,6 @@ kubectl version
echo ""
-echo ""
-echo "Starting Java Quickstart==="
-echo "[Current task] Java quickstart with direct runner"
-if [[ "$java_quickstart_direct" = true ]]; then
- echo "*"
- echo "* Running Java Quickstart with DirectRunner"
- echo "*"
- ./gradlew :runners:direct-java:runQuickstartJavaDirect \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with direct runner"
-fi
-
-echo "[Current task] Java quickstart with Flink local runner"
-if [[ "$java_quickstart_flink_local" = true ]]; then
- echo "*"
- echo "* Running Java Quickstart with Flink local runner"
- echo "*"
- ./gradlew :runners:flink:1.13:runQuickstartJavaFlinkLocal \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with Flink local runner"
-fi
-
-echo "[Current task] Java quickstart with Spark local runner"
-if [[ "$java_quickstart_spark_local" = true ]]; then
- echo "*"
- echo "* Running Java Quickstart with Spark local runner"
- echo "*"
- ./gradlew :runners:spark:2:runQuickstartJavaSpark \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with Spark local runner"
-fi
-
-echo "[Current task] Java quickstart with Dataflow runner"
-if [[ "$java_quickstart_dataflow" = true && ! -z `which gcloud` ]]; then
- echo "*"
- echo "* Running Java Quickstart with DataflowRunner"
- echo "*"
- ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix
-else
- echo "* Skip Java quickstart with Dataflow runner. Google Cloud SDK is
required."
-fi
-
-echo "[Current task] Java quickstart with Twister2 local runner"
-if [[ "$java_quickstart_twister2_local" = true ]]; then
- echo "*"
- echo "* Running Java Quickstart with Twister2 local runner"
- echo "*"
- ./gradlew :runners:twister2:runQuickstartJavaTwister2 \
-
[beam] branch master updated: [BEAM-13337] Periodically delete stale spanner databases.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7a5bdcd [BEAM-13337] Periodically delete stale spanner databases.
new 2b5087a Merge pull request #16082 from ibzib/BEAM-13337
7a5bdcd is described below
commit 7a5bdcd3c8a5d990a82bdb24af2bdac285a23d60
Author: Kyle Weaver
AuthorDate: Mon Nov 29 13:45:09 2021 -0800
[BEAM-13337] Periodically delete stale spanner databases.
---
.test-infra/tools/stale_dataflow_jobs_cleaner.sh | 7 +++
1 file changed, 7 insertions(+)
diff --git a/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
b/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
index e563048..e6df705 100755
--- a/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
+++ b/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
@@ -27,3 +27,10 @@ if [[ ${STALE_JOBS} ]]; then
else
echo "No stale jobs found."
fi
+
+# Delete spanner databases older than 1 day.
+gcloud spanner databases list \
+--instance beam-test \
+--filter="createTime < $(date --iso-8601=s -d '1 day ago')" \
+--format="value(name)" | \
+xargs -I{} gcloud spanner databases delete {} --instance beam-test --quiet
[beam] branch master updated: Increase timeout for Dataflow Streaming VR
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f05bcfe Increase timeout for Dataflow Streaming VR
new 3769d75 Merge pull request #16040 from kileys/BEAM-13238-timeout
f05bcfe is described below
commit f05bcfe9be1815bd063cc19bb5fd16ca36b30688
Author: kileys
AuthorDate: Mon Nov 22 19:24:58 2021 +
Increase timeout for Dataflow Streaming VR
---
.../job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
index 2735db9..8a5afb3 100644
---
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
+++
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
@@ -27,7 +27,7 @@
PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo
description('Runs the ValidatesRunner suite on the Dataflow runner
forcing streaming mode.')
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 480)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 540)
// Publish all test results to Jenkins
publishers {
[beam] branch master updated: [BEAM-13313] Fix WindowingStrategy proto to use snake case field names.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a92bc09 [BEAM-13313] Fix WindowingStrategy proto to use snake case
field names.
new e51c7a2 Merge pull request #16054 from lukecwik/beam13313
a92bc09 is described below
commit a92bc0922afa3ccc03426b304261c06294f06c63
Author: Luke Cwik
AuthorDate: Tue Nov 23 13:36:46 2021 -0800
[BEAM-13313] Fix WindowingStrategy proto to use snake case field names.
Also update the one usage within Python.
Go and Java convert snake case to camel case method names so they didn't
have to change.
---
model/pipeline/src/main/proto/beam_runner_api.proto | 2 +-
sdks/python/apache_beam/transforms/core.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/model/pipeline/src/main/proto/beam_runner_api.proto
b/model/pipeline/src/main/proto/beam_runner_api.proto
index 3c649bf..aaeb9da 100644
--- a/model/pipeline/src/main/proto/beam_runner_api.proto
+++ b/model/pipeline/src/main/proto/beam_runner_api.proto
@@ -1095,7 +1095,7 @@ message WindowingStrategy {
int64 allowed_lateness = 8;
// (Required) Indicate whether empty on-time panes should be omitted.
- OnTimeBehavior.Enum OnTimeBehavior = 9;
+ OnTimeBehavior.Enum on_time_behavior = 9;
// (Required) Whether or not the window fn assigns inputs to exactly one
window
//
diff --git a/sdks/python/apache_beam/transforms/core.py
b/sdks/python/apache_beam/transforms/core.py
index 0038a85..bd999f8 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -2981,7 +2981,7 @@ class Windowing(object):
output_time=self.timestamp_combiner,
# TODO(robertwb): Support EMIT_IF_NONEMPTY
closing_behavior=beam_runner_api_pb2.ClosingBehavior.EMIT_ALWAYS,
-OnTimeBehavior=beam_runner_api_pb2.OnTimeBehavior.FIRE_ALWAYS,
+on_time_behavior=beam_runner_api_pb2.OnTimeBehavior.FIRE_ALWAYS,
allowed_lateness=self.allowed_lateness.micros // 1000,
environment_id=environment_id)
[beam] annotated tag v2.34.0 updated (b3b1843 -> 63ea678)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag v2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.34.0 was modified! *** from b3b1843 (commit) to 63ea678 (tag) tagging 15867770295f4f1e27273984d5188a0bde62e13e (tag) length 147 bytes by Kyle Weaver on Thu Nov 11 11:44:07 2021 -0800 - Log - Apache Beam 2.34.0 release -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEE8R431/AG0IYjKHZ5e21mc8ea6nIFAmGNcq0ACgkQe21mc8ea 6nIpQg//foycL85HggIg/SfVTLEhU9+IOrnNKI1Z56KN1drKM7RAHj8U1Jy1AR7s aboDQyVuIGIL4ekORcbEomq6+UyuqvT95U2c3ic6/jgZ47+tnnMVmLPEzV3QVBDo a4D7MdCLejGC5AnZT4xob/mObCDNGNeCdtAqGILQoNGYfpRnsh0AjoycSIASXyH2 K/YBl3m3ajPwCPjyWQzmoLUwWzuLwuwKDlWwLiRIEFr6Szsl4SWmDMsSiGARbhKx Gl1z4Dl2ORPi56QChaU6SqY013syPL9v5hdq1BydgsGrzF4VkDNRZOQHu3L16as4 kreLX/tXb8fFn/ElVGUpuJ0zoPPsRyqbBrj1xGJ3WFajnCkfUaJkyVtsrvs+63T0 wuXntzv9xx5ZGbN2XsPD04ID3nEvrgWT8yqD897x+DI0/Y1e2w+Ybeto/cR3qExK A3T7AGL9ouDPcdud35L2tt4RrgyUEuMcHSJx6Dlb0bsTm3JGuxhgqY726FzqKYM/ B5A6XT8yRzmjCOoEyvcXWG9ksUeE0mMDU4nSLpeOtre+ToSmE7GZgb2bDh4Y7LVC a119JJIXjTCA2nTmNiiZ1bNUN60A1VZQ4M/OiRakWnXLrBDZu9GKQqhbwru1TbjA yuQEvDr4SSLuGwlc7EaaWD30+cP91TPDBNDMZMjeMWX+Z1xVKxs= =6f1o -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[beam] annotated tag sdks/v2.34.0 updated (b3b1843 -> 28a65ef)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag sdks/v2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag sdks/v2.34.0 was modified! *** from b3b1843 (commit) to 28a65ef (tag) tagging 15867770295f4f1e27273984d5188a0bde62e13e (tag) length 147 bytes by Kyle Weaver on Thu Nov 11 11:42:45 2021 -0800 - Log - Go SDK v2.34.0 -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEE8R431/AG0IYjKHZ5e21mc8ea6nIFAmGNcmEACgkQe21mc8ea 6nIuSw//Yc5Cb/2ZS8wwVdByU7oG15ZEFkGj+Ue6/veUdY5yml7v1AUU0pnSM5I5 G34w9bQ9S5J12QbNupM7nI+cbSXzbeL3yl1ramRxlcSe0JRhoC/yOCFAXa0Ih3ad naaAyr0yuvzrfDL4kWmejhCy7QlzQO3xt3OuctKtJOAp3T4lVBCQvQM/fbG+oO7Q cpC8jZuw0cjjYYQU/52mLOyuicAF6/E7f1ZsFYHIyVfrX1dcr4tMCo5F/iE5jzLo z+fLC+RnVD5AKLEXCUcrsgO3TU3YvZzytjEMDYM43uCAu6Q0F91c/YRWLBjAuVeL b3oIjJm9qqZNSmCu10bjw5ZEKkOJAkHCNuYKXi6Bf6Q/9O+zVR0oCSnK5BgCfesG q6QQoffM9PRPGlFLasXUSLqWDX7n3fdXFFz0DW5ckdAgmlBQKvBsppqc7cnkZU8E O3DBRFOq5nko3/rv+TzCyPgBJ6nRcZntXIi6fBeanY/4k5UcQXWX9+6ZHtcXMk+J 8qaN25Npk5eOP9EKJbIunkI3+DtAZbSPM6bYQJ/9XMbAHvgYvzt5mITGT2wWYX/k qiC04yc/aHwNVOnssuyBJ8nD1iI4NoVnzB2vkJkAzxe0mMSX8ojBbiffwL7DUq3I QxKau+vJ6+BOt74Q8AT+9+XO3EL3tJ7rY9naV/CfiO7LRELsv+8= =zkzX -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[beam] branch master updated (0ba6eca -> 7a5b47b)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 0ba6eca [BEAM-13133] Loosen partitioning requirement for sample (#15818) new ff497ff Update Beam website to release 2.34.0. new 3437306 Prepare docs for 2.34.0 release. new 7a5b47b Merge pull request #15834 from ibzib/website-234 The 33538 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGES.md | 18 +-- website/www/site/config.toml | 2 +- website/www/site/content/en/blog/beam-2.34.0.md| 152 + .../www/site/content/en/get-started/downloads.md | 8 ++ website/www/site/static/.htaccess | 2 +- 5 files changed, 164 insertions(+), 18 deletions(-) create mode 100644 website/www/site/content/en/blog/beam-2.34.0.md
[beam] branch master updated (9b1512d -> 585e081)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 9b1512d Merge pull request #15771 from Enable BQ Standard SQL dialect style table specifications. new cb54dca [BEAM-12976] Add Java interfaces for projection pushdown. new cd2ac1f fix cast new 3f10676 [BEAM-12976] Address review comments. new 2249c9f Hard-code output tag instead of making it public. new 72a5054 revert MAIN_OUTPUT_TAG to private new 31b66f4 Remove WITHOUT_FIELD_ORDERING. Field reordering will be a prerequisite for pushdown support. new 585e081 Merge pull request #15712 from ibzib/generic-pushdown The 33488 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../beam/sdk/schemas/ProjectionConsumer.java | 41 ++ .../beam/sdk/schemas/ProjectionProducer.java | 46 + .../beam/sdk/schemas/io/PushdownProjector.java | 48 -- .../provider/SchemaIOTableProviderWrapper.java | 19 + .../provider/TestSchemaIOTableProviderWrapper.java | 25 ++- 5 files changed, 112 insertions(+), 67 deletions(-) create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionConsumer.java create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/PushdownProjector.java
[beam] branch master updated: [BEAM-5172] Temporary ignore testSplit and testSizes tests waiting for a fix because they are flaky.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ea1bf40 [BEAM-5172] Temporary ignore testSplit and testSizes tests
waiting for a fix because they are flaky.
new 2f2e4fa Merge pull request #15922 from ibzib/es-flake
ea1bf40 is described below
commit ea1bf40ef61b77bfd6a4d0e73fab9ae147a70f69
Author: Etienne Chauchot
AuthorDate: Fri Sep 3 09:49:04 2021 +0200
[BEAM-5172] Temporary ignore testSplit and testSizes tests waiting for a
fix because they are flaky.
---
.../java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 3 +++
.../java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 3 +++
.../java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 3 +++
3 files changed, 9 insertions(+)
diff --git
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
index 4023b24..da593bf 100644
---
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
+++
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -33,6 +33,7 @@ import org.elasticsearch.client.RestClient;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -83,6 +84,7 @@ public class ElasticsearchIOTest implements Serializable {
@Rule public TestPipeline pipeline = TestPipeline.create();
+ @Ignore("https://issues.apache.org/jira/browse/BEAM-5172;)
@Test
public void testSizes() throws Exception {
// need to create the index using the helper method (not create it at
first insertion)
@@ -154,6 +156,7 @@ public class ElasticsearchIOTest implements Serializable {
elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
}
+ @Ignore("https://issues.apache.org/jira/browse/BEAM-5172;)
@Test
public void testSplit() throws Exception {
// need to create the index using the helper method (not create it at
first insertion)
diff --git
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
index 3e77be8..d9124cd 100644
---
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
+++
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -33,6 +33,7 @@ import org.elasticsearch.client.RestClient;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -82,6 +83,7 @@ public class ElasticsearchIOTest implements Serializable {
@Rule public TestPipeline pipeline = TestPipeline.create();
+ @Ignore("https://issues.apache.org/jira/browse/BEAM-5172;)
@Test
public void testSizes() throws Exception {
// need to create the index using the helper method (not create it at
first insertion)
@@ -153,6 +155,7 @@ public class ElasticsearchIOTest implements Serializable {
elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
}
+ @Ignore("https://issues.apache.org/jira/browse/BEAM-5172;)
@Test
public void testSplit() throws Exception {
// need to create the index using the helper method (not create it at
first insertion)
diff --git
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
index 4d78666..761e60a 100644
---
a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
+++
b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -33,6 +33,7 @@ import org.elasticsearch.client.RestClient;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -83,6 +84,7 @@ public class
svn commit: r50806 - /dev/beam/2.34.0/python/
Author: ibzib Date: Fri Nov 5 00:55:18 2021 New Revision: 50806 Log: Staging Python artifacts for Apache Beam 2.34.0 RC2 Modified: dev/beam/2.34.0/python/apache-beam-2.34.0.zip dev/beam/2.34.0/python/apache-beam-2.34.0.zip.asc dev/beam/2.34.0/python/apache-beam-2.34.0.zip.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_i686.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_x86_64.whl dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38
svn commit: r50804 - in /dev/beam/2.34.0: apache-beam-2.34.0-source-release.zip apache-beam-2.34.0-source-release.zip.asc apache-beam-2.34.0-source-release.zip.sha512
Author: ibzib Date: Thu Nov 4 22:49:34 2021 New Revision: 50804 Log: Staging Java artifacts for Apache Beam 2.34.0 RC2 Added: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip (with props) Modified: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 Added: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc == --- dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc (original) +++ dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc Thu Nov 4 22:49:34 2021 @@ -1,16 +1,16 @@ -BEGIN PGP SIGNATURE- -iQIzBAABCgAdFiEE8R431/AG0IYjKHZ5e21mc8ea6nIFAmF4yxwACgkQe21mc8ea -6nL0Yw/9EXTJHbOOEkfxg7aqB3CeYD8+xAiCIiGE1a1vqa6nEYbA0HckdGJniShA -CXjjPW8gwqAkQtMGzzcDpm/XFMVWbVr+V6cvUeHsax0ynUZ3QcTW4CSLBfxaz9gj -G1H6AYol+17+zltifeXuvmsT7hlWZkTud70OXeM2UO6FhlxDeCwbBU/fbRSR9TwY -S1FlUsiQlgwTywh94bs4nP/2+8sKBSdQ65CbVzL65GTjiKpGRnWswGCB3ESkIZ1k -CQP5k9PCNtmGw5nIZgERrbpAil9mLnpLR8GVkOKO7rFJ5SISrye0hciXTD8+zl88 -napKlVrSHPrdPwKU1guFDP041KTOWnFBAEJcRxApD7rPE4yMUoHgznHfbC1Qci8A -2YHUijQfT6cwYw/XWwWOjET4yduMP16etBVwUR+8OSOJQF+a4kikLuzlaFU8lsN4 -OD+HDBnJAKFPWyv8DcWfmj8S+rFo3AjZ5BfN3RYW5KTESxhWG5eznDhek5g8X8yM -9Q4KyvbazqB72Oem7RiLbywFbTOWr6USlHvPrTp+CAfadU0dBfhNUyt4PEBvdvES -qlBF9S2SYE7S+ut0Whak/9uk4BC4J8dd2EjKD2TER+xh1IdU+wgQCH3XJOqqesRL -BlZT98Des2F0PcRMiW0mAngOMnuHxS+UVTGFDlzYxptNIkwwj6I= -=POBo +iQIzBAABCgAdFiEE8R431/AG0IYjKHZ5e21mc8ea6nIFAmGEY1gACgkQe21mc8ea +6nIrwA/9HA0FgijzDVKRg4bOxp7CZ//nkAb6ycwD6U65FB+DSxtuWBUsn9GL6PCJ +gP7RznxHQ6NCA7M4cNuGEFl3T34KH5y7VlhowTVs7hBIowK6k1HdS+eo6n5YzESB +QKJ2QDIAyjX5OvoVPywoavV8A3v+g+uBGSFUwhE1Ry2Dk8LBbgGfNlRKr2ydGENW +XwW4LdUJax0w+ZWe4EoxjqMCw3WV42fbHvaTYJ6FZO4N6qiJXGecflGiZbaj7OyW +A17xE4g/0SyqqgA/Gwv4NsQbhX1T8bB7LGtAbVWLeqwog/SfDBK8ojAO5k/e8087 +aK48OhDpAkJ98RYsjvjKTYPq+rINWgIdFaOWB8KDNBi8scqfhYQxuQ+wSztpnCBG +7ioEGsi0Ihj6XRM8UQCsswu8p8ttu7Bw94Ew60SAwVFCTZBfyGo75SWvfR6LLYyR +dD97U0XpQZn5hzyrcNGmU+fgIDLzkCQQPtglSQb4JVqbNFLJJXicZopZB81TbVmi +NogqTJqsxd+Snx4k2SePFMY3ZEWkYHdWayA+GJ+1a1/8bTLfDD2HZ3sK36h1BrD4 +S0KhJpNAOg4xmU1TIi9f8xUg8vAc9ZoCSZGPo5DtgvizxB5awmRzxbJq9ef0N/ay +P1bJJJCmbPkOpZw9wQTu4BwL1HqOWQaEr3X5TrPWI4+53uXaH+Y= +=ZOqX -END PGP SIGNATURE- Modified: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 == --- dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 (original) +++ dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 Thu Nov 4 22:49:34 2021 @@ -1 +1 @@ -c7ac361491e44c4a82a888d752280f11cce0ee261d570b12db50063472d38b51143be5bf1f377d4f56ed367fc5d1addd3e5aded0d7e3eec63d0f13cfdd1b1371 apache-beam-2.34.0-source-release.zip +4beb6e3fea17c1b0c107fcea886b4d6637264ddd030a5613a5b7c2b43ee127fba8e7da429cac86b6c02b677432a553e9e2a8f9fcef51beb5ddbed56b51f294c7 apache-beam-2.34.0-source-release.zip
[beam] annotated tag v2.34.0-RC2 updated (b3b1843 -> 1586777)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag v2.34.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.34.0-RC2 was modified! *** from b3b1843 (commit) to 1586777 (tag) tagging b3b184361100492f92fcc51ff82a0fcd962d5ee0 (commit) replaces jupyterlab-sidepanel-v1.0.0 by Kyle Weaver on Thu Nov 4 14:45:57 2021 -0700 - Log - v2.34.0-RC2 --- No new revisions were added by this update. Summary of changes:
[beam] annotated tag sdks/v2.34.0-RC2 updated (b3b1843 -> 4f8964f)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag sdks/v2.34.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag sdks/v2.34.0-RC2 was modified! *** from b3b1843 (commit) to 4f8964f (tag) tagging b3b184361100492f92fcc51ff82a0fcd962d5ee0 (commit) replaces jupyterlab-sidepanel-v1.0.0 by Kyle Weaver on Thu Nov 4 14:45:57 2021 -0700 - Log - Go SDK v2.34.0-RC2 --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.34.0 updated: [BEAM-13143] Fix python doc generator error.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new 86e2e0e [BEAM-13143] Fix python doc generator error.
new bec9149 Merge pull request #15906 from ibzib/BEAM-13143-cp
86e2e0e is described below
commit 86e2e0ed7736c97e843d3120940dd6dd7d36d2db
Author: Kyle Weaver
AuthorDate: Thu Oct 28 15:09:06 2021 -0700
[BEAM-13143] Fix python doc generator error.
---
sdks/python/tox.ini | 1 +
1 file changed, 1 insertion(+)
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index ca3e0b8..4fc99d3 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -142,6 +142,7 @@ extras = test,gcp,docs,interactive,dataframe
deps =
Sphinx==1.8.5
sphinx_rtd_theme==0.4.3
+ docutils<0.18
commands =
time {toxinidir}/scripts/generate_pydoc.sh
[beam] branch release-2.34.0 updated: Merge pull request #15891: [BEAM-13187] Set filesToStage after full jar resolution.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new c3e6521 Merge pull request #15891: [BEAM-13187] Set filesToStage
after full jar resolution.
new 7e777d8 Merge pull request #15902 from TheNeuralBit/BEAM-13187-cp
c3e6521 is described below
commit c3e6521e4aa867c4ee2df81133cef24aedb66704
Author: Chamikara Jayalath
AuthorDate: Thu Nov 4 10:22:18 2021 -0700
Merge pull request #15891: [BEAM-13187] Set filesToStage after full jar
resolution.
---
sdks/python/apache_beam/transforms/external.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/external.py
b/sdks/python/apache_beam/transforms/external.py
index ab02ae6..008b076 100644
--- a/sdks/python/apache_beam/transforms/external.py
+++ b/sdks/python/apache_beam/transforms/external.py
@@ -656,8 +656,6 @@ class JavaJarExpansionService(object):
transform.
"""
def __init__(self, path_to_jar, extra_args=None):
-if extra_args is None:
- extra_args = ['{{PORT}}', f'--filesToStage={path_to_jar}']
self._path_to_jar = path_to_jar
self._extra_args = extra_args
self._service_count = 0
@@ -666,6 +664,8 @@ class JavaJarExpansionService(object):
if self._service_count == 0:
self._path_to_jar = subprocess_server.JavaJarServer.local_jar(
self._path_to_jar)
+ if self._extra_args is None:
+self._extra_args = ['{{PORT}}', f'--filesToStage={self._path_to_jar}']
# Consider memoizing these servers (with some timeout).
self._service_provider = subprocess_server.JavaJarServer(
ExpansionAndArtifactRetrievalStub,
[beam] branch master updated: Fix typo: s/spark/twister2
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 013da55 Fix typo: s/spark/twister2 new 0faa3b9 Merge pull request #15885 from ibzib/wc-twister2 013da55 is described below commit 013da5568a64e46204a33e58cc2233eae7e0c577 Author: Kyle Weaver AuthorDate: Wed Nov 3 15:21:13 2021 -0700 Fix typo: s/spark/twister2 --- release/src/main/groovy/quickstart-java-twister2.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/src/main/groovy/quickstart-java-twister2.groovy b/release/src/main/groovy/quickstart-java-twister2.groovy index 7c7e382..b46538c 100644 --- a/release/src/main/groovy/quickstart-java-twister2.groovy +++ b/release/src/main/groovy/quickstart-java-twister2.groovy @@ -28,7 +28,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Twister2' t.intent 'Gets the WordCount Example Code' QuickstartArchetype.generate(t) - t.intent 'Runs the WordCount Code with Spark runner' + t.intent 'Runs the WordCount Code with Twister2 runner' // Run the wordcount example with the Twister2 runner t.run """mvn compile exec:java -q \ -Dmaven.wagon.http.retryHandler.class=default \
svn commit: r50639 - /dev/beam/2.34.0/python/
Author: ibzib Date: Wed Oct 27 03:53:37 2021 New Revision: 50639 Log: Staging Python artifacts for Apache Beam 2.34.0 RC1 Added: dev/beam/2.34.0/python/ dev/beam/2.34.0/python/apache-beam-2.34.0.zip (with props) dev/beam/2.34.0/python/apache-beam-2.34.0.zip.asc dev/beam/2.34.0/python/apache-beam-2.34.0.zip.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux1_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2010_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-manylinux2014_aarch64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win32.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp36-cp36m-win_amd64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux1_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_i686.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2010_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-manylinux2014_aarch64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win32.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp37-cp37m-win_amd64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_i686.whl (with props) dev/beam/2.34.0/python/apache_beam-2.34.0-cp38-cp38-manylinux1_i686.whl.asc dev/beam/2.34.0/python/apache_beam-2.34.0-cp38
svn commit: r50638 - in /dev/beam/2.34.0: ./ apache-beam-2.34.0-source-release.zip apache-beam-2.34.0-source-release.zip.asc apache-beam-2.34.0-source-release.zip.sha512
Author: ibzib Date: Wed Oct 27 03:46:20 2021 New Revision: 50638 Log: Staging Java artifacts for Apache Beam 2.34.0 RC1 Added: dev/beam/2.34.0/ dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip (with props) dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 Added: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc == --- dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc (added) +++ dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.asc Wed Oct 27 03:46:20 2021 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEE8R431/AG0IYjKHZ5e21mc8ea6nIFAmF4yxwACgkQe21mc8ea +6nL0Yw/9EXTJHbOOEkfxg7aqB3CeYD8+xAiCIiGE1a1vqa6nEYbA0HckdGJniShA +CXjjPW8gwqAkQtMGzzcDpm/XFMVWbVr+V6cvUeHsax0ynUZ3QcTW4CSLBfxaz9gj +G1H6AYol+17+zltifeXuvmsT7hlWZkTud70OXeM2UO6FhlxDeCwbBU/fbRSR9TwY +S1FlUsiQlgwTywh94bs4nP/2+8sKBSdQ65CbVzL65GTjiKpGRnWswGCB3ESkIZ1k +CQP5k9PCNtmGw5nIZgERrbpAil9mLnpLR8GVkOKO7rFJ5SISrye0hciXTD8+zl88 +napKlVrSHPrdPwKU1guFDP041KTOWnFBAEJcRxApD7rPE4yMUoHgznHfbC1Qci8A +2YHUijQfT6cwYw/XWwWOjET4yduMP16etBVwUR+8OSOJQF+a4kikLuzlaFU8lsN4 +OD+HDBnJAKFPWyv8DcWfmj8S+rFo3AjZ5BfN3RYW5KTESxhWG5eznDhek5g8X8yM +9Q4KyvbazqB72Oem7RiLbywFbTOWr6USlHvPrTp+CAfadU0dBfhNUyt4PEBvdvES +qlBF9S2SYE7S+ut0Whak/9uk4BC4J8dd2EjKD2TER+xh1IdU+wgQCH3XJOqqesRL +BlZT98Des2F0PcRMiW0mAngOMnuHxS+UVTGFDlzYxptNIkwwj6I= +=POBo +-END PGP SIGNATURE- Added: dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 == --- dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 (added) +++ dev/beam/2.34.0/apache-beam-2.34.0-source-release.zip.sha512 Wed Oct 27 03:46:20 2021 @@ -0,0 +1 @@ +c7ac361491e44c4a82a888d752280f11cce0ee261d570b12db50063472d38b51143be5bf1f377d4f56ed367fc5d1addd3e5aded0d7e3eec63d0f13cfdd1b1371 apache-beam-2.34.0-source-release.zip
[beam] annotated tag sdks/v2.34.0-RC1 updated (f635f86 -> 8759203)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag sdks/v2.34.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag sdks/v2.34.0-RC1 was modified! *** from f635f86 (commit) to 8759203 (tag) tagging f635f8683979027acadbf785c34396f1dceffdaf (commit) replaces jupyterlab-sidepanel-v1.0.0 by Kyle Weaver on Tue Oct 26 16:33:11 2021 -0700 - Log - sdks/v2.34.0-RC1 --- No new revisions were added by this update. Summary of changes:
[beam] annotated tag v2.34.0-RC1 updated (f635f86 -> f19a87b)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to annotated tag v2.34.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.34.0-RC1 was modified! *** from f635f86 (commit) to f19a87b (tag) tagging f635f8683979027acadbf785c34396f1dceffdaf (commit) replaces jupyterlab-sidepanel-v1.0.0 by Kyle Weaver on Tue Oct 26 16:26:20 2021 -0700 - Log - v2.34.0-RC1 --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.34.0 updated: [BEAM-13104] ParquetIO: SplitReadFn must read the whole block
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new 2d655ea [BEAM-13104] ParquetIO: SplitReadFn must read the whole block
new 621a4f9 Merge pull request #15806 from ibzib/parquet-cp
2d655ea is described below
commit 2d655eac45d036dc23962d09e45642b77c6f5cea
Author: Alexey Romanenko
AuthorDate: Mon Oct 25 17:21:47 2021 +0200
[BEAM-13104] ParquetIO: SplitReadFn must read the whole block
---
.../src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java | 6 +++---
.../src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java | 3 ++-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git
a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
index 81f5978..c733f67 100644
---
a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
+++
b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
@@ -888,12 +888,12 @@ public class ParquetIO {
continue;
}
if (record == null) {
- // only happens with FilteredRecordReader at end of block
+ // it happens when a record is filtered out in this block
LOG.debug(
- "filtered record reader reached end of block in block {}
in file {}",
+ "record is filtered out by reader in block {} in file
{}",
currentBlock,
file.toString());
- break;
+ continue;
}
if (recordReader.shouldSkipCurrentRecord()) {
// this record is being filtered via the filter2 package
diff --git
a/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
b/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
index d2609b4..261abd9 100644
---
a/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
+++
b/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
@@ -574,7 +574,8 @@ public class ParquetIOTest implements Serializable {
readPipeline.apply(
ParquetIO.read(SCHEMA)
.from(temporaryFolder.getRoot().getAbsolutePath() + "/*")
-.withConfiguration(configuration));
+.withConfiguration(configuration)
+.withSplit());
PAssert.that(readBack).containsInAnyOrder(expectedRecords);
readPipeline.run().waitUntilFinish();
}
[beam] branch master updated: [BEAM-13104] ParquetIO: SplitReadFn must read the whole block
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 136eadc [BEAM-13104] ParquetIO: SplitReadFn must read the whole block
new 46c649a Merge pull request #15789 from
aromanenko-dev/BEAM-13104-ParquetIO-filter
136eadc is described below
commit 136eadc121e136e25aafc2b65f130526e7f20142
Author: Alexey Romanenko
AuthorDate: Mon Oct 25 17:21:47 2021 +0200
[BEAM-13104] ParquetIO: SplitReadFn must read the whole block
---
.../src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java | 6 +++---
.../src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java | 3 ++-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git
a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
index 81f5978..c733f67 100644
---
a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
+++
b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java
@@ -888,12 +888,12 @@ public class ParquetIO {
continue;
}
if (record == null) {
- // only happens with FilteredRecordReader at end of block
+ // it happens when a record is filtered out in this block
LOG.debug(
- "filtered record reader reached end of block in block {}
in file {}",
+ "record is filtered out by reader in block {} in file
{}",
currentBlock,
file.toString());
- break;
+ continue;
}
if (recordReader.shouldSkipCurrentRecord()) {
// this record is being filtered via the filter2 package
diff --git
a/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
b/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
index d2609b4..261abd9 100644
---
a/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
+++
b/sdks/java/io/parquet/src/test/java/org/apache/beam/sdk/io/parquet/ParquetIOTest.java
@@ -574,7 +574,8 @@ public class ParquetIOTest implements Serializable {
readPipeline.apply(
ParquetIO.read(SCHEMA)
.from(temporaryFolder.getRoot().getAbsolutePath() + "/*")
-.withConfiguration(configuration));
+.withConfiguration(configuration)
+.withSplit());
PAssert.that(readBack).containsInAnyOrder(expectedRecords);
readPipeline.run().waitUntilFinish();
}
[beam] branch release-2.34.0 updated: [BEAM-12694] Include datetime in dicom test dataset name.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch release-2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.34.0 by this push: new fd0338c [BEAM-12694] Include datetime in dicom test dataset name. new f3c1a19 Merge pull request #15791 from ibzib/dicom-cp fd0338c is described below commit fd0338cf027168a24355bfabbabcfe040f254c7d Author: Kyle Weaver AuthorDate: Fri Oct 8 16:11:14 2021 -0700 [BEAM-12694] Include datetime in dicom test dataset name. Hopefully this should prevent random collisions. If it doesn't, it will at least give us another clue about what's going on. --- sdks/python/apache_beam/io/gcp/dicomio_integration_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py b/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py index 7970dd5..0a1daae 100644 --- a/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py @@ -25,6 +25,7 @@ in order to check if the connectors are functioning correctly. """ # pytype: skip-file +import datetime import random import string import unittest @@ -126,7 +127,8 @@ class DICOMIoIntegrationTest(unittest.TestCase): METADATA_DIR_PATH + META_DATA_REFINED_NAME) # create a temp Dicom store based on the time stamp -self.temp_dicom_store = "DICOM_store_" + random_string_generator(RAND_LEN) +self.temp_dicom_store = "DICOM_store_" + datetime.datetime.now().strftime( +'%Y-%m-%d_%H%M%S.%f_') + random_string_generator(RAND_LEN) create_dicom_store(self.project, DATA_SET_ID, REGION, self.temp_dicom_store) def tearDown(self):
[beam] branch master updated: [BEAM-11758] Update basics page: Aggregation, Runner, UDF, Schema (#15763)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 39613ea [BEAM-11758] Update basics page: Aggregation, Runner, UDF, Schema (#15763) 39613ea is described below commit 39613ea89fee6ae888095fcb5ac9ca5fd6d687f9 Author: Melissa Pashniak AuthorDate: Mon Oct 25 11:26:28 2021 -0700 [BEAM-11758] Update basics page: Aggregation, Runner, UDF, Schema (#15763) * [BEAM-11758] Update basics page: Aggregation, Runner, UDF, Schema * Address most review feedback * Fix type in intro list * Address remaining two feedback comments --- .../www/site/content/en/documentation/basics.md| 205 - .../content/en/documentation/programming-guide.md | 17 ++ website/www/site/static/images/aggregation.png | Bin 0 -> 14065 bytes 3 files changed, 171 insertions(+), 51 deletions(-) diff --git a/website/www/site/content/en/documentation/basics.md b/website/www/site/content/en/documentation/basics.md index bf7f20d..cb79e5d 100644 --- a/website/www/site/content/en/documentation/basics.md +++ b/website/www/site/content/en/documentation/basics.md @@ -17,10 +17,9 @@ limitations under the License. # Basics of the Beam model -Suppose you have a data processing engine that can pretty easily process graphs -of operations. You want to integrate it with the Beam ecosystem to get access -to other languages, great event time processing, and a library of connectors. -You need to know the core vocabulary: +Apache Beam is a unified model for defining both batch and streaming +data-parallel processing pipelines. To get started with Beam, you'll need to +understand an important set of core concepts: * [_Pipeline_](#pipeline) - A pipeline is a user-constructed graph of transformations that defines the desired data processing operations. @@ -30,16 +29,22 @@ You need to know the core vocabulary: data processing operation, or a step, in your pipeline. A transform is applied to zero or more `PCollection` objects, and produces zero or more `PCollection` objects. - * _SDK_ - A language-specific library for pipeline authors (we often call them - "users" even though we have many kinds of users) to build transforms, - construct their pipelines and submit them to a runner - * _Runner_ - You are going to write a piece of software called a runner that - takes a Beam pipeline and executes it using the capabilities of your data - processing engine. - -These concepts may be very similar to your processing engine's concepts. Since -Beam's design is for cross-language operation and reusable libraries of -transforms, there are some special features worth highlighting. + * [_Aggregation_](#aggregation) - Aggregation is computing a value from + multiple (1 or more) input elements. + * [_User-defined function (UDF)_](#user-defined-function-udf) - Some Beam + operations allow you to run user-defined code as a way to configure the + transform. + * [_Schema_](#schema) - A schema is a language-independent type definition for + a `PCollection`. The schema for a `PCollection` defines elements of that + `PCollection` as an ordered list of named fields. + * [_SDK_](/documentation/sdks/java/) - A language-specific library that lets + pipeline authors build transforms, construct their pipelines, and submit + them to a runner. + * [_Runner_](#runner) - A runner runs a Beam pipeline using the capabilities of + your chosen data processing engine. + +The following sections cover these concepts in more detail and provide links to +additional documentation. ### Pipeline @@ -215,45 +220,143 @@ For more information about PCollections, see the following page: * [Beam Programming Guide: PCollections](/documentation/programming-guide/#pcollections) -### User-Defined Functions (UDFs) - -Beam has seven varieties of user-defined function (UDF). A Beam pipeline -may contain UDFs written in a language other than your runner, or even multiple -languages in the same pipeline (see the [Runner API](#the-runner-api)) so the -definitions are language-independent (see the [Fn API](#the-fn-api)). - -The UDFs of Beam are: - - * _DoFn_ - per-element processing function (used in ParDo) - * _WindowFn_ - places elements in windows and merges windows (used in Window - and GroupByKey) - * _Source_ - emits data read from external sources, including initial and - dynamic splitting for parallelism (used in Read) - * _ViewFn_ - adapts a materialized PCollection to a particular interface (used - in side inputs) - * _WindowMappingFn_ - maps one element's window to another, and specifies - bounds on how far in the past the result window will be (used in side - inputs) - * _CombineFn_ - associative and commutative aggregation (used in Combine and - state) - * _Coder_ -
[beam] branch master updated (ca6f4f0 -> 3d8213a)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from ca6f4f0 lazy creation of source splits for export-based ReadFromBigQuery (#15610) add 3d8213a [BEAM-11275] Support remote package download from remote filesystems in Stager (#15105) No new revisions were added by this update. Summary of changes: CHANGES.md | 1 + .../runners/portability/artifact_service.py| 4 +- .../apache_beam/runners/portability/stager.py | 21 + .../apache_beam/runners/portability/stager_test.py | 55 ++ 4 files changed, 78 insertions(+), 3 deletions(-)
[beam] branch master updated: [BEAM-13055] Use unshallow clone to create PR.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8892810 [BEAM-13055] Use unshallow clone to create PR.
new bfe5fea Merge pull request #15755 from ibzib/depth1
8892810 is described below
commit 8892810ad04b9c16f59334be5c2954fe2204a0a2
Author: Kyle Weaver
AuthorDate: Tue Oct 19 11:10:12 2021 -0700
[BEAM-13055] Use unshallow clone to create PR.
---
release/src/main/scripts/run_rc_validation.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/release/src/main/scripts/run_rc_validation.sh
b/release/src/main/scripts/run_rc_validation.sh
index 292fc50..5045372 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -120,7 +120,7 @@ else
echo "* Creating local Beam workspace: ${LOCAL_BEAM_DIR}"
mkdir -p ${LOCAL_BEAM_DIR}
echo "* Cloning Beam repo"
- git clone --depth 1 --branch ${RC_TAG} ${GIT_REPO_URL} ${LOCAL_BEAM_DIR}
+ git clone --branch ${RC_TAG} ${GIT_REPO_URL} ${LOCAL_BEAM_DIR}
cd ${LOCAL_BEAM_DIR}
git checkout -b ${WORKING_BRANCH} ${RC_TAG} --quiet
echo "* Setting up git config"
[beam] branch release-2.34.0 updated (a4d5907 -> 9c729f2)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch release-2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git. from a4d5907 Merge pull request #15735 from ibzib/runnerv2 new c45a4d8 [BEAM-9487] Multiple Trigger.may_lose_data fixes new 5e5b407 [BEAM-9487] Remove CONDITION_NOT_GUARANTEED as potential data loss reason and update GBK test to get around DirectRunner bug new 7eff6fa [BEAM-9487] Do AfterAny, AfterAll, and AfterEach checks properly (i.e. no shortcuts) new d007520 [BEAM-9487] Remove unused import new c38a9b1 [BEAM-9487] Reintroduce flag but do not use it new 4c81293 [BEAM-9487] Add test that shows AfterCount finishing new 8837e1d [BEAM-9487] Make _ParallelTriggerFn.may_finish clearer new 9c729f2 Merge pull request #15745 from ibzib/trigger The 33176 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache_beam/transforms/ptransform_test.py | 7 +- sdks/python/apache_beam/transforms/trigger.py | 80 +++--- sdks/python/apache_beam/transforms/trigger_test.py | 117 ++--- 3 files changed, 101 insertions(+), 103 deletions(-)
[beam] branch master updated (ffde2a6 -> abe3aa7)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from ffde2a6 Merge pull request #15719 from [BEAM-13040] Revert breaking PR and add some test cases. new b9edf86 [BEAM-9487] Multiple Trigger.may_lose_data fixes new d01c152 [BEAM-9487] Remove CONDITION_NOT_GUARANTEED as potential data loss reason and update GBK test to get around DirectRunner bug new 0bb51d7 [BEAM-9487] Do AfterAny, AfterAll, and AfterEach checks properly (i.e. no shortcuts) new e2b6f56 [BEAM-9487] Remove unused import new a9708ab [BEAM-9487] Reintroduce flag but do not use it new 0e9260a [BEAM-9487] Add test that shows AfterCount finishing new 9343f37 [BEAM-9487] Make _ParallelTriggerFn.may_finish clearer new abe3aa7 Merge pull request #15603 from zhoufek/fat The 33275 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache_beam/transforms/ptransform_test.py | 7 +- sdks/python/apache_beam/transforms/trigger.py | 80 +++--- sdks/python/apache_beam/transforms/trigger_test.py | 117 ++--- 3 files changed, 101 insertions(+), 103 deletions(-)
[beam] branch release-2.34.0 updated: [BEAM-13053] Avoid runner v2 when streaming engine explicitly disabled.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new 34938cf [BEAM-13053] Avoid runner v2 when streaming engine explicitly
disabled.
new a4d5907 Merge pull request #15735 from ibzib/runnerv2
34938cf is described below
commit 34938cf5f68a2ab582f5e6c52031f9b9a972a7b6
Author: Robert Bradshaw
AuthorDate: Thu Oct 14 10:49:58 2021 -0700
[BEAM-13053] Avoid runner v2 when streaming engine explicitly disabled.
---
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index e78b973..36cd2c2 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -597,6 +597,7 @@ class DataflowRunner(PipelineRunner):
debug_options = options.view_as(DebugOptions)
# Streaming is always portable, default to runner v2.
if (options.view_as(StandardOptions).streaming and
+not debug_options.lookup_experiment('disable_streaming_engine') and
not options.view_as(GoogleCloudOptions).dataflow_kms_key):
if not debug_options.lookup_experiment('disable_runner_v2'):
debug_options.add_experiment('beam_fn_api')
[beam] branch master updated: [BEAM-13053] Avoid runner v2 when streaming engine explicitly disabled.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7634dfb [BEAM-13053] Avoid runner v2 when streaming engine explicitly
disabled.
new b0954e2 Merge pull request #15723 from robertwb/beam-13053
7634dfb is described below
commit 7634dfb815ae6e21490938eba250a8ab453956da
Author: Robert Bradshaw
AuthorDate: Thu Oct 14 10:49:58 2021 -0700
[BEAM-13053] Avoid runner v2 when streaming engine explicitly disabled.
---
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index e78b973..36cd2c2 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -597,6 +597,7 @@ class DataflowRunner(PipelineRunner):
debug_options = options.view_as(DebugOptions)
# Streaming is always portable, default to runner v2.
if (options.view_as(StandardOptions).streaming and
+not debug_options.lookup_experiment('disable_streaming_engine') and
not options.view_as(GoogleCloudOptions).dataflow_kms_key):
if not debug_options.lookup_experiment('disable_runner_v2'):
debug_options.add_experiment('beam_fn_api')
[beam] branch master updated (bb4c17c -> 4b11efd)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from bb4c17c Merge pull request #15707 from ihji/update_python_container_tag add 767b329 Revert "[BEAM-10913] - Forcing update of YAML file by running kubectl apply (#15663)" add 4b11efd Merge pull request #15705 from apache/revert-15663-BEAM-10913 No new revisions were added by this update. Summary of changes: .test-infra/metrics/build.gradle | 5 - 1 file changed, 5 deletions(-)
[beam] branch release-2.34.0 updated: Merge pull request #15631 from TheNeuralBit/dataframe-changes-update
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch release-2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.34.0 by this push: new 1c0174e Merge pull request #15631 from TheNeuralBit/dataframe-changes-update new 2876f6b Merge pull request #15710 from TheNeuralBit/cp-dataframe-changes 1c0174e is described below commit 1c0174ecca4c917c2e50ce84d39fd507e7939d69 Author: Kyle Weaver AuthorDate: Fri Oct 8 13:35:05 2021 -0700 Merge pull request #15631 from TheNeuralBit/dataframe-changes-update Minor: Add DataFrame changes to CHANGES.md --- CHANGES.md | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a7bf99b..44adea5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -69,6 +69,10 @@ * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Upgrade to Calcite 1.26.0 ([BEAM-9379](https://issues.apache.org/jira/browse/BEAM-9379)). +* Added a new `dataframe` extra to the Python SDK that tracks `pandas` versions + we've verified compatibility with. We now recommend installing Beam with `pip + install apache-beam[dataframe]` when you intend to use the DataFrame API + ([BEAM-12906](https://issues.apache.org/jira/browse/BEAM-12906)). ## Breaking Changes @@ -84,8 +88,8 @@ ## Bugfixes * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). -* Fixed error while writing multiple DeferredFrames to csv (Python) -([BEAM-12701](https://issues.apache.org/jira/browse/BEAM-12701)). +* Fixed error while writing multiple DeferredFrames to csv (Python) ([BEAM-12701](https://issues.apache.org/jira/browse/BEAM-12701)). +* Fixed error when importing the DataFrame API with pandas 1.0.x installed ([BEAM-12945](https://issues.apache.org/jira/browse/BEAM-12945)). * Fixed top.SmallestPerKey implementation in the Go SDK ([BEAM-12946](https://issues.apache.org/jira/browse/BEAM-12946)). ## Known Issues
[beam] branch master updated: [BEAM-12875] Register file systems in SparkExecutableStageFunction
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 36cc7b4 [BEAM-12875] Register file systems in SparkExecutableStageFunction new 38305d9 Merge pull request #15502 from meowcakes/register_filesystems_for_artifact_retrieval_service 36cc7b4 is described below commit 36cc7b42bd77eb1a87b3a469e5aeb58e534524db Author: Rogan Morrow AuthorDate: Wed Oct 6 16:18:04 2021 +0800 [BEAM-12875] Register file systems in SparkExecutableStageFunction --- .../translation/SparkBatchPortablePipelineTranslator.java | 2 ++ .../spark/translation/SparkExecutableStageFunction.java | 11 +++ .../translation/SparkStreamingPortablePipelineTranslator.java | 1 + .../spark/translation/SparkExecutableStageFunctionTest.java | 5 + 4 files changed, 19 insertions(+) diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java index f0d666c..62b39d7 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java @@ -250,6 +250,7 @@ public class SparkBatchPortablePipelineTranslator groupByKeyPair(inputDataset, keyCoder, wvCoder); SparkExecutableStageFunction function = new SparkExecutableStageFunction<>( + context.getSerializableOptions(), stagePayload, context.jobInfo, outputExtractionMap, @@ -262,6 +263,7 @@ public class SparkBatchPortablePipelineTranslator JavaRDD> inputRdd2 = ((BoundedDataset) inputDataset).getRDD(); SparkExecutableStageFunction function2 = new SparkExecutableStageFunction<>( + context.getSerializableOptions(), stagePayload, context.jobInfo, outputExtractionMap, diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java index 24296bf..a78b082 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java @@ -34,6 +34,7 @@ import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey.TypeCase; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.runners.core.InMemoryTimerInternals; import org.apache.beam.runners.core.TimerInternals; +import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.runners.core.construction.Timer; import org.apache.beam.runners.core.construction.graph.ExecutableStage; import org.apache.beam.runners.core.metrics.MetricsContainerImpl; @@ -51,11 +52,13 @@ import org.apache.beam.runners.fnexecution.state.StateRequestHandler; import org.apache.beam.runners.fnexecution.state.StateRequestHandlers; import org.apache.beam.runners.fnexecution.translation.BatchSideInputHandlerFactory; import org.apache.beam.runners.fnexecution.translation.PipelineTranslatorUtils; +import org.apache.beam.runners.spark.SparkPipelineOptions; import org.apache.beam.runners.spark.coders.CoderHelpers; import org.apache.beam.runners.spark.metrics.MetricsContainerStepMapAccumulator; import org.apache.beam.runners.spark.util.ByteArray; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.fn.data.FnDataReceiver; +import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.transforms.join.RawUnionValue; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; @@ -86,6 +89,8 @@ class SparkExecutableStageFunction private static final Logger LOG = LoggerFactory.getLogger(SparkExecutableStageFunction.class); + // Pipeline options for initializing the FileSystems + private final SerializablePipelineOptions pipelineOptions; private final RunnerApi.ExecutableStagePayload stagePayload; private final Map outputMap; private final SparkExecutableStageContextFactory contextFactory; @@ -100,6 +105,7 @@ class SparkExecutableStageFunction private transient Object currentTimerKey; SparkExecutableStageFunction( + SerializablePipelineOptions pipelineOptions, RunnerApi.ExecutableStagePayload stagePayload, JobInfo jobInfo, Map outputMap, @@ -107,6 +113,7 @@ class SparkExecutableStageFunction Map>, WindowedValue
[beam] branch master updated: [BEAM-12694] Include datetime in dicom test dataset name.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new eabfc15 [BEAM-12694] Include datetime in dicom test dataset name. new d2b785a Merge pull request #15697 from ibzib/BEAM-12694 eabfc15 is described below commit eabfc15673702e360b2ac0beb812154d3f7f3470 Author: Kyle Weaver AuthorDate: Fri Oct 8 16:11:14 2021 -0700 [BEAM-12694] Include datetime in dicom test dataset name. Hopefully this should prevent random collisions. If it doesn't, it will at least give us another clue about what's going on. --- sdks/python/apache_beam/io/gcp/dicomio_integration_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py b/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py index 7970dd5..0a1daae 100644 --- a/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/dicomio_integration_test.py @@ -25,6 +25,7 @@ in order to check if the connectors are functioning correctly. """ # pytype: skip-file +import datetime import random import string import unittest @@ -126,7 +127,8 @@ class DICOMIoIntegrationTest(unittest.TestCase): METADATA_DIR_PATH + META_DATA_REFINED_NAME) # create a temp Dicom store based on the time stamp -self.temp_dicom_store = "DICOM_store_" + random_string_generator(RAND_LEN) +self.temp_dicom_store = "DICOM_store_" + datetime.datetime.now().strftime( +'%Y-%m-%d_%H%M%S.%f_') + random_string_generator(RAND_LEN) create_dicom_store(self.project, DATA_SET_ID, REGION, self.temp_dicom_store) def tearDown(self):
[beam] branch release-2.34.0 updated: [BEAM-12829] Fix Release Gradle Build (#15444)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new c4562a4 [BEAM-12829] Fix Release Gradle Build (#15444)
new 401c2c1 Merge pull request #15695 from ibzib/release-build
c4562a4 is described below
commit c4562a45a158cb5c23261f55ee6ea0a5fed97820
Author: Robert Burke
AuthorDate: Thu Sep 2 10:15:52 2021 -0700
[BEAM-12829] Fix Release Gradle Build (#15444)
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
sdks/go/build.gradle| 1 +
sdks/go/container/build.gradle | 1 +
sdks/go/examples/build.gradle | 1 +
sdks/go/test/build.gradle | 1 +
sdks/go/test/load/build.gradle | 1 +
sdks/java/container/build.gradle| 1 +
sdks/python/container/build.gradle | 1 +
8 files changed, 8 insertions(+)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7ef3a40..2bbf4af 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2322,6 +2322,7 @@ class BeamModulePlugin implements Plugin {
'**/build/**',
'**/dist/**',
'**/target/**',
+'**/.gogradle/**',
'**/*.pyc',
'sdks/python/*.egg*/**',
'sdks/python/test-suites/**',
diff --git a/sdks/go/build.gradle b/sdks/go/build.gradle
index 0752a2f..f71cfe3 100644
--- a/sdks/go/build.gradle
+++ b/sdks/go/build.gradle
@@ -25,6 +25,7 @@ description = "Apache Beam :: SDKs :: Go"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
golang {
packagePath = 'github.com/apache/beam/sdks/v2/go'
diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle
index 643ffdc..7eebec7 100644
--- a/sdks/go/container/build.gradle
+++ b/sdks/go/container/build.gradle
@@ -26,6 +26,7 @@ description = "Apache Beam :: SDKs :: Go :: Container"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
clean.dependsOn cleanVendor
diff --git a/sdks/go/examples/build.gradle b/sdks/go/examples/build.gradle
index 26f0fa8..2c35199 100644
--- a/sdks/go/examples/build.gradle
+++ b/sdks/go/examples/build.gradle
@@ -25,6 +25,7 @@ description = "Apache Beam :: SDKs :: Go :: Examples"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
def getLocalPlatform = {
String hostOs = com.github.blindpirate.gogradle.crossplatform.Os.getHostOs()
diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle
index de2720a..e67992f 100644
--- a/sdks/go/test/build.gradle
+++ b/sdks/go/test/build.gradle
@@ -26,6 +26,7 @@ description = "Apache Beam :: SDKs :: Go :: Test"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
clean.dependsOn cleanVendor
diff --git a/sdks/go/test/load/build.gradle b/sdks/go/test/load/build.gradle
index a006fcd..3af32c8 100644
--- a/sdks/go/test/load/build.gradle
+++ b/sdks/go/test/load/build.gradle
@@ -25,6 +25,7 @@ description = "Apache Beam :: SDKs :: Go :: Test :: Load"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
def getLocalPlatform = {
String hostOs = com.github.blindpirate.gogradle.crossplatform.Os.getHostOs()
diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
index 23cfd42..b71010f 100644
--- a/sdks/java/container/build.gradle
+++ b/sdks/java/container/build.gradle
@@ -32,6 +32,7 @@ description = "Apache Beam :: SDKs :: Java :: Container"
installDependencies.enabled = false
resolveBuildDependencies.enabled = false
resolveTestDependencies.enabled = false
+gofmt.enabled = false
configurations {
dockerDependency
diff --git a/sdks/python/container/build.gradle
b/sdks/python/container/build.gradle
index 32d9bf3..3dc14e2 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -25,6 +25,7 @@ description = "Apache Be
[beam] branch release-2.34.0 updated: [BEAM-10114] Bump Pub/Sub Lite version (#15640)
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.34.0 by this push:
new f9c1007 [BEAM-10114] Bump Pub/Sub Lite version (#15640)
new 119f5b0 Merge pull request #15696 from ibzib/pslite-version
f9c1007 is described below
commit f9c1007cef0a44191f5ce67fcf61414e6847c475
Author: dpcollins-google <40498610+dpcollins-goo...@users.noreply.github.com>
AuthorDate: Fri Oct 8 18:03:20 2021 -0400
[BEAM-10114] Bump Pub/Sub Lite version (#15640)
* Bump Pub/Sub Lite version
* [BEAM-10114] force old checker-qual version
Co-authored-by: Kyle Weaver
---
.../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 3 ++-
sdks/java/io/google-cloud-platform/build.gradle | 8
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7ef3a40..7a679df 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -452,7 +452,7 @@ class BeamModulePlugin implements Plugin {
def errorprone_version = "2.3.4"
def google_clients_version = "1.32.1"
def google_cloud_bigdataoss_version = "2.2.2"
-def google_cloud_pubsublite_version = "1.0.4"
+def google_cloud_pubsublite_version = "1.2.0"
def google_code_gson_version = "2.8.6"
def google_oauth_clients_version = "1.31.0"
// Try to keep grpc_version consistent with gRPC version in
google_cloud_platform_libraries_bom
@@ -526,6 +526,7 @@ class BeamModulePlugin implements Plugin {
bigdataoss_util :
"com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version",
cassandra_driver_core :
"com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version",
cassandra_driver_mapping:
"com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version",
+checker_qual:
"org.checkerframework:checker-qual:$checkerframework_version",
classgraph :
"io.github.classgraph:classgraph:$classgraph_version",
commons_codec :
"commons-codec:commons-codec:1.15",
commons_compress:
"org.apache.commons:commons-compress:1.21",
diff --git a/sdks/java/io/google-cloud-platform/build.gradle
b/sdks/java/io/google-cloud-platform/build.gradle
index 731cf33..d16fb16 100644
--- a/sdks/java/io/google-cloud-platform/build.gradle
+++ b/sdks/java/io/google-cloud-platform/build.gradle
@@ -146,6 +146,14 @@ dependencies {
testRuntimeOnly library.java.slf4j_jdk14
}
+// Don't pull in newer versions of the checker framework from dependencies.
+// TODO(BEAM-11125) Remove this when Beam upgrades to newest checker framework
version.
+configurations.compile {
+ resolutionStrategy {
+force library.java.checker_qual
+ }
+}
+
/**
* These are integration tests with the real Pubsub service and the
DirectRunner.
[beam] 01/01: Revert "[BEAM-12908] Change to use PubsubSignal for information propagation so this works on dataflow"
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch revert-15607-pubsub-signal-psl-it
in repository https://gitbox.apache.org/repos/asf/beam.git
commit cb09eb8f314e44023897ddbf49b0c9df8cb6f7e5
Author: Kyle Weaver
AuthorDate: Fri Oct 8 15:06:18 2021 -0700
Revert "[BEAM-12908] Change to use PubsubSignal for information propagation
so this works on dataflow"
---
.../beam/sdk/io/gcp/pubsublite/ReadWriteIT.java| 88 +++---
1 file changed, 59 insertions(+), 29 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java
index bd15310..80c362a 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.gcp.pubsublite;
import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
+import static org.junit.Assert.fail;
import com.google.cloud.pubsublite.AdminClient;
import com.google.cloud.pubsublite.AdminClientSettings;
@@ -35,18 +36,19 @@ import com.google.cloud.pubsublite.proto.Subscription;
import
com.google.cloud.pubsublite.proto.Subscription.DeliveryConfig.DeliveryRequirement;
import com.google.cloud.pubsublite.proto.Topic;
import com.google.cloud.pubsublite.proto.Topic.PartitionConfig.Capacity;
+import com.google.errorprone.annotations.concurrent.GuardedBy;
import com.google.protobuf.ByteString;
import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.Deque;
-import java.util.Set;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
-import org.apache.beam.sdk.io.gcp.pubsub.TestPubsubSignal;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -55,11 +57,12 @@ import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.FlatMapElements;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.values.PCollection;
+import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
import org.joda.time.Duration;
import org.junit.After;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -68,12 +71,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RunWith(JUnit4.class)
+@Ignore("https://issues.apache.org/jira/browse/BEAM-12908;)
public class ReadWriteIT {
private static final Logger LOG = LoggerFactory.getLogger(ReadWriteIT.class);
private static final CloudZone ZONE = CloudZone.parse("us-central1-b");
private static final int MESSAGE_COUNT = 90;
- @Rule public transient TestPubsubSignal signal = TestPubsubSignal.create();
@Rule public transient TestPipeline pipeline = TestPipeline.create();
private static ProjectId getProject(PipelineOptions options) {
@@ -205,21 +208,28 @@ public class ReadWriteIT {
"dedupeMessages",
PubsubLiteIO.deduplicate(UuidDeduplicationOptions.newBuilder().build()));
}
- public static SimpleFunction extractIds() {
-return new SimpleFunction() {
- @Override
- public Integer apply(SequencedMessage input) {
-return Integer.parseInt(input.getMessage().getData().toStringUtf8());
- }
-};
+ // This static out of band communication is needed to retain serializability.
+ @GuardedBy("ReadWriteIT.class")
+ private static final List received = new ArrayList<>();
+
+ private static synchronized void addMessageReceived(SequencedMessage
message) {
+received.add(message);
}
- public static SerializableFunction, Boolean> testIds() {
-return ids -> {
- LOG.info("Ids are: {}", ids);
- Set target = IntStream.range(0,
MESSAGE_COUNT).boxed().collect(Collectors.toSet());
- return target.equals(ids);
-};
+ private static synchronized List
getTestQuickstartReceived() {
+return ImmutableList.copyOf(received);
+ }
+
+ private static PTransform,
PCollection>
+ collectTestQuickstart() {
+return MapElements.via(
+new SimpleFunction() {
+
[beam] branch revert-15607-pubsub-signal-psl-it created (now cb09eb8)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch revert-15607-pubsub-signal-psl-it in repository https://gitbox.apache.org/repos/asf/beam.git. at cb09eb8 Revert "[BEAM-12908] Change to use PubsubSignal for information propagation so this works on dataflow" This branch includes the following new commits: new cb09eb8 Revert "[BEAM-12908] Change to use PubsubSignal for information propagation so this works on dataflow" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] branch master updated (a0b933c -> 512fb80)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from a0b933c [BEAM-10955] Flink Java Runner test flake: Could not find Flink job (#15664) add 512fb80 [BEAM-10114] Bump Pub/Sub Lite version (#15640) No new revisions were added by this update. Summary of changes: .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 3 ++- sdks/java/io/google-cloud-platform/build.gradle | 8 2 files changed, 10 insertions(+), 1 deletion(-)
[beam] branch master updated (9bc8c96 -> a0b933c)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 9bc8c96 [BEAM-10913] - Forcing update of YAML file by running kubectl apply (#15663) add a0b933c [BEAM-10955] Flink Java Runner test flake: Could not find Flink job (#15664) No new revisions were added by this update. Summary of changes: .../java/org/apache/beam/runners/flink/FlinkSavepointTest.java| 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-)
[beam] branch master updated (2bcfb68 -> 9bc8c96)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 2bcfb68 Merge pull request #15631 from TheNeuralBit/dataframe-changes-update add 9bc8c96 [BEAM-10913] - Forcing update of YAML file by running kubectl apply (#15663) No new revisions were added by this update. Summary of changes: .test-infra/metrics/build.gradle | 5 + 1 file changed, 5 insertions(+)
[beam] branch master updated: Add DataFrame changes to CHANGES.md
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 628b2df Add DataFrame changes to CHANGES.md new 2bcfb68 Merge pull request #15631 from TheNeuralBit/dataframe-changes-update 628b2df is described below commit 628b2df1c84e397c88e519aec244f583148559ef Author: Brian Hulette AuthorDate: Thu Sep 30 16:16:57 2021 -0700 Add DataFrame changes to CHANGES.md --- CHANGES.md | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1c132df..fa78b52 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -67,6 +67,10 @@ * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Upgrade to Calcite 1.26.0 ([BEAM-9379](https://issues.apache.org/jira/browse/BEAM-9379)). +* Added a new `dataframe` extra to the Python SDK that tracks `pandas` versions + we've verified compatibility with. We now recommend installing Beam with `pip + install apache-beam[dataframe]` when you intend to use the DataFrame API + ([BEAM-12906](https://issues.apache.org/jira/browse/BEAM-12906)). ## Breaking Changes @@ -80,8 +84,8 @@ ## Bugfixes * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). -* Fixed error while writing multiple DeferredFrames to csv (Python) -([BEAM-12701](https://issues.apache.org/jira/browse/BEAM-12701)). +* Fixed error while writing multiple DeferredFrames to csv (Python) ([BEAM-12701](https://issues.apache.org/jira/browse/BEAM-12701)). +* Fixed error when importing the DataFrame API with pandas 1.0.x installed ([BEAM-12945](https://issues.apache.org/jira/browse/BEAM-12945)). * Fixed top.SmallestPerKey implementation in the Go SDK ([BEAM-12946](https://issues.apache.org/jira/browse/BEAM-12946)). ## Known Issues
[beam] branch release-2.34.0 updated: [BEAM-12769] Few fixes related to Java Class Lookup based cross-language expansion (#15677)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch release-2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.34.0 by this push: new 71ffc66 [BEAM-12769] Few fixes related to Java Class Lookup based cross-language expansion (#15677) new a91cc8a Merge pull request #15687 from chamikaramj/cherry_pick_15677 71ffc66 is described below commit 71ffc6614500fcd95a30e2df2e8ac7a081b7a5e9 Author: Chamikara Jayalath AuthorDate: Thu Oct 7 16:00:20 2021 -0700 [BEAM-12769] Few fixes related to Java Class Lookup based cross-language expansion (#15677) * Few fixes related to Java Class Lookup based cross-language expansion * Fix for JavaExternalTransform --- sdks/python/apache_beam/portability/common_urns.py | 4 ++-- sdks/python/apache_beam/transforms/external.py | 13 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sdks/python/apache_beam/portability/common_urns.py b/sdks/python/apache_beam/portability/common_urns.py index 178f7d0..4e23c4f 100644 --- a/sdks/python/apache_beam/portability/common_urns.py +++ b/sdks/python/apache_beam/portability/common_urns.py @@ -29,7 +29,7 @@ from apache_beam.portability.api.beam_runner_api_pb2_urns import StandardPTransf from apache_beam.portability.api.beam_runner_api_pb2_urns import StandardRequirements from apache_beam.portability.api.beam_runner_api_pb2_urns import StandardResourceHints from apache_beam.portability.api.beam_runner_api_pb2_urns import StandardSideInputTypes -from apache_beam.portability.api.external_transforms_pb2 import ExpansionMethods +from apache_beam.portability.api.external_transforms_pb2_urns import ExpansionMethods from apache_beam.portability.api.metrics_pb2_urns import MonitoringInfo from apache_beam.portability.api.metrics_pb2_urns import MonitoringInfoSpecs from apache_beam.portability.api.metrics_pb2_urns import MonitoringInfoTypeUrns @@ -68,4 +68,4 @@ requirements = StandardRequirements.Enum displayData = StandardDisplayData.DisplayData -java_class_lookup = ExpansionMethods.JAVA_CLASS_LOOKUP +java_class_lookup = ExpansionMethods.Enum.JAVA_CLASS_LOOKUP diff --git a/sdks/python/apache_beam/transforms/external.py b/sdks/python/apache_beam/transforms/external.py index 005cb18..ab02ae6 100644 --- a/sdks/python/apache_beam/transforms/external.py +++ b/sdks/python/apache_beam/transforms/external.py @@ -287,7 +287,12 @@ class JavaExternalTransform(ptransform.PTransform): """ def __init__(self, class_name, expansion_service=None): self._payload_builder = JavaClassLookupPayloadBuilder(class_name) -self._expansion_service = None +self._expansion_service = expansion_service + +# Beam explicitly looks for following attributes. Hence adding +# 'None' values here to prevent '__getattr__' from being called. +self.inputs = None +self._fn_api_payload = None def __call__(self, *args, **kwargs): self._payload_builder.with_constructor(*args, **kwargs) @@ -310,8 +315,8 @@ class JavaExternalTransform(ptransform.PTransform): def expand(self, pcolls): return pcolls | ExternalTransform( -common_urns.java_class_lookup, -self._payload_builder.build(), +common_urns.java_class_lookup.urn, +self._payload_builder, self._expansion_service) @@ -384,7 +389,7 @@ class ExternalTransform(ptransform.PTransform): """ expansion_service = expansion_service or DEFAULT_EXPANSION_SERVICE if not urn and isinstance(payload, JavaClassLookupPayloadBuilder): - urn = common_urns.java_class_lookup + urn = common_urns.java_class_lookup.urn self._urn = urn self._payload = ( payload.payload() if isinstance(payload, PayloadBuilder) else payload)
[beam] branch master updated: Release script fixes
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3fbe369 Release script fixes
new 3de6bee Merge pull request #15544 from udim/master
3fbe369 is described below
commit 3fbe369714c6f7bcae6efb2e188c125463ade943
Author: Udi Meiri
AuthorDate: Tue Sep 21 10:38:57 2021 -0700
Release script fixes
---
.../src/main/scripts/build_release_candidate.sh| 33 --
.../main/scripts/deploy_release_candidate_pypi.sh | 8 --
2 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/release/src/main/scripts/build_release_candidate.sh
b/release/src/main/scripts/build_release_candidate.sh
index 76e2b73..8e60ea6 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -62,6 +62,16 @@ function usage() {
echo 'Usage: build_release_candidate.sh --release --rc
--github-user --signing-key [--debug]'
}
+function wipe_local_clone_dir() {
+ if [[ -d ${LOCAL_CLONE_DIR} ]]; then
+# Go modules leave behind directories with no write permissions, so readd
+# write perms to allow rm to work. Also, some files are owned by root and
+# will fail chmod, hence the `|| true`. Luckily they're world writable.
+chmod -R u+w ${LOCAL_CLONE_DIR} || true
+rm -rf ${LOCAL_CLONE_DIR}
+ fi
+}
+
RELEASE=
RC_NUM=
SIGNING_KEY=
@@ -161,9 +171,7 @@ if [[ $confirmation = "y" ]]; then
echo "Building and Staging Java Artifacts="
echo "Cloning Beam Repo and Checkout Release Tag---"
cd ~
- if [[ -d ${LOCAL_CLONE_DIR} ]]; then
-rm -rf ${LOCAL_CLONE_DIR}
- fi
+ wipe_local_clone_dir
mkdir -p ${LOCAL_CLONE_DIR}
cd ${LOCAL_CLONE_DIR}
git clone --depth 1 --branch "${RC_TAG}" ${GIT_REPO_URL} "${BEAM_ROOT_DIR}"
@@ -173,10 +181,17 @@ if [[ $confirmation = "y" ]]; then
git config credential.helper store
echo "-Staging Java Artifacts into Maven---"
+ # Cache the key/passphrase in gpg-agent by signing an arbitrary file.
gpg --local-user ${SIGNING_KEY} --output /dev/null --sign ~/.bashrc
- ./gradlew publish -Psigning.gnupg.keyName=${SIGNING_KEY} -PisRelease
--no-daemon
+ # Too many workers can overload (?) gpg-agent, causing gpg to prompt for a
+ # passphrase, and gradle doesn't play nice with pinentry.
+ # https://github.com/gradle/gradle/issues/11706
+ # --max-workers=6 works, but parallelism also seems to cause
+ # multiple Nexus repos to be created, so parallelism is disabled.
+ # https://issues.apache.org/jira/browse/BEAM-11813
+ ./gradlew publish -Psigning.gnupg.keyName=${SIGNING_KEY} -PisRelease
--no-daemon --no-parallel
echo "You need to close the staging repository manually on Apache Nexus. See
the release guide for instructions."
- rm -rf ~/${LOCAL_CLONE_DIR}
+ wipe_local_clone_dir
fi
echo "[Current Step]: Stage source release on dist.apache.org"
@@ -256,6 +271,7 @@ if [[ $confirmation = "y" ]]; then
echo '---Creating Python Virtualenv-'
python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
+ pip install -U pip
pip install requests python-dateutil
echo '--Fetching GitHub Actions Artifacts--'
@@ -306,9 +322,7 @@ read confirmation
if [[ $confirmation = "y" ]]; then
echo "Staging SDK docker images on docker hub="
cd ~
- if [[ -d ${LOCAL_CLONE_DIR} ]]; then
-rm -rf ${LOCAL_CLONE_DIR}
- fi
+ wipe_local_clone_dir
mkdir -p ${LOCAL_CLONE_DIR}
cd ${LOCAL_CLONE_DIR}
@@ -319,7 +333,7 @@ if [[ $confirmation = "y" ]]; then
./gradlew :pushAllDockerImages -Pdocker-pull-licenses
-Pdocker-tag=${RELEASE}_rc${RC_NUM}
- rm -rf ~/${LOCAL_CLONE_DIR}
+ wipe_local_clone_dir
fi
echo "[Current Step]: Update beam-site"
@@ -341,6 +355,7 @@ if [[ $confirmation = "y" ]]; then
python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
cd ${LOCAL_PYTHON_DOC}
+ pip install -U pip
pip install tox
git clone --branch "${RC_TAG}" --depth 1 ${GIT_REPO_URL}
cd ${BEAM_ROOT_DIR}
diff --git a/release/src/main/scripts/deploy_release_candidate_pypi.sh
b/release/src/main/scripts/deploy_release_candidate_pypi.sh
index 7502eb5..c52a993 100755
--- a/release/src/main/scripts/deploy_release_candidate_pypi.sh
+++ b/release/src/main/scripts/deploy_release_candidate_pypi.sh
@@ -127,6 +127,10 @@ RELEASE_COMMIT=$(git rev-list -n 1 $RC_TAG)
echo "Download python artifacts
[beam] branch master updated: Add 2.35.0 section to changelog.
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 25387ef Add 2.35.0 section to changelog. new 237d67d Merge pull request #15670 from ibzib/changes-2.35 25387ef is described below commit 25387ef5a98685f1e0ad88b37af1fe9ad141c766 Author: Kyle Weaver AuthorDate: Wed Oct 6 14:54:55 2021 -0700 Add 2.35.0 section to changelog. --- CHANGES.md | 30 ++ 1 file changed, 30 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 9909607..7507b53 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -49,6 +49,36 @@ * ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). --> +# [2.35.0] - Unreleased + +## Highlights + +* New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* New highly anticipated feature Y added to Java SDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + +## I/Os + +* Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## New Features / Improvements + +* X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Breaking Changes + +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Deprecations + +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Bugfixes + +* Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Known Issues + +* ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). # [2.34.0] - Unreleased * Add an [example](https://github.com/cometta/python-apache-beam-spark) of deploying Python Apache Beam job with Spark Cluster
[beam] 01/01: Set Dataflow container to release version.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch release-2.34.0
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 2bb924965f1160b20868b61efa81a4828ab99ed0
Author: Kyle Weaver
AuthorDate: Wed Oct 6 14:33:15 2021 -0700
Set Dataflow container to release version.
---
runners/google-cloud-dataflow-java/build.gradle | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 0416b1b..02f6b3c 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -45,8 +45,8 @@ processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
'dataflow.legacy_environment_major_version' : '8',
'dataflow.fnapi_environment_major_version' : '8',
-'dataflow.legacy_container_version' : 'beam-master-20210913',
-'dataflow.fnapi_container_version' : 'beam-master-20210913',
+'dataflow.legacy_container_version' : 'beam-2.34.0',
+'dataflow.fnapi_container_version' : 'beam-2.34.0',
'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3',
]
}
[beam] branch release-2.34.0 created (now 2bb9249)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch release-2.34.0 in repository https://gitbox.apache.org/repos/asf/beam.git. at 2bb9249 Set Dataflow container to release version. This branch includes the following new commits: new 2bb9249 Set Dataflow container to release version. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] branch master updated: Moving to 2.35.0-SNAPSHOT on master branch.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 371b4fd Moving to 2.35.0-SNAPSHOT on master branch.
371b4fd is described below
commit 371b4fd80ebb7b7618bc7dc606ec39cf4d5eb9f7
Author: Kyle Weaver
AuthorDate: Wed Oct 6 14:31:11 2021 -0700
Moving to 2.35.0-SNAPSHOT on master branch.
---
.../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 2 +-
gradle.properties | 4 ++--
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/python/apache_beam/version.py| 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7ef3a40..3f01717 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -378,7 +378,7 @@ class BeamModulePlugin implements Plugin {
// Automatically use the official release version if we are performing a
release
// otherwise append '-SNAPSHOT'
-project.version = '2.34.0'
+project.version = '2.35.0'
if (!isRelease(project)) {
project.version += '-SNAPSHOT'
}
diff --git a/gradle.properties b/gradle.properties
index e3a005d..c5e3f76 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository
signing.gnupg.executable=gpg
signing.gnupg.useLegacyGpg=true
-version=2.34.0-SNAPSHOT
-sdk_version=2.34.0.dev
+version=2.35.0-SNAPSHOT
+sdk_version=2.35.0.dev
javaVersion=1.8
diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go
index 74e42b4..b405bc6 100644
--- a/sdks/go/pkg/beam/core/core.go
+++ b/sdks/go/pkg/beam/core/core.go
@@ -27,5 +27,5 @@ const (
// SdkName is the human readable name of the SDK for UserAgents.
SdkName = "Apache Beam SDK for Go"
// SdkVersion is the current version of the SDK.
- SdkVersion = "2.34.0.dev"
+ SdkVersion = "2.35.0.dev"
)
diff --git a/sdks/python/apache_beam/version.py
b/sdks/python/apache_beam/version.py
index 264029d..435125c 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -17,4 +17,4 @@
"""Apache Beam SDK version information and utilities."""
-__version__ = '2.34.0.dev'
+__version__ = '2.35.0.dev'
[beam] branch master updated (64ec15f -> fa7065e)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 64ec15f Merge pull request #15585 Add dead letter pattern to Beam Python DoFns. new addc398 Fix "too many pings" errors. new 5d50ab9 Increase keepalive timeout to 5 minutes. new 1d0a16f Fix yapf complaints. new fa7065e Merge pull request #15314 from stefanistrate/stefan-fix-too-many-pings The 33154 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../runners/portability/abstract_job_service.py| 5 - .../portability/fn_api_runner/worker_handlers.py | 26 -- .../runners/portability/local_job_service.py | 13 ++- .../apache_beam/runners/worker/channel_factory.py | 5 - .../apache_beam/runners/worker/worker_pool_main.py | 4 +++- 5 files changed, 33 insertions(+), 20 deletions(-)
[beam] branch master updated: Fix whitespace lint
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new f3b67e7 Fix whitespace lint new 1965eba Merge pull request #15589 from TheNeuralBit/fix-whitespace f3b67e7 is described below commit f3b67e7d63eeadc4e99f736fd2724a270f5ae0a4 Author: Brian Hulette AuthorDate: Fri Sep 24 14:28:57 2021 -0700 Fix whitespace lint --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c686091..508303c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -123,7 +123,7 @@ * Code depending on beam imports need to include v2 on the module path. * Fix by'v2' to the import paths, turning `.../sdks/go/...` to `.../sdks/v2/go/...` * No other code change should be required to use v2.33.0 of the Go SDK. - + ## Deprecations * X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)).
[beam] branch master updated (314e260 -> b31a7ea)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 314e260 Merge pull request #15558 from y1chi/beam-12898 new d030576 [BEAM-12898] Use new postbuildscript dsl in Flink load tests new a84d9cb run postbuild regardless of test result new e27d8d6 spotless new b31a7ea Merge pull request #15547 from ibzib/BEAM-12898 The 33015 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .test-infra/jenkins/Flink.groovy | 23 ++- 1 file changed, 18 insertions(+), 5 deletions(-)
[beam] branch master updated: [BEAM-12898] Disable Flink Load tests which are leading Dataproc clusters/GCE VMs
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 645f3c8 [BEAM-12898] Disable Flink Load tests which are leading
Dataproc clusters/GCE VMs
new abfd4c6 Merge pull request #15519 from ajamato/master
645f3c8 is described below
commit 645f3c83484dd350c80f582eae8f91dff01fcd3d
Author: Alex Amato
AuthorDate: Wed Sep 15 16:43:49 2021 -0700
[BEAM-12898] Disable Flink Load tests which are leading Dataproc
clusters/GCE VMs
---
.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy | 3 ++-
.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy | 3 ++-
.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy | 3 ++-
.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 3 ++-
.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy | 6 --
.test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy | 3 ++-
.test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy | 3 ++-
7 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy
b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy
index 0143121..d0d3d4e 100644
--- a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy
+++ b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy
@@ -132,5 +132,6 @@
CronJobBuilder.cronJob('beam_LoadTests_Go_Combine_Flink_Batch', 'H 8 * * *', thi
influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
]
- loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
+ // TODO(BEAM-12898): Re-enable this test once fixed.
+ // loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
}
diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy
b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy
index 5055d1a..5cfe97e 100644
--- a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy
@@ -177,5 +177,6 @@
CronJobBuilder.cronJob('beam_LoadTests_Python_Combine_Flink_Streaming', 'H 18 *
influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
]
- loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'streaming')
+ // TODO(BEAM-12898): Re-enable this test once fixed.
+ // loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'streaming')
}
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy
b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy
index fee2e7d..f0cd611 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy
@@ -225,5 +225,6 @@ CronJobBuilder.cronJob('beam_LoadTests_Go_GBK_Flink_Batch',
'H 10 * * *', this)
influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
]
- loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
+ // TODO(BEAM-12898): Re-enable this test once fixed.
+ // loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
}
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy
b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy
index 567e327..0c86342 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy
@@ -146,5 +146,6 @@
CronJobBuilder.cronJob('beam_LoadTests_Go_ParDo_Flink_Batch', 'H 10 * * *', this
influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
]
- loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
+ // TODO(BEAM-12898): Re-enable this test once fixed.
+ // loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
}
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy
b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy
index 9b80c61..7c0e554 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy
@@ -350,7 +350,8 @@
CronJobBuilder.cronJob('beam_LoadTests_Python_ParDo_Flink_Batch', 'H 13 * * *',
influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
]
- loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT,
'batch')
+ // TODO(BEAM-12898): Re-enable this test once fixed.
+ // loadTestJob(delegate
[beam] branch master updated: [BEAM-12882] - fix test that is flaky when jenkins is overloaded
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new b5f1ccc [BEAM-12882] - fix test that is flaky when jenkins is overloaded new 3a3933c Merge pull request #15515 from dpcollins-google/fix-jenkins-only-failure b5f1ccc is described below commit b5f1ccc532a13bd90c941bfbe6697f04e307a2fb Author: Daniel Collins AuthorDate: Wed Sep 15 14:08:58 2021 -0400 [BEAM-12882] - fix test that is flaky when jenkins is overloaded --- .../beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java index e411d80..866e922 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionLoader.java @@ -92,9 +92,7 @@ class SubscriptionPartitionLoader extends PTransform SubscriptionPartition.of(subscription, kv.getValue(;
[beam] branch master updated: [BEAM-10459] Unignore numeric aggregation tests.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 05a0ae1 [BEAM-10459] Unignore numeric aggregation tests.
new d46cd9e Merge pull request #15517 from ibzib/BEAM-10459
05a0ae1 is described below
commit 05a0ae1836381d34cda1f51e9187cd3b655716fa
Author: Kyle Weaver
AuthorDate: Wed Sep 15 13:39:12 2021 -0700
[BEAM-10459] Unignore numeric aggregation tests.
These are passing now thanks to PR #15174.
---
.../beam/sdk/extensions/sql/zetasql/ZetaSqlMathFunctionsTest.java | 3 ---
1 file changed, 3 deletions(-)
diff --git
a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlMathFunctionsTest.java
b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlMathFunctionsTest.java
index 830bb84..92e509c 100644
---
a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlMathFunctionsTest.java
+++
b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlMathFunctionsTest.java
@@ -26,7 +26,6 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.Row;
import org.joda.time.Duration;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -1002,7 +1001,6 @@ public class ZetaSqlMathFunctionsTest extends
ZetaSqlTestBase {
}
@Test
- @Ignore("[BEAM-10459] Aggregation functions on NUMERIC is not supported yet")
public void testSumNumeric() {
String sql = "SELECT SUM(numeric_field) FROM table_with_numeric";
@@ -1019,7 +1017,6 @@ public class ZetaSqlMathFunctionsTest extends
ZetaSqlTestBase {
}
@Test
- @Ignore("[BEAM-10459] Aggregation functions on NUMERIC is not supported yet")
public void testAvgNumeric() {
String sql = "SELECT AVG(numeric_field) FROM table_with_numeric";
[beam] branch master updated (65f8946 -> 9a40e41)
This is an automated email from the ASF dual-hosted git repository. ibzib pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 65f8946 [BEAM-12543] Fix DataFrrame typo (#15509) add 9a40e41 [BEAM-12794] Remove obsolete uses of sys.exc_info. (#15507) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/runners/direct/executor.py | 20 ++-- sdks/python/apache_beam/runners/worker/data_plane.py | 17 + sdks/python/apache_beam/runners/worker/sdk_worker.py | 17 + 3 files changed, 16 insertions(+), 38 deletions(-)
