[beam] branch master updated: [BEAM-4522] Re-enable CalcRemoveRule
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 83ad43b [BEAM-4522] Re-enable CalcRemoveRule
new 0eba0e4 Merge pull request #7739: [BEAM-4522] Re-enable CalcRemoveRule
83ad43b is described below
commit 83ad43be80ee711e9e28be2323bd67f7d6ed5f89
Author: amaliujia
AuthorDate: Tue Feb 5 13:15:33 2019 -0800
[BEAM-4522] Re-enable CalcRemoveRule
---
.../apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java | 4 ++--
.../java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java | 6 +-
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
index 41c22f6..51367c6 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
@@ -40,6 +40,7 @@ import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.AggregateRemoveRule;
import org.apache.calcite.rel.rules.AggregateUnionAggregateRule;
import org.apache.calcite.rel.rules.CalcMergeRule;
+import org.apache.calcite.rel.rules.CalcRemoveRule;
import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
import org.apache.calcite.rel.rules.FilterCalcMergeRule;
import org.apache.calcite.rel.rules.FilterJoinRule;
@@ -73,8 +74,7 @@ public class BeamRuleSets {
ProjectCalcMergeRule.INSTANCE,
FilterToCalcRule.INSTANCE,
ProjectToCalcRule.INSTANCE,
- // https://issues.apache.org/jira/browse/BEAM-4522
- // CalcRemoveRule.INSTANCE,
+ CalcRemoveRule.INSTANCE,
CalcMergeRule.INSTANCE,
// push a filter into a join
diff --git
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
index c6968b3..0774039 100644
---
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
+++
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
@@ -232,11 +232,7 @@ public class BeamSqlCliTest {
+ "COMMENT '' LOCATION '/home/admin/orders'");
String plan = cli.explainQuery("select * from person");
-assertThat(
-plan,
-equalTo(
-"BeamCalcRel(expr#0..2=[{inputs}], proj#0..2=[{exprs}])\n"
-+ " BeamIOSourceRel(table=[[beam, person]])\n"));
+assertThat(plan, equalTo("BeamIOSourceRel(table=[[beam, person]])\n"));
}
@Test
[beam] branch master updated: Fix a bug in use of no_use_multiple_sdk_containers experiment flag. Setting this flag should prevent the addition of use_multiple_sdk_containers experiment.
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d8c67a6 Fix a bug in use of no_use_multiple_sdk_containers experiment
flag. Setting this flag should prevent the addition of
use_multiple_sdk_containers experiment.
new d85452e Merge pull request #7741 from aaltay/expb
d8c67a6 is described below
commit d8c67a6386605ef4c41d7153c2ea127084eda919
Author: Ahmet Altay
AuthorDate: Tue Feb 5 16:04:06 2019 -0800
Fix a bug in use of no_use_multiple_sdk_containers experiment flag.
Setting this flag should prevent the addition of
use_multiple_sdk_containers experiment.
---
.../runners/dataflow/internal/apiclient.py | 5 ++--
.../runners/dataflow/internal/apiclient_test.py| 28 ++
2 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index bad5222..8bcfbe1 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -184,8 +184,9 @@ class Environment(object):
# add the flag if 'no_use_multiple_sdk_containers' is present.
# TODO: Cleanup use_multiple_sdk_containers once we deprecate Python SDK
# till version 2.4.
- if ('use_multiple_sdk_containers' not in self.proto.experiments and
- 'no_use_multiple_sdk_containers' not in self.proto.experiments):
+ debug_options_experiments = self.debug_options.experiments
+ if ('use_multiple_sdk_containers' not in debug_options_experiments and
+ 'no_use_multiple_sdk_containers' not in debug_options_experiments):
self.debug_options.experiments.append('use_multiple_sdk_containers')
# Experiments
if self.debug_options.experiments:
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index d5d15f6..6614ab1 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -356,6 +356,34 @@ class UtilTest(unittest.TestCase):
self.assertEqual('key5', job.proto.labels.additionalProperties[4].key)
self.assertEqual('', job.proto.labels.additionalProperties[4].value)
+ def test_experiment_use_multiple_sdk_containers(self):
+pipeline_options = PipelineOptions(
+['--project', 'test_project', '--job_name', 'test_job_name',
+ '--temp_location', 'gs://test-location/temp',
+ '--experiments', 'beam_fn_api'])
+environment = apiclient.Environment(
+[], pipeline_options, 1, FAKE_PIPELINE_URL)
+self.assertIn("use_multiple_sdk_containers", environment.proto.experiments)
+
+pipeline_options = PipelineOptions(
+['--project', 'test_project', '--job_name', 'test_job_name',
+ '--temp_location', 'gs://test-location/temp',
+ '--experiments', 'beam_fn_api',
+ '--experiments', 'use_multiple_sdk_containers'])
+environment = apiclient.Environment(
+[], pipeline_options, 1, FAKE_PIPELINE_URL)
+self.assertIn("use_multiple_sdk_containers", environment.proto.experiments)
+
+pipeline_options = PipelineOptions(
+['--project', 'test_project', '--job_name', 'test_job_name',
+ '--temp_location', 'gs://test-location/temp',
+ '--experiments', 'beam_fn_api',
+ '--experiments', 'no_use_multiple_sdk_containers'])
+environment = apiclient.Environment(
+[], pipeline_options, 1, FAKE_PIPELINE_URL)
+self.assertNotIn(
+"use_multiple_sdk_containers", environment.proto.experiments)
+
if __name__ == '__main__':
unittest.main()
[beam] branch master updated (f31ae86 -> a071455)
This is an automated email from the ASF dual-hosted git repository. robertwb pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from f31ae86 Got rid of reference to windmill_service_streaming_rpc_batching experiment (#7728) new ab5d9bb Revert "Revert "Merge pull request #7316 [BEAM-6269] Cross-SDK transform expansion protocol."" new 5feff78 Use ints rather than strings for language boundary crossing. new e3dd68d Revert "Create a standard coder for utf-8 encoded strings." new 4dec940 Make grpc dependency optional. new a071455 [BEAM-6269] Cross-SDK transform expansion protocol. (#7729) The 19943 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: model/job-management/build.gradle | 5 +- .../src/main/proto/beam_expansion_api.proto| 68 +++ runners/core-construction-java/build.gradle| 18 ++ .../core/construction/ExpansionService.java| 205 +++ .../core/construction/RehydratedComponents.java| 11 ++ .../runners/core/construction/SdkComponents.java | 57 +- .../core/construction/ExpansionServiceTest.java| 103 ++ .../core/construction/TestExpansionService.java| 56 ++ .../beam/sdk/runners/TransformHierarchy.java | 28 ++- sdks/python/apache_beam/coders/coders.py | 32 +++ sdks/python/apache_beam/pipeline.py| 30 +-- sdks/python/apache_beam/pvalue.py | 12 +- .../python/apache_beam/runners/pipeline_context.py | 34 +++- .../apache_beam/runners/pipeline_context_test.py | 8 + .../runners/portability/expansion_service.py | 118 +++ sdks/python/apache_beam/transforms/__init__.py | 1 + sdks/python/apache_beam/transforms/external.py | 216 + .../python/apache_beam/transforms/external_test.py | 209 sdks/python/apache_beam/transforms/ptransform.py | 46 - sdks/python/build.gradle | 12 ++ 20 files changed, 1226 insertions(+), 43 deletions(-) create mode 100644 model/job-management/src/main/proto/beam_expansion_api.proto create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExpansionService.java create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ExpansionServiceTest.java create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestExpansionService.java create mode 100644 sdks/python/apache_beam/runners/portability/expansion_service.py create mode 100644 sdks/python/apache_beam/transforms/external.py create mode 100644 sdks/python/apache_beam/transforms/external_test.py
[beam] branch master updated: Got rid of reference to windmill_service_streaming_rpc_batching experiment (#7728)
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f31ae86 Got rid of reference to
windmill_service_streaming_rpc_batching experiment (#7728)
f31ae86 is described below
commit f31ae860892d8b4e1bd26df946acfe7185140d36
Author: drieber
AuthorDate: Tue Feb 5 15:28:49 2019 -0800
Got rid of reference to windmill_service_streaming_rpc_batching experiment
(#7728)
* There is no need to predicate
WindmillServiceStreamingRpcBatchLimitFactory on the
windmill_service_streaming_rpc_batching experiment. That experiment is now
fully rolled out.
* Fixed argument to streamingEngineEnabled.
---
.../dataflow/worker/options/StreamingDataflowWorkerOptions.java | 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java
index a136d7e..35c9b57 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java
@@ -200,11 +200,7 @@ public interface StreamingDataflowWorkerOptions extends
DataflowWorkerHarnessOpt
public Integer create(PipelineOptions options) {
StreamingDataflowWorkerOptions streamingOptions =
options.as(StreamingDataflowWorkerOptions.class);
- if (streamingEngineEnabled(streamingOptions)
- && hasExperiment(streamingOptions,
"windmill_service_streaming_rpc_batching")) {
-return Integer.MAX_VALUE;
- }
- return 1;
+ return streamingEngineEnabled(streamingOptions) ? Integer.MAX_VALUE : 1;
}
}
}
[beam] branch asf-site updated: Publishing website 2019/02/05 23:11:30 at commit 8ee1a27
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/asf-site by this push: new ed56178 Publishing website 2019/02/05 23:11:30 at commit 8ee1a27 ed56178 is described below commit ed56178605087697de9c71466ef9cfde27e7c8fc Author: jenkins AuthorDate: Tue Feb 5 23:11:31 2019 + Publishing website 2019/02/05 23:11:30 at commit 8ee1a27 --- .../documentation/io/built-in/google-bigquery/index.html| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/generated-content/documentation/io/built-in/google-bigquery/index.html b/website/generated-content/documentation/io/built-in/google-bigquery/index.html index 47dd15c..abd41a4 100644 --- a/website/generated-content/documentation/io/built-in/google-bigquery/index.html +++ b/website/generated-content/documentation/io/built-in/google-bigquery/index.html @@ -418,7 +418,7 @@ a string, or use a https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/index.html?com/google/api/services/bigquery/model/TableReference.html;>TableReference - https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.table.TableReference.html#google.cloud.bigquery.table.TableReference;>TableReference + https://github.com/googleapis/google-cloud-python/blob/master/bigquery/google/cloud/bigquery/table.py#L153;>TableReference object.
[beam] branch master updated: Fix broken link in BigQuery Python connector docs
This is an automated email from the ASF dual-hosted git repository. ccy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 52e1265 Fix broken link in BigQuery Python connector docs new 8ee1a27 Merge pull request #7740 from charlesccychen/fix-broken-bq 52e1265 is described below commit 52e1265b63457ccf8d2936d229f82830786178b5 Author: Charles Chen AuthorDate: Tue Feb 5 11:17:39 2019 -0800 Fix broken link in BigQuery Python connector docs --- website/src/documentation/io/built-in-google-bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/documentation/io/built-in-google-bigquery.md b/website/src/documentation/io/built-in-google-bigquery.md index 2edb363..72fa326 100644 --- a/website/src/documentation/io/built-in-google-bigquery.md +++ b/website/src/documentation/io/built-in-google-bigquery.md @@ -95,7 +95,7 @@ a string, or use a [TableReference](https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/index.html?com/google/api/services/bigquery/model/TableReference.html) - [TableReference](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.table.TableReference.html#google.cloud.bigquery.table.TableReference) + [TableReference](https://github.com/googleapis/google-cloud-python/blob/master/bigquery/google/cloud/bigquery/table.py#L153) object.
svn commit: r32355 - /dev/beam/2.10.0/python/
Author: kenn Date: Tue Feb 5 22:56:38 2019 New Revision: 32355 Log: Sign Beam 2.10.0 RC2 wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.asc dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.sha512 dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.asc dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.sha512 dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl.asc dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl.sha512 dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl.asc dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl.sha512 dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl.asc dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl.sha512 Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.asc == --- dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.asc (added) +++ dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.asc Tue Feb 5 22:56:38 2019 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEA9uj5qvdBL/RVY3BbtVRqK4CRhwFAlxaFIwACgkQbtVRqK4C +Rhz75hAAoxxH+4Bo8jaoqAtwLHUZlnxRUMqB0We6K26DtJ74WIy/5IV3/Q8dx6gC +Rs71i6XbJeNAAQ+GdD2f4w4G9ZsCvtCNuo1dXZrS8IO3AnrZ0c2px06pmm0s4nAu +KeX+lWv0LE5pef04q0/idbvs4gQTi2j6ByPnurolcuVhLKnQ4GVcDFIGpS4/CBqc +gQFLPWktOTSbmJLR99+N3P93dQMo4/TeNk/OQZR0YHzN5iJYstMh9o0dJ3f1XnaQ +J++tsmqa/cMlTATIsycwwoUKakucQUeSU6DhAu+dEfE9OLf1/R4YqKMawxWWAxK3 +d/eO2huhuE8SUAZcZCxrFiO7gpn1FJ5GbBSrNH9j1Q758PZQ5+61Kzu0nYhaOnJw +eCebPbUjcDs++K3eP6xx3LIlV1QtIag4lG3RkubNT0wMpK2nd5kFbDUG0dF0bzyw +nsbUzjzCxBGu0OY0aAOuzVnpUr8t8mhsqefOvE1/dYPXWT8NUW8euTf/wv6WmKkE +5aCHRuHJ1WSON3tw+BuSDmR8TIj49GKh1TyH8z0oJ2SgvnQqvGU8tQj0V/KoBwwU +NRGGg20YlEvRYmoKNFQRvQDegogJMmPXmNEg9AweB+UMLGO1VVbxkr3N91LTqkQ5 +uXbjuEp27Mxeeoac1giETyd2BwKOqzk0otmmdsotyj5Zs6tLT9Y= +=ssbO +-END PGP SIGNATURE- Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.sha512 == --- dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.sha512 (added) +++ dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl.sha512 Tue Feb 5 22:56:38 2019 @@ -0,0 +1 @@ +cfb3e2d70090d73162f25bbd95b01be06b9da0757b6f6382982519828c407c48f2abf2a36af46215ffd29bc69c1b9b0bb54381d7ea04be44cd89233e51c84282 apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.asc == --- dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.asc (added) +++ dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.asc Tue Feb 5 22:56:38 2019 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEA9uj5qvdBL/RVY3BbtVRqK4CRhwFAlxaFJAACgkQbtVRqK4C +Rhz41Q/9Gq+P7zxVnVI4nEccQbZHd9EBhpnYiK9JUB8w95lu/6NDV+Pi4pmHOAnU +/rDl9R9pvGxtCCSjBT41MbkdJhZgoGeM0R3wB+ePqhDDEsNi/Bm1RhbQeCL+mcVA +v4xPNFFfz9L0mM7PAfLgasbulXubNO+a5wUtbH80Iw8KVRYHnxnxgldynjQxnNw2 +x2asPUrIugjyULqt7WkBzb6Rndba33q5j8JPpEu8utqsPvfuNb8d+JpkOEr3Vt5+ +AWMEmmO+3/dUpy7cL5LgksA70BGzcPj4HYQLzf/X6tlpV1OmEpZnOqY3mlLNF4a7 +gz6V5ZmlpMGIDtJ31h4HkkDUn3XPGl/6hL22MlCb5BpnszqUXyd4qxzc0+jwvaOG +YNO2pc6r8FN7udymtRCwaieoX2Xl6NbIt8JaB/tgLO3tO8Wt+FvkR8D++LDAsnxt +5RiTCrY9xhL0+O4KNwv2jqNnN3tHOWGU5N0LTqan3sGfJGlCAA843Ziht9HXN0QE +C1nFHqFKJlno/FbRw0Zcb+kSJxhyKci9sYCXKJ1QOyjk6sPLcAWPc+s3CJ9cSHPx +687K9MFedtzRwF3prCcyk7ddR1WW+o6N3fMGW0NWl5zAj1e6CynEpXVs/f/OxEpH +blsZf2cff+PovNj/qbqa8MonJnnZ3pSAsIgxf/tt+g4fuiNdPzU= +=yTOV +-END PGP SIGNATURE- Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.sha512 == --- dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl.sha512 (added) +++
[beam] branch master updated: Update starcgen for new types
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 1f9cc01 Update starcgen for new types
new 212a2ab Merge pull request #7738 from lostluck/fixshims
1f9cc01 is described below
commit 1f9cc0185d69c107fdf1fd4c5c4ec9735c34fcb4
Author: Robert Burke
AuthorDate: Tue Feb 5 20:45:06 2019 +
Update starcgen for new types
---
sdks/go/pkg/beam/beam.shims.go | 84 +-
.../pkg/beam/core/runtime/coderx/coderx.shims.go | 181 ++---
sdks/go/pkg/beam/testing/passert/passert.shims.go | 19 +--
sdks/go/pkg/beam/transforms/filter/filter.shims.go | 1 -
sdks/go/pkg/beam/transforms/stats/stats.shims.go | 17 +-
sdks/go/pkg/beam/transforms/top/top.shims.go | 17 +-
sdks/go/pkg/beam/util/shimx/generate.go| 21 ++-
sdks/go/pkg/beam/util/shimx/generate_test.go | 22 ++-
sdks/go/pkg/beam/util/starcgenx/starcgenx.go | 23 ++-
sdks/go/pkg/beam/util/starcgenx/starcgenx_test.go | 94 ++-
sdks/go/pkg/beam/x/debug/debug.shims.go| 35 ++--
11 files changed, 312 insertions(+), 202 deletions(-)
diff --git a/sdks/go/pkg/beam/beam.shims.go b/sdks/go/pkg/beam/beam.shims.go
index 8c95289..67d5c71 100644
--- a/sdks/go/pkg/beam/beam.shims.go
+++ b/sdks/go/pkg/beam/beam.shims.go
@@ -42,16 +42,14 @@ func init() {
runtime.RegisterFunction(swapKVFn)
runtime.RegisterType(reflect.TypeOf((*createFn)(nil)).Elem())
runtime.RegisterType(reflect.TypeOf((*reflect.Type)(nil)).Elem())
- runtime.RegisterType(reflect.TypeOf((*typex.T)(nil)).Elem())
- runtime.RegisterType(reflect.TypeOf((*typex.X)(nil)).Elem())
- runtime.RegisterType(reflect.TypeOf((*typex.Y)(nil)).Elem())
+ runtime.RegisterType(reflect.TypeOf((*reflectx.Func)(nil)).Elem())
reflectx.RegisterStructWrapper(reflect.TypeOf((*createFn)(nil)).Elem(),
wrapMakerCreateFn)
- reflectx.RegisterFunc(reflect.TypeOf((*func(reflect.Type,[]byte)
(typex.T,error))(nil)).Elem(), funcMakerReflect۰TypeSliceofByteГTypex۰TError)
- reflectx.RegisterFunc(reflect.TypeOf((*func([]byte,func(typex.T))
(error))(nil)).Elem(), funcMakerSliceofByteEmitTypex۰TГError)
- reflectx.RegisterFunc(reflect.TypeOf((*func([]typex.T,func(typex.T))
())(nil)).Elem(), funcMakerSliceofTypex۰TEmitTypex۰TГ)
- reflectx.RegisterFunc(reflect.TypeOf((*func(string,reflect.Type,[]byte)
(reflectx.Func))(nil)).Elem(),
funcMakerStringReflect۰TypeSliceofByteГReflectx۰Func)
+ reflectx.RegisterFunc(reflect.TypeOf((*func(reflect.Type,[]byte)
(typex.T,error))(nil)).Elem(), funcMakerReflect۰TypeSliceOfByteГTypex۰TError)
+ reflectx.RegisterFunc(reflect.TypeOf((*func([]byte,func(typex.T))
(error))(nil)).Elem(), funcMakerSliceOfByteEmitTypex۰TГError)
+ reflectx.RegisterFunc(reflect.TypeOf((*func([]typex.T,func(typex.T))
())(nil)).Elem(), funcMakerSliceOfTypex۰TEmitTypex۰TГ)
+ reflectx.RegisterFunc(reflect.TypeOf((*func(string,reflect.Type,[]byte)
(reflectx.Func))(nil)).Elem(),
funcMakerStringReflect۰TypeSliceOfByteГReflectx۰Func)
reflectx.RegisterFunc(reflect.TypeOf((*func(typex.T)
(int,typex.T))(nil)).Elem(), funcMakerTypex۰TГIntTypex۰T)
- reflectx.RegisterFunc(reflect.TypeOf((*func(typex.T)
([]byte,error))(nil)).Elem(), funcMakerTypex۰TГSliceofByteError)
+ reflectx.RegisterFunc(reflect.TypeOf((*func(typex.T)
([]byte,error))(nil)).Elem(), funcMakerTypex۰TГSliceOfByteError)
reflectx.RegisterFunc(reflect.TypeOf((*func(typex.X,typex.Y)
(typex.X))(nil)).Elem(), funcMakerTypex۰XTypex۰YГTypex۰X)
reflectx.RegisterFunc(reflect.TypeOf((*func(typex.X,typex.Y)
(typex.Y))(nil)).Elem(), funcMakerTypex۰XTypex۰YГTypex۰Y)
reflectx.RegisterFunc(reflect.TypeOf((*func(typex.X,typex.Y)
(typex.Y,typex.X))(nil)).Elem(), funcMakerTypex۰XTypex۰YГTypex۰YTypex۰X)
@@ -65,107 +63,107 @@ func wrapMakerCreateFn(fn interface{})
map[string]reflectx.Func {
}
}
-type callerReflect۰TypeSliceofByteГTypex۰TError struct {
+type callerReflect۰TypeSliceOfByteГTypex۰TError struct {
fn func(reflect.Type,[]byte) (typex.T,error)
}
-func funcMakerReflect۰TypeSliceofByteГTypex۰TError(fn interface{})
reflectx.Func {
+func funcMakerReflect۰TypeSliceOfByteГTypex۰TError(fn interface{})
reflectx.Func {
f := fn.(func(reflect.Type,[]byte) (typex.T,error))
- return ۰TypeSliceofByteГTypex۰TError{fn: f}
+ return ۰TypeSliceOfByteГTypex۰TError{fn: f}
}
-func (c *callerReflect۰TypeSliceofByteГTypex۰TError) Name() string {
+func (c *callerReflect۰TypeSliceOfByteГTypex۰TError) Name() string {
return reflectx.FunctionName(c.fn)
}
-func (c *callerReflect۰TypeSliceofByteГTypex۰TError) Type() reflect.Type {
+func (c *callerReflect۰TypeSliceOfByteГTypex۰TError) Type() reflect.Type {
svn commit: r32354 - /dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl
Author: kenn Date: Tue Feb 5 22:53:58 2019 New Revision: 32354 Log: Upload python wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl (with props) Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_i686.whl -- svn:mime-type = application/octet-stream
svn commit: r32353 - /dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl
Author: kenn Date: Tue Feb 5 22:51:25 2019 New Revision: 32353 Log: Upload python wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl (with props) Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27mu-manylinux1_x86_64.whl -- svn:mime-type = application/octet-stream
svn commit: r32352 - /dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl
Author: kenn Date: Tue Feb 5 22:50:20 2019 New Revision: 32352 Log: Upload python wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl (with props) Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl -- svn:mime-type = application/octet-stream
svn commit: r32351 - /dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl
Author: kenn Date: Tue Feb 5 22:50:05 2019 New Revision: 32351 Log: Upload python wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl (with props) Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_x86_64.whl -- svn:mime-type = application/octet-stream
svn commit: r32350 - /dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl
Author: kenn Date: Tue Feb 5 22:49:30 2019 New Revision: 32350 Log: Upload python wheels Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl (with props) Added: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache_beam-2.10.0-cp27-cp27m-manylinux1_i686.whl -- svn:mime-type = application/octet-stream
svn commit: r32349 - in /dev/beam/2.10.0/python: ./ apache-beam-2.10.0.zip apache-beam-2.10.0.zip.asc apache-beam-2.10.0.zip.sha512
Author: kenn Date: Tue Feb 5 22:15:16 2019 New Revision: 32349 Log: Add Python artifacts for 2.10.0 RC2 Added: dev/beam/2.10.0/python/ dev/beam/2.10.0/python/apache-beam-2.10.0.zip (with props) dev/beam/2.10.0/python/apache-beam-2.10.0.zip.asc dev/beam/2.10.0/python/apache-beam-2.10.0.zip.sha512 Added: dev/beam/2.10.0/python/apache-beam-2.10.0.zip == Binary file - no diff available. Propchange: dev/beam/2.10.0/python/apache-beam-2.10.0.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.10.0/python/apache-beam-2.10.0.zip.asc == --- dev/beam/2.10.0/python/apache-beam-2.10.0.zip.asc (added) +++ dev/beam/2.10.0/python/apache-beam-2.10.0.zip.asc Tue Feb 5 22:15:16 2019 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEA9uj5qvdBL/RVY3BbtVRqK4CRhwFAlxaCYkACgkQbtVRqK4C +RhyHsQ//SMFFdt2B2ZHSed6pKGgHAfEsJd5mMLcZ3AB7rTcupAsyIRh9AdvLXoWh +Db+Ef3XzJceNr0K6e/cXbofkocR4CH0tV4EPlpLPvc3c2Pfa/Drwcb3UVnc5ioiV +6F/t/wLofMKQ/9949lKclEAitfUAn7DhlLyZnTlzfQ1ELRI0WlljXqtcTl8XXd+c +kBeF3yvVoSEG68ivo1eqVOmjgUByQ2JKC1mfV3UPPuIXJ7RTkEmPKERxfQ8iH6fm +D+cknyuIFTq+Dbbx31foXvKjqgO/hGTHCT6g4vRtxyLU2P4AhWt33W0cywWDiPNx +ua+f/44nnoo2lnUbGYTTtmshFDoI75yKaO66X78IoNj3pxt0QyzrMhTsBe23kkVh +DSBF20XvjOPngI9A3+VOM5F22rIJyyAFSQ1ThaPwMOx2wFGIIpV/v12PkJPGWl2B +5KjhxLFcSy1Cr/FIh8zddbTV6Hj7pX1N9/JalUZddP6r3D8dRqPNxXNN0LY99Cda +MICoF2AxLgFMsIFT3btuAF9SB0wl7JZR2lIMbH7kF6xhc+/CaMjn46B40GgatQMd +swr4hMxD692zCFJqWLopI4v8aOn/0F0p3l9dAyQsuoGq0GELcm8q61zHmhqPyFEk +viCT1dZMcyXAiJggi1M7SVlZwyAKPyycNn8kX4ytDneWcERrAyY= +=YPqD +-END PGP SIGNATURE- Added: dev/beam/2.10.0/python/apache-beam-2.10.0.zip.sha512 == --- dev/beam/2.10.0/python/apache-beam-2.10.0.zip.sha512 (added) +++ dev/beam/2.10.0/python/apache-beam-2.10.0.zip.sha512 Tue Feb 5 22:15:16 2019 @@ -0,0 +1 @@ +337dd359aa8a6162256b1adedd1d8e7cbf58f3404a22706ffc23391eb4bcc812fcc5a09686243bbb4f1d6fc74c69bb0bb7341cebb445a4a08d5d06f7151899f8 apache-beam-2.10.0.zip
svn commit: r32348 - in /dev/beam/2.10.0: ./ apache-beam-2.10.0-source-release.zip apache-beam-2.10.0-source-release.zip.asc apache-beam-2.10.0-source-release.zip.sha512
Author: kenn Date: Tue Feb 5 22:08:41 2019 New Revision: 32348 Log: Stage Beam 2.10.0 RC2 Added: dev/beam/2.10.0/ dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip (with props) dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.asc dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.sha512 Added: dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.asc == --- dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.asc (added) +++ dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.asc Tue Feb 5 22:08:41 2019 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEA9uj5qvdBL/RVY3BbtVRqK4CRhwFAlxaCUEACgkQbtVRqK4C +RhzQWQ//X5EG23GLXhaMgXIDNiyTAIdGahkkHfqB+od4mpkTwgfzQJTGMSvJPYD4 +fq+IkVmeoLyoXRq/sFCtCQZQNkTDjepAqOiGv4k+q5B1kKcigouoV6BnbpV3JJqE +K1RIo8oE7//I22szRt1OGpSDL9kDpyqbhNhEy5tBvejUUH+bV7VHjJgrfVL4JIKY +axhDzZdbUw9kPzvjAvG/HUuGP1s1KNkoeRm5quRJlFcUAIVYU9jkQ0OxP/DgnbJ6 +cA5g0iU5DNfvp7h8Va9l9nGUzD1IMYxpGfqNpiFCKLWZtJFjYShUQgNO38dm9qb4 +dzCtGqX0c7IZZ9ijqjMMbgGbNVz4Je8eyanAVEor+y1aES6aFqECODHVxVkotMbx +v3CRusiRI9JzsUnZ4RJVjeELin/9RGmDES0ZobcaVDdNlRnlGEPqXUp5UedkFxQU +jcIkL6BEfTvMG8og8r4TCHpXzhPtswkyscCQamm8Z7/YB3zvP6ofhHeV6S/9z4HT +bCGB+VspF1VQqsOinIzGTTlImwSUK/byfycbWTfGBC83OPuKZ7vRu3y6s7V6aWyO +O+TNRTV50jFdVmtkQwL2DwqaD/EkCWgKHd40qDZtYQLnBTMLtIyxb3tgoT3LjMXf +UbdgWNFcjNP/fkjvLYz7MLnbSvr7rffeAqsNYBtPSKPAjCM5UK8= +=6wrX +-END PGP SIGNATURE- Added: dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.sha512 == --- dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.sha512 (added) +++ dev/beam/2.10.0/apache-beam-2.10.0-source-release.zip.sha512 Tue Feb 5 22:08:41 2019 @@ -0,0 +1 @@ +ba2c05c9d8b205d09bf5031c4ce8b100027536a622dda8de2ebe8a7d1468e164b58e508c37368ba945577514213f1336d0c2b6863be96759a535abe97b9345e8 apache-beam-2.10.0-source-release.zip
[beam] branch master updated (63dbbf4 -> 25051a5)
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 63dbbf4 [BEAM-3742] Checkpointing for SDF over FnAPI in Python SDK
and FnApiRunner. (#7722)
new bf93dcc Py3 IT on DirectRunner in PreCommit
new d404bc0 fixup! fix scripts directory
new 25051a5 Merge pull request #7734 from markflyhigh/py3-wc-direct
The 19933 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 31 +-
sdks/python/build.gradle | 28 +--
sdks/python/precommit/dataflow/build.gradle| 4 ++-
.../{dataflow => direct/py3}/build.gradle | 21 ---
settings.gradle| 2 ++
6 files changed, 48 insertions(+), 39 deletions(-)
copy sdks/python/precommit/{dataflow => direct/py3}/build.gradle (63%)
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 81dc056 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. 81dc056 is described below commit 81dc0561f488265035d166894ded775954785ab1 Author: Kenneth Knowles AuthorDate: Tue Feb 5 13:45:28 2019 -0800 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e8bb128..1dcfd01 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-RC2 +version=2.10.0-SNAPSHOT
[beam] annotated tag v2.10.0-RC2 updated (cedadcb -> f586b9b)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was modified! *** from cedadcb (commit) to f586b9b (tag) tagging cedadcb445273b3faaa6d01d15873cee60506c24 (commit) replaces v2.10.0-RC1 by Kenneth Knowles on Tue Feb 5 13:45:21 2019 -0800 - Log - [Gradle Release Plugin] - creating tag: 'v2.10.0-RC2'. --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new cedadcb [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. cedadcb is described below commit cedadcb445273b3faaa6d01d15873cee60506c24 Author: Kenneth Knowles AuthorDate: Tue Feb 5 13:45:15 2019 -0800 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1dcfd01..e8bb128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-SNAPSHOT +version=2.10.0-RC2
[beam] annotated tag v2.10.0-RC2 deleted (was e3fd03c)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was deleted! *** tag was e3fd03c This change permanently discards the following revisions: discard 868d282 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
[beam] branch release-2.10.0 updated (0288262 -> 8e610ad)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git. discard 0288262 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. omit 868d282 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/release-2.10.0 (8e610ad) \ O -- O -- O (0288262) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 0288262 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. 0288262 is described below commit 0288262d6cd2bccf51db1b79f300469b18d10f08 Author: Kenneth Knowles AuthorDate: Tue Feb 5 13:31:37 2019 -0800 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e8bb128..1dcfd01 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-RC2 +version=2.10.0-SNAPSHOT
[beam] annotated tag v2.10.0-RC2 updated (868d282 -> e3fd03c)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was modified! *** from 868d282 (commit) to e3fd03c (tag) tagging 868d282c428c192d359519c10803e196f02fe9d1 (commit) replaces v2.10.0-RC1 by Kenneth Knowles on Tue Feb 5 13:31:30 2019 -0800 - Log - [Gradle Release Plugin] - creating tag: 'v2.10.0-RC2'. --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 868d282 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. 868d282 is described below commit 868d282c428c192d359519c10803e196f02fe9d1 Author: Kenneth Knowles AuthorDate: Tue Feb 5 13:31:24 2019 -0800 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1dcfd01..e8bb128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-SNAPSHOT +version=2.10.0-RC2
[beam] branch master updated: [BEAM-3742] Checkpointing for SDF over FnAPI in Python SDK and FnApiRunner. (#7722)
This is an automated email from the ASF dual-hosted git repository. robertwb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 63dbbf4 [BEAM-3742] Checkpointing for SDF over FnAPI in Python SDK and FnApiRunner. (#7722) 63dbbf4 is described below commit 63dbbf4affc2082015d35f27f393f1b6d0a0493b Author: Robert Bradshaw AuthorDate: Tue Feb 5 22:27:24 2019 +0100 [BEAM-3742] Checkpointing for SDF over FnAPI in Python SDK and FnApiRunner. (#7722) --- sdks/python/apache_beam/io/restriction_trackers.py | 12 +- sdks/python/apache_beam/portability/common_urns.py | 2 + sdks/python/apache_beam/runners/common.pxd | 3 + sdks/python/apache_beam/runners/common.py | 46 ++- .../apache_beam/runners/direct/direct_runner.py| 4 - .../runners/direct/sdf_direct_runner_test.py | 3 +- .../runners/portability/flink_runner_test.py | 3 + .../runners/portability/fn_api_runner.py | 40 -- .../runners/portability/fn_api_runner_test.py | 38 ++ .../portability/fn_api_runner_transforms.py| 137 +++-- .../apache_beam/runners/worker/bundle_processor.py | 126 +-- .../apache_beam/runners/worker/operations.pxd | 6 + .../apache_beam/runners/worker/operations.py | 16 ++- .../apache_beam/runners/worker/sdk_worker.py | 3 +- sdks/python/apache_beam/transforms/core.py | 10 ++ 15 files changed, 405 insertions(+), 44 deletions(-) diff --git a/sdks/python/apache_beam/io/restriction_trackers.py b/sdks/python/apache_beam/io/restriction_trackers.py index b9b1f17..e72d508 100644 --- a/sdks/python/apache_beam/io/restriction_trackers.py +++ b/sdks/python/apache_beam/io/restriction_trackers.py @@ -80,8 +80,9 @@ class OffsetRestrictionTracker(RestrictionTracker): self._range = OffsetRange(start_position, stop_position) self._current_position = None self._last_claim_attempt = None +self._deferred_residual = None self._checkpointed = False -self._lock = threading.Lock() +self._lock = threading.RLock() def check_done(self): with self._lock: @@ -139,3 +140,12 @@ class OffsetRestrictionTracker(RestrictionTracker): self._range = OffsetRange(self._range.start, end_position) return residual_range + + def defer_remainder(self, watermark=None): +with self._lock: + self._deferred_watermark = watermark + self._deferred_residual = self.checkpoint() + + def deferred_status(self): +if self._deferred_residual: + return (self._deferred_residual, self._deferred_watermark) diff --git a/sdks/python/apache_beam/portability/common_urns.py b/sdks/python/apache_beam/portability/common_urns.py index 4ee32a7..a000f81 100644 --- a/sdks/python/apache_beam/portability/common_urns.py +++ b/sdks/python/apache_beam/portability/common_urns.py @@ -49,6 +49,8 @@ composites = PropertiesFromEnumType( beam_runner_api_pb2.StandardPTransforms.Composites) combine_components = PropertiesFromEnumType( beam_runner_api_pb2.StandardPTransforms.CombineComponents) +sdf_components = PropertiesFromEnumType( +beam_runner_api_pb2.StandardPTransforms.SplittableParDoComponents) side_inputs = PropertiesFromEnumType( beam_runner_api_pb2.StandardSideInputTypes.Enum) diff --git a/sdks/python/apache_beam/runners/common.pxd b/sdks/python/apache_beam/runners/common.pxd index 49f4c44..b5ab88d 100644 --- a/sdks/python/apache_beam/runners/common.pxd +++ b/sdks/python/apache_beam/runners/common.pxd @@ -82,6 +82,9 @@ cdef class PerWindowInvoker(DoFnInvoker): cdef bint has_windowed_inputs cdef bint cache_globally_windowed_args cdef object process_method + cdef bint is_splittable + cdef object restriction_tracker + cdef WindowedValue current_windowed_value cdef class DoFnRunner(Receiver): diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index 99a4bca..3d9b07f 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -211,7 +211,7 @@ class DoFnSignature(object): self.start_bundle_method = MethodWrapper(do_fn, 'start_bundle') self.finish_bundle_method = MethodWrapper(do_fn, 'finish_bundle') -restriction_provider = self._get_restriction_provider(do_fn) +restriction_provider = self.get_restriction_provider() self.initial_restriction_method = ( MethodWrapper(restriction_provider, 'initial_restriction') if restriction_provider else None) @@ -237,7 +237,7 @@ class DoFnSignature(object): method = timer_spec._attached_callback self.timer_methods[timer_spec] = MethodWrapper(do_fn, method.__name__) - def _get_restriction_provider(self, do_fn): + def get_restriction_provider(self): result = _find_param_with_default(self.process_method,
[beam] branch master updated: [BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b05ccd2 [BEAM-5091] Run wordcount_it in Python 3 in
ValidatesContainer test
new f85d953 Merge pull request #7706 from markflyhigh/py3-wordcount-it
b05ccd2 is described below
commit b05ccd2c63c1af66ceac833fd3e6dc43f9a26329
Author: Mark Liu
AuthorDate: Fri Feb 1 15:56:56 2019 -0800
[BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
---
.../python/container/run_validatescontainer_py3.sh | 40 --
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git a/sdks/python/container/run_validatescontainer_py3.sh
b/sdks/python/container/run_validatescontainer_py3.sh
index 35f3d6c..f852a41 100755
--- a/sdks/python/container/run_validatescontainer_py3.sh
+++ b/sdks/python/container/run_validatescontainer_py3.sh
@@ -47,13 +47,16 @@ gcloud -v
# Build the container image
TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python
+CONTAINER=us.gcr.io/$PROJECT/$USER/python3
echo "Building Python 3 container $CONTAINER"
./gradlew :beam-sdks-python-container-py3:docker
-Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Ppython3
--info
# Verify it exists
docker images | grep $TAG
+# Push the container
+gcloud docker -- push $CONTAINER
+
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
@@ -61,4 +64,37 @@ function cleanup_container {
}
trap cleanup_container EXIT
-echo "Successfully built Python 3 container $CONTAINER"
+echo ">>> Successfully built Python 3 container $CONTAINER"
+
+# Python 3 virtualenv for the rest of the script to run setup & e2e test
+virtualenv sdks/python/container/py3/venv -p python3.5
+. sdks/python/container/py3/venv/bin/activate
+cd sdks/python
+pip install -e .[gcp,test]
+
+# Install test dependencies for ValidatesContainer tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+
+# Create a tarball
+python setup.py sdist
+SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+
+# Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
+python setup.py nosetests \
+ --attr Py3IT \
+ --nologcapture \
+ --processes=1 \
+ --process-timeout=900 \
+ --test-pipeline-options=" \
+--runner=TestDataflowRunner \
+--project=$PROJECT \
+--worker_harness_container_image=$CONTAINER:$TAG \
+--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
+--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
+--output=$GCS_LOCATION/output \
+--sdk_location=$SDK_LOCATION \
+--requirements_file=postcommit_requirements.txt \
+--num_workers=1"
+
+echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
[beam] branch release-2.10.0 updated (0144f54 -> 8e610ad)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git. discard 0144f54 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. discard d14c60d [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/release-2.10.0 (8e610ad) \ O -- O -- O (0144f54) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[beam] annotated tag v2.10.0-RC2 deleted (was d322e8f)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was deleted! *** tag was d322e8f The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[beam] branch master updated (c3b0f45 -> 696cf22)
This is an automated email from the ASF dual-hosted git repository. scott pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from c3b0f45 Merge pull request #7699: [BEAM-1132] add jacoco report on javaPreCommit new 2aa8a9f [BEAM-6431] Implement Execution Time metrics start,process,finish in the Java SDK new 4a785ba spotless and others new 887cf8a spotless new 0479831 added some test new 67bc9a4 Add more tests and the SimpleStateRegistry new eb6b4dd Iterated on PR new 696cf22 Merge pull request #7676: [BEAM-6431] Implement Execution Time metrics start,process,finish in the Java SDK The 19927 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../runners/core/metrics/SimpleExecutionState.java | 60 + .../core/metrics/SimpleMonitoringInfoBuilder.java | 14 +++- .../runners/core/metrics/SimpleStateRegistry.java | 49 +++ .../core/metrics/MonitoringInfoMatchers.java | 98 ++ .../core/metrics/SimpleExecutionStateTest.java | 56 + .../core/metrics/SimpleStateRegistryTest.java | 78 + runners/java-fn-execution/build.gradle | 1 + .../fnexecution/control/RemoteExecutionTest.java | 75 - .../java/org/apache/beam/fn/harness/FnHarness.java | 1 + .../fn/harness/control/ProcessBundleHandler.java | 40 ++--- .../harness/data/PCollectionConsumerRegistry.java | 34 +++- .../harness/data/PTransformFunctionRegistry.java | 49 --- .../beam/fn/harness/AssignWindowsRunnerTest.java | 5 +- .../beam/fn/harness/BeamFnDataReadRunnerTest.java | 11 ++- .../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 12 ++- .../beam/fn/harness/BoundedSourceRunnerTest.java | 11 ++- .../apache/beam/fn/harness/CombineRunnersTest.java | 38 ++--- .../apache/beam/fn/harness/FlattenRunnerTest.java | 8 +- .../beam/fn/harness/FnApiDoFnRunnerTest.java | 47 +++ .../apache/beam/fn/harness/MapFnRunnersTest.java | 32 --- .../data/PCollectionConsumerRegistryTest.java | 10 ++- .../data/PTransformFunctionRegistryTest.java | 8 +- 22 files changed, 631 insertions(+), 106 deletions(-) create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleExecutionState.java create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleStateRegistry.java create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MonitoringInfoMatchers.java create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleExecutionStateTest.java create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleStateRegistryTest.java
[beam] annotated tag v2.10.0-RC2 updated (d14c60d -> d322e8f)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was modified! *** from d14c60d (commit) to d322e8f (tag) tagging d14c60d38a5b22e8806a41c5477c12abc7a42f5c (commit) replaces v2.10.0-RC1 by Kenneth Knowles on Tue Feb 5 12:38:59 2019 -0800 - Log - [Gradle Release Plugin] - creating tag: 'v2.10.0-RC2'. --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 0144f54 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. 0144f54 is described below commit 0144f5451de4a8581a51b091cd95f81ae3901cee Author: Kenneth Knowles AuthorDate: Tue Feb 5 12:39:05 2019 -0800 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e8bb128..1dcfd01 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-RC2 +version=2.10.0-SNAPSHOT
[beam] annotated tag v2.10.0-RC2 deleted (was 0a6206e)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was deleted! *** tag was 0a6206e This change permanently discards the following revisions: discard 5093683 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new d14c60d [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. d14c60d is described below commit d14c60d38a5b22e8806a41c5477c12abc7a42f5c Author: Kenneth Knowles AuthorDate: Tue Feb 5 12:25:51 2019 -0800 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1dcfd01..e8bb128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-SNAPSHOT +version=2.10.0-RC2
[beam] branch release-2.10.0 updated (b5e00de -> 8e610ad)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git. discard b5e00de [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. discard 1c2e4a5 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. omit 5093683 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/release-2.10.0 (8e610ad) \ O -- O -- O (b5e00de) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new b5e00de [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. b5e00de is described below commit b5e00de0f8a6f1ed696caa67ec431bc3d69e0ca2 Author: Kenneth Knowles AuthorDate: Tue Feb 5 10:53:20 2019 -0800 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1dcfd01..e8bb128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-SNAPSHOT +version=2.10.0-RC2
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 1c2e4a5 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. 1c2e4a5 is described below commit 1c2e4a52a4f7e8ed3b3e36bbf5ce65620da26020 Author: Kenneth Knowles AuthorDate: Tue Feb 5 10:46:06 2019 -0800 [Gradle Release Plugin] - new version commit: 'v2.10.0-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e8bb128..1dcfd01 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-RC2 +version=2.10.0-SNAPSHOT
[beam] annotated tag v2.10.0-RC2 updated (5093683 -> 0a6206e)
This is an automated email from the ASF dual-hosted git repository. kenn pushed a change to annotated tag v2.10.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag v2.10.0-RC2 was modified! *** from 5093683 (commit) to 0a6206e (tag) tagging 5093683b9be4d7b4b421bb7be13ba7a7166c5e17 (commit) replaces v2.10.0-RC1 by Kenneth Knowles on Tue Feb 5 10:45:59 2019 -0800 - Log - [Gradle Release Plugin] - creating tag: 'v2.10.0-RC2'. --- No new revisions were added by this update. Summary of changes:
[beam] branch release-2.10.0 updated: [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'.
This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch release-2.10.0 in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/release-2.10.0 by this push: new 5093683 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. 5093683 is described below commit 5093683b9be4d7b4b421bb7be13ba7a7166c5e17 Author: Kenneth Knowles AuthorDate: Tue Feb 5 10:45:51 2019 -0800 [Gradle Release Plugin] - pre tag commit: 'v2.10.0-RC2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1dcfd01..e8bb128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,4 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.10.0-SNAPSHOT +version=2.10.0-RC2
svn commit: r32343 - /dev/beam/2.10.0/
Author: kenn Date: Tue Feb 5 18:40:06 2019 New Revision: 32343 Log: Remove Beam 2.10.0 RC1 Removed: dev/beam/2.10.0/
[beam] branch master updated: [BEAM-1132] add jacoco report on javaPreCommit
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 925f2ca [BEAM-1132] add jacoco report on javaPreCommit
new c3b0f45 Merge pull request #7699: [BEAM-1132] add jacoco report on
javaPreCommit
925f2ca is described below
commit 925f2ca0203d9b8c33b810d05ff834c3180f4aea
Author: Michael Luckey <25622840+adude3...@users.noreply.github.com>
AuthorDate: Fri Feb 1 02:30:07 2019 +0100
[BEAM-1132] add jacoco report on javaPreCommit
---
.test-infra/jenkins/job_PreCommit_Java.groovy | 3 +++
.../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/.test-infra/jenkins/job_PreCommit_Java.groovy
b/.test-infra/jenkins/job_PreCommit_Java.groovy
index 6ed262e..7397428 100644
--- a/.test-infra/jenkins/job_PreCommit_Java.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Java.groovy
@@ -47,5 +47,8 @@ builder.build {
}
enabledForFailure(true)
}
+jacocoCodeCoverage {
+ execPattern('**/build/jacoco/*.exec')
+}
}
}
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 396b7aa..746865a 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -300,7 +300,7 @@ class BeamModulePlugin implements Plugin {
project.gradle.taskGraph.whenReady { graph ->
// Disable jacoco unless report requested such that task outputs can be
properly cached.
//
https://discuss.gradle.org/t/do-not-cache-if-condition-matched-jacoco-agent-configured-with-append-true-satisfied/23504
- def enabled = graph.allTasks.any { it instanceof JacocoReport }
+ def enabled = graph.allTasks.any { it instanceof JacocoReport ||
it.name.contains("javaPreCommit") }
project.tasks.withType(Test) { jacoco.enabled = enabled }
}
[beam] branch release-2.10.0 updated: Merge pull request #7714: [BEAM-6579] Fix create disposition for the case of temporary tables
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch release-2.10.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.10.0 by this push:
new e46bdcb Merge pull request #7714: [BEAM-6579] Fix create disposition
for the case of temporary tables
new 8e610ad Merge pull request #7733: Cherrypick #7714 for 2.10.0:
[BEAM-6579] Fix create disposition for the case of temporary tables
e46bdcb is described below
commit e46bdcb63bb244f88eddf0820beea0d709899117
Author: reuvenlax
AuthorDate: Mon Feb 4 22:09:49 2019 -0800
Merge pull request #7714: [BEAM-6579] Fix create disposition for the case
of temporary tables
---
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 29 +--
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 31
.../beam/sdk/io/gcp/bigquery/WritePartition.java | 37 +++---
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 26 +++---
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 59 ++
5 files changed, 154 insertions(+), 28 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
index 67bcf5a..abbcd7b 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
@@ -91,11 +91,11 @@ class BatchLoads
@VisibleForTesting
// Maximum number of files in a single partition.
- static final int MAX_NUM_FILES = 1;
+ static final int DEFAULT_MAX_FILES_PER_PARTITION = 1;
@VisibleForTesting
// Maximum number of bytes in a single partition -- 11 TiB just under BQ's
12 TiB limit.
- static final long MAX_SIZE_BYTES = 11 * (1L << 40);
+ static final long DEFAULT_MAX_BYTES_PER_PARTITION = 11 * (1L << 40);
// The maximum size of a single file - 4TiB, just under the 5 TiB limit.
static final long DEFAULT_MAX_FILE_SIZE = 4 * (1L << 40);
@@ -123,6 +123,8 @@ class BatchLoads
private final Coder destinationCoder;
private int maxNumWritersPerBundle;
private long maxFileSize;
+ private int maxFilesPerPartition;
+ private long maxBytesPerPartition;
private int numFileShards;
private Duration triggeringFrequency;
private ValueProvider customGcsTempLocation;
@@ -149,6 +151,8 @@ class BatchLoads
this.maxNumWritersPerBundle = DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE;
this.maxFileSize = DEFAULT_MAX_FILE_SIZE;
this.numFileShards = DEFAULT_NUM_FILE_SHARDS;
+this.maxFilesPerPartition = DEFAULT_MAX_FILES_PER_PARTITION;
+this.maxBytesPerPartition = DEFAULT_MAX_BYTES_PER_PARTITION;
this.triggeringFrequency = null;
this.customGcsTempLocation = customGcsTempLocation;
this.loadJobProjectId = loadJobProjectId;
@@ -190,6 +194,16 @@ class BatchLoads
this.maxFileSize = maxFileSize;
}
+ @VisibleForTesting
+ void setMaxFilesPerPartition(int maxFilesPerPartition) {
+this.maxFilesPerPartition = maxFilesPerPartition;
+ }
+
+ @VisibleForTesting
+ void setMaxBytesPerPartition(long maxBytesPerPartition) {
+this.maxBytesPerPartition = maxBytesPerPartition;
+ }
+
@Override
public void validate(PipelineOptions options) {
// We will use a BigQuery load job -- validate the temp location.
@@ -278,6 +292,8 @@ class BatchLoads
singletonTable,
dynamicDestinations,
tempFilePrefixView,
+maxFilesPerPartition,
+maxBytesPerPartition,
multiPartitionsTag,
singlePartitionTag))
.withSideInputs(tempFilePrefixView)
@@ -345,6 +361,8 @@ class BatchLoads
singletonTable,
dynamicDestinations,
tempFilePrefixView,
+maxFilesPerPartition,
+maxBytesPerPartition,
multiPartitionsTag,
singlePartitionTag))
.withSideInputs(tempFilePrefixView)
@@ -515,7 +533,8 @@ class BatchLoads
ShardedKeyCoder.of(NullableCoder.of(destinationCoder)),
ListCoder.of(StringUtf8Coder.of()));
-// If WriteBundlesToFiles produced more than MAX_NUM_FILES files or
MAX_SIZE_BYTES bytes, then
+// If WriteBundlesToFiles produced more than
DEFAULT_MAX_FILES_PER_PARTITION files or
+// DEFAULT_MAX_BYTES_PER_PARTITION bytes, then
// the import needs to be split into multiple partitions, and those
partitions will be
// specified in multiPartitionsTag.
return input
@@ -526,7
[beam] branch master updated: [BEAM-6291] Generic BigQuery schema load tests metrics (#7614)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 09996b6 [BEAM-6291] Generic BigQuery schema load tests metrics (#7614)
09996b6 is described below
commit 09996b62a7e683c135d505c20f54daabaa1100ea
Author: Kasia Kucharczyk <2536609+kkuch...@users.noreply.github.com>
AuthorDate: Tue Feb 5 18:05:56 2019 +0100
[BEAM-6291] Generic BigQuery schema load tests metrics (#7614)
* [BEAM-6291] Generic schema for BQ load tests
* [BEAM-6291] Added pipeline option to check if metrics are required
* [BEAM-6291] Common environment variable to disable load tests.
* [BEAM-6291] Refactored BQ and Metrics load tests utils. Added missing
documentation and pipelineoption.
---
.../testing/load_tests/co_group_by_key_test.py | 42 +++---
.../apache_beam/testing/load_tests/combine_test.py | 38 ++---
.../testing/load_tests/group_by_key_test.py| 39 ++---
.../testing/load_tests/load_test_metrics_utils.py | 168 -
.../apache_beam/testing/load_tests/pardo_test.py | 49 +++---
.../testing/load_tests/sideinput_test.py | 30 ++--
6 files changed, 198 insertions(+), 168 deletions(-)
diff --git a/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
b/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
index 8b3c026..4ebbf85 100644
--- a/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
@@ -20,10 +20,12 @@ input options there are additional options:
* project (optional) - the gcp project in case of saving
metrics in Big Query (in case of Dataflow Runner
it is required to specify project of runner),
-* metrics_namespace (optional) - name of BigQuery table where metrics
+* publish_to_big_query - if metrics should be published in big query,
+* metrics_namespace (optional) - name of BigQuery dataset where metrics
will be stored,
-in case of lack of any of both options metrics won't be saved
-* input_options - options for Synthetic Sources
+* metrics_table (optional) - name of BigQuery table where metrics
+will be stored,
+* input_options - options for Synthetic Sources,
* co_input_options - options for Synthetic Sources.
Example test run on DirectRunner:
@@ -31,6 +33,7 @@ Example test run on DirectRunner:
python setup.py nosetests \
--test-pipeline-options="
--project=big-query-project
+ --publish_to_big_query=true
--metrics_dataset=python_load_tests
--metrics_table=co_gbk
--input_options='{
@@ -58,6 +61,7 @@ python setup.py nosetests \
--staging_location=gs://...
--temp_location=gs://...
--sdk_location=./dist/apache-beam-x.x.x.dev0.tar.gz
+--publish_to_big_query=true
--metrics_dataset=python_load_tests
--metrics_table=co_gbk
--input_options='{
@@ -84,25 +88,23 @@ from __future__ import absolute_import
import json
import logging
+import os
import unittest
import apache_beam as beam
from apache_beam.testing import synthetic_pipeline
+from apache_beam.testing.load_tests.load_test_metrics_utils import MeasureTime
+from apache_beam.testing.load_tests.load_test_metrics_utils import
MetricsMonitor
from apache_beam.testing.test_pipeline import TestPipeline
-try:
- from apache_beam.testing.load_tests.load_test_metrics_utils import
MeasureTime
- from apache_beam.testing.load_tests.load_test_metrics_utils import
MetricsMonitor
- from google.cloud import bigquery as bq
-except ImportError:
- bq = None
-
INPUT_TAG = 'pc1'
CO_INPUT_TAG = 'pc2'
-RUNTIME_LABEL = 'runtime'
+load_test_enabled = False
+if os.environ.get('LOAD_TEST_ENABLED') == 'true':
+ load_test_enabled = True
-@unittest.skipIf(bq is None, 'BigQuery for storing metrics not installed')
+@unittest.skipIf(not load_test_enabled, 'Enabled only for phrase triggering.')
class CoGroupByKeyTest(unittest.TestCase):
def parseTestPipelineOptions(self, options):
@@ -122,30 +124,28 @@ class CoGroupByKeyTest(unittest.TestCase):
}
def setUp(self):
-self.pipeline = TestPipeline(is_integration_test=True)
+self.pipeline = TestPipeline()
self.input_options = json.loads(self.pipeline.get_option('input_options'))
self.co_input_options = json.loads(
self.pipeline.get_option('co_input_options'))
+self.metrics_monitor = self.pipeline.get_option('publish_to_big_query')
metrics_project_id = self.pipeline.get_option('project')
self.metrics_namespace = self.pipeline.get_option('metrics_table')
metrics_dataset = self.pipeline.get_option('metrics_dataset')
-self.metrics_monitor = None
check = metrics_project_id and self.metrics_namespace and metrics_dataset\
is not None
-if check:
-
[beam] branch master updated: [BEAM-6589] include org.objenesis into shaded jar
This is an automated email from the ASF dual-hosted git repository.
dmvk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new dc3bc33 [BEAM-6589] include org.objenesis into shaded jar
new 228a7d5 Merge pull request #7732: [BEAM-6589] include org.objenesis
into shaded jar
dc3bc33 is described below
commit dc3bc33cf3de36953199f6410a13eefe689f624b
Author: Jan Lukavsky
AuthorDate: Tue Feb 5 15:04:05 2019 +0100
[BEAM-6589] include org.objenesis into shaded jar
---
sdks/java/extensions/kryo/build.gradle | 2 ++
1 file changed, 2 insertions(+)
diff --git a/sdks/java/extensions/kryo/build.gradle
b/sdks/java/extensions/kryo/build.gradle
index 305eb57..14878cc 100644
--- a/sdks/java/extensions/kryo/build.gradle
+++ b/sdks/java/extensions/kryo/build.gradle
@@ -28,9 +28,11 @@ applyJavaNature(
dependencies {
include(dependency('com.esotericsoftware:.*'))
include(dependency('org.ow2.asm:asm'))
+include(dependency('org.objenesis:objenesis'))
}
relocate 'com.esotericsoftware',
getJavaRelocatedPath('com.esotericsoftware')
relocate 'org.objectweb', getJavaRelocatedPath('org.objectweb')
+relocate 'org.objenesis', getJavaRelocatedPath('org.objenesis')
}
)
[beam] branch master updated: Remove EXPERIMENTAL_BEAM_PY3 and add a warning for Python 3 execution
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new afb1af6 Remove EXPERIMENTAL_BEAM_PY3 and add a warning for Python 3
execution
new d268fd4 Merge pull request #7726 from charlesccychen/py3-warning
afb1af6 is described below
commit afb1af6e8ef79dac54094b84e24d53f22c766292
Author: Charles Chen
AuthorDate: Mon Feb 4 15:48:11 2019 -0800
Remove EXPERIMENTAL_BEAM_PY3 and add a warning for Python 3 execution
---
sdks/python/apache_beam/__init__.py | 18 --
sdks/python/container/py3/Dockerfile | 1 -
sdks/python/setup.py | 13 ++---
sdks/python/tox.ini | 4
4 files changed, 22 insertions(+), 14 deletions(-)
diff --git a/sdks/python/apache_beam/__init__.py
b/sdks/python/apache_beam/__init__.py
index ce8bdc2..1ab56ad 100644
--- a/sdks/python/apache_beam/__init__.py
+++ b/sdks/python/apache_beam/__init__.py
@@ -74,16 +74,22 @@ has some examples.
from __future__ import absolute_import
-import sys
+import logging
import os
+import sys
-if not ((sys.version_info[0] == 2 and sys.version_info[1] == 7) or
-(sys.version_info[0] == 3 and
- os.environ.get('BEAM_EXPERIMENTAL_PY3', False))):
+if sys.version_info[0] == 3:
+ logging.warning(
+ 'Running the Apache Beam SDK on Python 3 is not yet fully supported. '
+ 'You may encounter buggy behavior or missing features.')
+elif sys.version_info[0] == 2 and sys.version_info[1] == 7:
+ pass
+else:
raise RuntimeError(
- 'The Apache Beam SDK for Python is supported only on Python 2.7. '
- 'It is not supported on Python ['+ str(sys.version_info) + '].')
+ 'The Apache Beam SDK for Python is only supported on Python 2.7 or '
+ 'Python 3. It is not supported on Python [' +
+ str(sys.version_info) + '].')
# pylint: disable=wrong-import-position
import apache_beam.internal.pickler
diff --git a/sdks/python/container/py3/Dockerfile
b/sdks/python/container/py3/Dockerfile
index 3425653..5fa63fe 100644
--- a/sdks/python/container/py3/Dockerfile
+++ b/sdks/python/container/py3/Dockerfile
@@ -43,7 +43,6 @@ RUN \
rm -rf /root/.cache/pip
-ENV BEAM_EXPERIMENTAL_PY3=1
COPY target/apache-beam.tar.gz /opt/apache/beam/tars/
RUN pip install /opt/apache/beam/tars/apache-beam.tar.gz[gcp] && \
# Remove pip cache.
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 61f385c..f908638 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -20,8 +20,10 @@
from __future__ import absolute_import
from __future__ import print_function
+import logging
import os
import platform
+import sys
import warnings
from distutils.version import StrictVersion
@@ -164,9 +166,14 @@ def generate_protos_first(original_cmd):
return original_cmd
-python_requires = '>=2.7'
-if os.environ.get('BEAM_EXPERIMENTAL_PY3') is None:
- python_requires += ',<3.0'
+# TODO(BEAM-6583): audit Python 3.x version compatibility and refine this
+# requirement range if necessary.
+python_requires = '>=2.7<=3.7'
+
+if sys.version_info[0] == 3:
+ logging.warning(
+ 'Python 3 support for the Apache Beam SDK is not yet fully supported. '
+ 'You may encounter buggy behavior or missing features.')
setuptools.setup(
name=PACKAGE_NAME,
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 3d8abca..743e0bd 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -55,7 +55,6 @@ commands =
[testenv:py3]
setenv =
- BEAM_EXPERIMENTAL_PY3=1
RUN_SKIPPED_PY3_TESTS=0
commands =
python --version
@@ -91,7 +90,6 @@ commands =
[testenv:py3-gcp]
setenv =
- BEAM_EXPERIMENTAL_PY3=1
RUN_SKIPPED_PY3_TESTS=0
extras = test,gcp
modules =
@@ -138,8 +136,6 @@ deps =
future==0.16.0
isort==4.2.15
flake8==3.5.0
-setenv =
-BEAM_EXPERIMENTAL_PY3=1
commands =
pylint --version
pip --version
[beam] branch master updated: Fix BigQuery RowAsDictJsonCoder output type in Python 3
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 65c9759 Fix BigQuery RowAsDictJsonCoder output type in Python 3
new d11c128 Merge pull request #7727 from charlesccychen/fix-bq-coder
65c9759 is described below
commit 65c9759b90bb309ed1000ab3213cf22be3a3c6b6
Author: Charles Chen
AuthorDate: Mon Feb 4 16:12:53 2019 -0800
Fix BigQuery RowAsDictJsonCoder output type in Python 3
---
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index 16016d8..0d12392 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -857,9 +857,9 @@ class RowAsDictJsonCoder(coders.Coder):
# to the programmer that they have used NAN/INF values.
try:
return json.dumps(
- table_row, allow_nan=False, default=default_encoder)
+ table_row, allow_nan=False, default=default_encoder).encode('utf-8')
except ValueError as e:
raise ValueError('%s. %s' % (e, JSON_COMPLIANCE_ERROR))
def decode(self, encoded_table_row):
-return json.loads(encoded_table_row)
+return json.loads(encoded_table_row.decode('utf-8'))
