[beam] tag nightly-master updated (92386d7 -> 4e22ff3)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 92386d7 (commit)
to 4e22ff3 (commit)
from 92386d7 [BEAM-12410] SamzaPortablePipelineOptions to extend
PortablePipelineOptions (#14944)
add a571952 [BEAM-12438] Add Regression test for issue around LP coding
Row coders. (#14924)
add b6aad41 [BEAM-9547] Thorough testing for global aggregations (#14893)
add 9ba2e06 [BEAM-9547] Add support for Series.repeat (#14909)
add aef081c [BEAM-9547] Add support for `DataFrame.value_counts()`
(#14922)
add 9898493 Moving to 2.32.0-SNAPSHOT on master branch.
add 88d4712 [BEAM-12385] Handle VARCHAR and other SQL specific logical
types in AvroUtils
add 62e8f84 Merge pull request #14858: [BEAM-12385] Handle VARCHAR and
other SQL specific logical types in AvroUtils
add 3f2351f Add 2.32.0 section to CHANGES.md
add 88acd72 [WEBSITE] Add a note about how to unsubscribe from mailing
lists
add b9cfb22 Merge pull request #14935 from [WEBSITE] Add a note about how
to unsubscribe from mailing lists
add 183f27f [BEAM-12305] Fix test_pack_combiner for Flink and Spark
(#14842)
add f699f74 Cython 3.0 readiness (#14848)
add 4e22ff3 [BEAM-9547] More WontImplement operations (#14910)
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 36 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
gradle.properties | 4 +-
model/pipeline/src/main/proto/schema.proto | 6 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/translate.go| 9 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 56 ++-
.../beam/core/runtime/graphx/schema/schema_test.go | 16 +-
sdks/go/test/regression/lperror.go | 63 +++
.../lperror_test.go} | 21 +-
sdks/go/test/regression/pardo_test.go | 35 +-
.../regression_test.go}| 5 +-
sdks/go/test/run_validatesrunner_tests.sh | 4 +-
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 43 ++
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 244 ++
.../apache/beam/sdk/io/jdbc/SchemaUtilTest.java| 60 +++
sdks/python/apache_beam/coders/coder_impl.pxd | 1 +
sdks/python/apache_beam/coders/coder_impl.py | 14 +-
sdks/python/apache_beam/dataframe/frame_base.py| 6 +
sdks/python/apache_beam/dataframe/frames.py| 176 ++-
sdks/python/apache_beam/dataframe/frames_test.py | 512 +++--
.../apache_beam/dataframe/pandas_doctests_test.py | 14 +-
.../runners/portability/flink_runner_test.py | 10 +-
.../portability/fn_api_runner/fn_runner_test.py| 28 +-
.../runners/portability/spark_runner_test.py | 4 -
sdks/python/apache_beam/version.py | 2 +-
.../www/site/content/en/community/contact-us.md| 2 +
27 files changed, 1005 insertions(+), 370 deletions(-)
create mode 100644 sdks/go/test/regression/lperror.go
copy sdks/go/test/{integration/primitives/primitives_test.go =>
regression/lperror_test.go} (71%)
copy sdks/go/test/{integration/primitives/primitives_test.go =>
regression/regression_test.go} (83%)
[beam] branch master updated: [BEAM-9547] More WontImplement operations (#14910)
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4e22ff3 [BEAM-9547] More WontImplement operations (#14910)
4e22ff3 is described below
commit 4e22ff3d933c1ba085ab3a7597b7fc5b1079ccb4
Author: Brian Hulette
AuthorDate: Mon Jun 7 17:48:11 2021 -0700
[BEAM-9547] More WontImplement operations (#14910)
* Simple WontImplements
* wontimplement(event-time-semantics)
* Series.sparse too
* Update pandas_doctests_test.py for resample, rolling
---
sdks/python/apache_beam/dataframe/frame_base.py| 6 ++
sdks/python/apache_beam/dataframe/frames.py| 18 ++
.../apache_beam/dataframe/pandas_doctests_test.py | 6 +++---
3 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/sdks/python/apache_beam/dataframe/frame_base.py
b/sdks/python/apache_beam/dataframe/frame_base.py
index 33b33a8..bb649a3 100644
--- a/sdks/python/apache_beam/dataframe/frame_base.py
+++ b/sdks/python/apache_beam/dataframe/frame_base.py
@@ -591,6 +591,12 @@ _WONT_IMPLEMENT_REASONS = {
'explanation': "because it is a plotting tool",
'url': 'https://s.apache.org/dataframe-plotting-tools',
},
+'event-time-semantics': {
+'explanation': (
+"because implementing it would require integrating with Beam "
+"event-time semantics"),
+'url': 'https://s.apache.org/dataframe-event-time-semantics',
+},
'deprecated': {
'explanation': "because it is deprecated in pandas",
},
diff --git a/sdks/python/apache_beam/dataframe/frames.py
b/sdks/python/apache_beam/dataframe/frames.py
index 548f7a4..3cf0834 100644
--- a/sdks/python/apache_beam/dataframe/frames.py
+++ b/sdks/python/apache_beam/dataframe/frames.py
@@ -704,6 +704,14 @@ class DeferredDataFrameOrSeries(frame_base.DeferredFrame):
requires_partition_by=partitionings.Arbitrary(),
preserves_partition_by=partitionings.Singleton())
+ resample = frame_base.wont_implement_method(
+ pd.DataFrame, 'resample', reason='event-time-semantics')
+
+ rolling = frame_base.wont_implement_method(
+ pd.DataFrame, 'rolling', reason='event-time-semantics')
+
+ sparse = property(frame_base.not_implemented_method('sparse', 'BEAM-12425'))
+
@populate_not_implemented(pd.Series)
@frame_base.DeferredFrame._register_for(pd.Series)
@@ -2736,6 +2744,9 @@ class DeferredDataFrame(DeferredDataFrameOrSeries):
values = property(frame_base.wont_implement_method(
pd.DataFrame, 'values', reason="non-deferred-result"))
+ style = property(frame_base.wont_implement_method(
+ pd.DataFrame, 'style', reason="non-deferred-result"))
+
@frame_base.args_to_kwargs(pd.DataFrame)
@frame_base.populate_defaults(pd.DataFrame)
def melt(self, ignore_index, **kwargs):
@@ -2958,6 +2969,13 @@ class DeferredGroupBy(frame_base.DeferredFrame):
DataFrameGroupBy, '__len__', reason="non-deferred-result")
groups = property(frame_base.wont_implement_method(
DataFrameGroupBy, 'groups', reason="non-deferred-result"))
+ indices = property(frame_base.wont_implement_method(
+ DataFrameGroupBy, 'indices', reason="non-deferred-result"))
+
+ resample = frame_base.wont_implement_method(
+ DataFrameGroupBy, 'resample', reason='event-time-semantics')
+ rolling = frame_base.wont_implement_method(
+ DataFrameGroupBy, 'rolling', reason='event-time-semantics')
def _maybe_project_func(projection: Optional[List[str]]):
""" Returns identity func if projection is empty or None, else returns
diff --git a/sdks/python/apache_beam/dataframe/pandas_doctests_test.py
b/sdks/python/apache_beam/dataframe/pandas_doctests_test.py
index a36ba5d..ec3bb9b 100644
--- a/sdks/python/apache_beam/dataframe/pandas_doctests_test.py
+++ b/sdks/python/apache_beam/dataframe/pandas_doctests_test.py
@@ -77,6 +77,8 @@ class DoctestTest(unittest.TestCase):
'df.where(m, -df) == np.where(m, df, -df)'
],
'pandas.core.generic.NDFrame.interpolate': ['*'],
+'pandas.core.generic.NDFrame.resample': ['*'],
+'pandas.core.generic.NDFrame.rolling': ['*'],
},
not_implemented_ok={
'pandas.core.generic.NDFrame.asof': ['*'],
@@ -90,8 +92,6 @@ class DoctestTest(unittest.TestCase):
'pandas.core.generic.NDFrame.reindex': ['*'],
'pandas.core.generic.NDFrame.reindex_like': ['*'],
'pandas.core.generic.NDFrame.replace': ['*'],
-'pandas.core.generic.NDFrame.resample': ['*'],
-'pandas.core.generic.NDFrame.rolling': ['*'],
'pandas.core.generic.NDFrame.sample': ['*'],
'pandas.core.generic.NDFrame.set_flags': ['*'],
'pandas.core.generic.NDFrame.squeeze': ['*'],
@@ -591,10
[beam] branch master updated (183f27f -> f699f74)
This is an automated email from the ASF dual-hosted git repository. robertwb pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 183f27f [BEAM-12305] Fix test_pack_combiner for Flink and Spark (#14842) add f699f74 Cython 3.0 readiness (#14848) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/coders/coder_impl.pxd | 1 + sdks/python/apache_beam/coders/coder_impl.py | 14 -- 2 files changed, 9 insertions(+), 6 deletions(-)
[beam] branch master updated (b9cfb22 -> 183f27f)
This is an automated email from the ASF dual-hosted git repository. robertwb pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from b9cfb22 Merge pull request #14935 from [WEBSITE] Add a note about how to unsubscribe from mailing lists add 183f27f [BEAM-12305] Fix test_pack_combiner for Flink and Spark (#14842) No new revisions were added by this update. Summary of changes: .../runners/portability/flink_runner_test.py | 10 .../portability/fn_api_runner/fn_runner_test.py| 28 ++ .../runners/portability/spark_runner_test.py | 4 3 files changed, 19 insertions(+), 23 deletions(-)
[beam] branch aaltay-patch-1 updated (a12bf7a -> 81808d1)
This is an automated email from the ASF dual-hosted git repository. altay pushed a change to branch aaltay-patch-1 in repository https://gitbox.apache.org/repos/asf/beam.git. from a12bf7a Update grpcio add 81808d1 Add license info for keras-nightly package. No new revisions were added by this update. Summary of changes: sdks/python/container/license_scripts/dep_urls_py.yaml | 2 ++ 1 file changed, 2 insertions(+)
[beam] branch master updated: [WEBSITE] Add a note about how to unsubscribe from mailing lists
This is an automated email from the ASF dual-hosted git repository. pabloem pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 88acd72 [WEBSITE] Add a note about how to unsubscribe from mailing lists new b9cfb22 Merge pull request #14935 from [WEBSITE] Add a note about how to unsubscribe from mailing lists 88acd72 is described below commit 88acd72ef4eea41b88d8ec2f76108b75f21f877a Author: Alexey Romanenko AuthorDate: Thu Jun 3 15:32:57 2021 +0200 [WEBSITE] Add a note about how to unsubscribe from mailing lists --- website/www/site/content/en/community/contact-us.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index b59cc06..c071818 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -38,6 +38,8 @@ The official communication channels for Apache projects are their mailing lists, Prior to sending emails to these lists, you need to subscribe. To subscribe, send a blank email to [user-subscr...@beam.apache.org](user-subscr...@beam.apache.org) or [dev-subscr...@beam.apache.org](dev-subscr...@beam.apache.org) depending on the list you want to write to. +To unsubscribe, send a blank email to [user-unsubscr...@beam.apache.org](user-unsubscr...@beam.apache.org) or [dev-unsubscr...@beam.apache.org](dev-unsubscr...@beam.apache.org) depending on the list you want to unsubscribe. + ### Useful Tips for Sending Emails Tip 1: Use tags in your subject line.
[beam] branch master updated: Add 2.32.0 section to CHANGES.md
This is an automated email from the ASF dual-hosted git repository. apilloud pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 3f2351f Add 2.32.0 section to CHANGES.md 3f2351f is described below commit 3f2351f02b2d7e981a116ac614b53d170cef2236 Author: Andrew Pilloud AuthorDate: Mon Jun 7 13:16:51 2021 -0700 Add 2.32.0 section to CHANGES.md --- CHANGES.md | 39 +++ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 62b8e71..c88113d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -36,10 +36,6 @@ ## Breaking Changes * X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). -* Python Row objects are now sensitive to field order. So `Row(x=3, y=4)` is no - longer considered equal to `Row(y=4, x=3)` (BEAM-11929). -* Kafka Beam SQL tables now ascribe meaning to the LOCATION field; previously - it was ignored if provided. ## Deprecations @@ -50,7 +46,7 @@ * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). --> -# [2.31.0] - Unreleased +# [2.32.X] - Unreleased ## Highlights @@ -63,8 +59,6 @@ ## New Features / Improvements -* `CREATE FUNCTION` DDL statement added to Calcite SQL syntax. `JAR` and `AGGREGATE` are now reserved keywords. ([BEAM-12339](https://issues.apache.org/jira/browse/BEAM-12339)). -* Flink 1.13 is now supported by the Flink runner ([BEAM-12277](https://issues.apache.org/jira/browse/BEAM-12277)). * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Add support to convert Beam Schema to Avro Schema for JDBC LogicalTypes: `VARCHAR`, `NVARCHAR`, `LONGVARCHAR`, `LONGNVARCHAR`, `DATE`, `TIME` @@ -73,13 +67,42 @@ ## Breaking Changes * X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* ListShards (with DescribeStreamSummary) is used instead of DescribeStream to list shards in Kinesis streams. Due to this change, as mentioned in [AWS documentation](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html), for fine-grained IAM policies it is required to update them to allow calls to ListShards and DescribeStreamSummary APIs. For more information, see [Controlling Access to Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/c [...] + +## Deprecations + +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Known Issues + +* Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +# [2.31.0] - Unreleased + +## Highlights + +* New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* New highly anticipated feature Y added to Java SDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + +## I/Os + +* Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## New Features / Improvements + +* `CREATE FUNCTION` DDL statement added to Calcite SQL syntax. `JAR` and `AGGREGATE` are now reserved keywords. ([BEAM-12339](https://issues.apache.org/jira/browse/BEAM-12339)). +* Flink 1.13 is now supported by the Flink runner ([BEAM-12277](https://issues.apache.org/jira/browse/BEAM-12277)). +* X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Breaking Changes + +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Python Row objects are now sensitive to field order. So `Row(x=3, y=4)` is no longer considered equal to `Row(y=4, x=3)` (BEAM-11929). * Kafka Beam SQL tables now ascribe meaning to the LOCATION field; previously it was ignored if provided. * `TopCombineFn` disallow `compare` as its argument (Python) ([BEAM-7372](https://issues.apache.org/jira/browse/BEAM-7372)). * Drop support for Flink 1.10 ([BEAM-12281](https://issues.apache.org/jira/browse/BEAM-12281)). -* ListShards (with DescribeStreamSummary) is used instead of DescribeStream to list shards in Kinesis streams. Due to this change, as mentioned in [AWS documentation](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html), for fine-grained IAM policies it is required to update them to allow calls to ListShards and DescribeStreamSummary APIs. For more information, see [Controlling Access to Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/c [...] ## Deprecations
[beam] branch master updated: [BEAM-12385] Handle VARCHAR and other SQL specific logical types in AvroUtils
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 88d4712 [BEAM-12385] Handle VARCHAR and other SQL specific logical
types in AvroUtils
new 62e8f84 Merge pull request #14858: [BEAM-12385] Handle VARCHAR and
other SQL specific logical types in AvroUtils
88d4712 is described below
commit 88d4712147911744761cb385b8226c81e283d1fe
Author: Anant Damle
AuthorDate: Fri May 21 23:47:53 2021 +0800
[BEAM-12385] Handle VARCHAR and other SQL specific logical types in
AvroUtils
---
CHANGES.md | 3 +
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 43
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 244 +
.../apache/beam/sdk/io/jdbc/SchemaUtilTest.java| 60 +
4 files changed, 350 insertions(+)
diff --git a/CHANGES.md b/CHANGES.md
index 7029cd2..62b8e71 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -66,6 +66,9 @@
* `CREATE FUNCTION` DDL statement added to Calcite SQL syntax. `JAR` and
`AGGREGATE` are now reserved keywords.
([BEAM-12339](https://issues.apache.org/jira/browse/BEAM-12339)).
* Flink 1.13 is now supported by the Flink runner
([BEAM-12277](https://issues.apache.org/jira/browse/BEAM-12277)).
* X feature added (Java/Python)
([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)).
+* Add support to convert Beam Schema to Avro Schema for JDBC LogicalTypes:
+ `VARCHAR`, `NVARCHAR`, `LONGVARCHAR`, `LONGNVARCHAR`, `DATE`, `TIME`
+ (Java)([BEAM-12385](https://issues.apache.org/jira/browse/BEAM-12385)).
## Breaking Changes
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java
index 77b5445..0835f9b 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java
@@ -906,6 +906,26 @@ public class AvroUtils {
.map(x -> getFieldSchema(x.getType(), x.getName(),
namespace))
.collect(Collectors.toList()));
break;
+ case "CHAR":
+ case "NCHAR":
+baseType =
+buildHiveLogicalTypeSchema("char", (int)
fieldType.getLogicalType().getArgument());
+break;
+ case "NVARCHAR":
+ case "VARCHAR":
+ case "LONGNVARCHAR":
+ case "LONGVARCHAR":
+baseType =
+buildHiveLogicalTypeSchema(
+"varchar", (int) fieldType.getLogicalType().getArgument());
+break;
+ case "DATE":
+baseType =
LogicalTypes.date().addToSchema(org.apache.avro.Schema.create(Type.INT));
+break;
+ case "TIME":
+baseType =
+
LogicalTypes.timeMillis().addToSchema(org.apache.avro.Schema.create(Type.INT));
+break;
default:
throw new RuntimeException(
"Unhandled logical type " +
fieldType.getLogicalType().getIdentifier());
@@ -1017,6 +1037,15 @@ public class AvroUtils {
typeWithNullability.type.getTypes().get(oneOfValue.getCaseType().getValue()),
oneOfValue.getValue());
}
+ case "NVARCHAR":
+ case "VARCHAR":
+ case "LONGNVARCHAR":
+ case "LONGVARCHAR":
+return new Utf8((String) value);
+ case "DATE":
+return Days.daysBetween(Instant.EPOCH, (Instant) value).getDays();
+ case "TIME":
+return (int) ((Instant) value).getMillis();
default:
throw new RuntimeException(
"Unhandled logical type " +
fieldType.getLogicalType().getIdentifier());
@@ -1277,4 +1306,18 @@ public class AvroUtils {
checkArgument(
got.equals(expected), "Can't convert '%s' to %s, expected: %s", label,
got, expected);
}
+
+ /**
+ * Helper factory to build Avro Logical types schemas for SQL *CHAR types.
This method https://github.com/apache/hive/blob/5d268834a5f5278ea76399f8af0d0ab043ae0b45/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java#L110-L121;>represents
+ * the logical as Hive does.
+ */
+ private static org.apache.avro.Schema buildHiveLogicalTypeSchema(
+ String hiveLogicalType, int size) {
+String schemaJson =
+String.format(
+"{\"type\": \"string\", \"logicalType\": \"%s\", \"maxLength\":
%s}",
+hiveLogicalType, size);
+return new org.apache.avro.Schema.Parser().parse(schemaJson);
+ }
}
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java
[beam] branch master updated: Add 2.32.0 section to CHANGES.md
This is an automated email from the ASF dual-hosted git repository. apilloud pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 443f696 Add 2.32.0 section to CHANGES.md 443f696 is described below commit 443f69695106d5115bd6a75f5318d6851a8445b3 Author: Andrew Pilloud AuthorDate: Mon Jun 7 13:16:51 2021 -0700 Add 2.32.0 section to CHANGES.md --- CHANGES.md | 33 - 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7029cd2..a8b7797 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -36,10 +36,6 @@ ## Breaking Changes * X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). -* Python Row objects are now sensitive to field order. So `Row(x=3, y=4)` is no - longer considered equal to `Row(y=4, x=3)` (BEAM-11929). -* Kafka Beam SQL tables now ascribe meaning to the LOCATION field; previously - it was ignored if provided. ## Deprecations @@ -50,6 +46,34 @@ * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). --> +# [2.32.X] - Unreleased + +## Highlights + +* New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* New highly anticipated feature Y added to Java SDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + +## I/Os + +* Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## New Features / Improvements + +* X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Breaking Changes + +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* ListShards (with DescribeStreamSummary) is used instead of DescribeStream to list shards in Kinesis streams. Due to this change, as mentioned in [AWS documentation](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html), for fine-grained IAM policies it is required to update them to allow calls to ListShards and DescribeStreamSummary APIs. For more information, see [Controlling Access to Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/c [...] + +## Deprecations + +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Known Issues + +* Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + # [2.31.0] - Unreleased ## Highlights @@ -76,7 +100,6 @@ it was ignored if provided. * `TopCombineFn` disallow `compare` as its argument (Python) ([BEAM-7372](https://issues.apache.org/jira/browse/BEAM-7372)). * Drop support for Flink 1.10 ([BEAM-12281](https://issues.apache.org/jira/browse/BEAM-12281)). -* ListShards (with DescribeStreamSummary) is used instead of DescribeStream to list shards in Kinesis streams. Due to this change, as mentioned in [AWS documentation](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html), for fine-grained IAM policies it is required to update them to allow calls to ListShards and DescribeStreamSummary APIs. For more information, see [Controlling Access to Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/c [...] ## Deprecations
[beam] 01/01: Set Dataflow container to release version.
This is an automated email from the ASF dual-hosted git repository.
apilloud pushed a commit to branch release-2.31.0
in repository https://gitbox.apache.org/repos/asf/beam.git
commit df78d0726942542b45346adee419fa189ebfc85c
Author: Andrew Pilloud
AuthorDate: Mon Jun 7 13:06:03 2021 -0700
Set Dataflow container to release version.
---
runners/google-cloud-dataflow-java/build.gradle | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 261b2a2..006c603 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -45,8 +45,8 @@ processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
'dataflow.legacy_environment_major_version' : '8',
'dataflow.fnapi_environment_major_version' : '8',
-'dataflow.legacy_container_version' : 'beam-master-20210525',
-'dataflow.fnapi_container_version' : 'beam-master-20210524',
+'dataflow.legacy_container_version' : 'beam-2.31.0',
+'dataflow.fnapi_container_version' : 'beam-2.31.0',
'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3',
]
}
[beam] branch release-2.31.0 created (now df78d07)
This is an automated email from the ASF dual-hosted git repository. apilloud pushed a change to branch release-2.31.0 in repository https://gitbox.apache.org/repos/asf/beam.git. at df78d07 Set Dataflow container to release version. This branch includes the following new commits: new df78d07 Set Dataflow container to release version. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[beam] branch master updated: Moving to 2.32.0-SNAPSHOT on master branch.
This is an automated email from the ASF dual-hosted git repository.
apilloud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9898493 Moving to 2.32.0-SNAPSHOT on master branch.
9898493 is described below
commit 9898493a8c852ff3240d3a32f5a1c0bfb3a80222
Author: Andrew Pilloud
AuthorDate: Mon Jun 7 12:56:36 2021 -0700
Moving to 2.32.0-SNAPSHOT on master branch.
---
.../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 2 +-
gradle.properties | 4 ++--
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/python/apache_beam/version.py| 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 285dd61..b011646 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -352,7 +352,7 @@ class BeamModulePlugin implements Plugin {
// Automatically use the official release version if we are performing a
release
// otherwise append '-SNAPSHOT'
-project.version = '2.31.0'
+project.version = '2.32.0'
if (!isRelease(project)) {
project.version += '-SNAPSHOT'
}
diff --git a/gradle.properties b/gradle.properties
index 1d72550..9661844 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository
signing.gnupg.executable=gpg
signing.gnupg.useLegacyGpg=true
-version=2.31.0-SNAPSHOT
-sdk_version=2.31.0.dev
+version=2.32.0-SNAPSHOT
+sdk_version=2.32.0.dev
javaVersion=1.8
diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go
index 9b2c713..981570a 100644
--- a/sdks/go/pkg/beam/core/core.go
+++ b/sdks/go/pkg/beam/core/core.go
@@ -27,5 +27,5 @@ const (
// SdkName is the human readable name of the SDK for UserAgents.
SdkName = "Apache Beam SDK for Go"
// SdkVersion is the current version of the SDK.
- SdkVersion = "2.31.0.dev"
+ SdkVersion = "2.32.0.dev"
)
diff --git a/sdks/python/apache_beam/version.py
b/sdks/python/apache_beam/version.py
index ceda28a..a3eb9eb 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -17,4 +17,4 @@
"""Apache Beam SDK version information and utilities."""
-__version__ = '2.31.0.dev'
+__version__ = '2.32.0.dev'
[beam] branch master updated (9ba2e06 -> aef081c)
This is an automated email from the ASF dual-hosted git repository. bhulette pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 9ba2e06 [BEAM-9547] Add support for Series.repeat (#14909) add aef081c [BEAM-9547] Add support for `DataFrame.value_counts()` (#14922) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/dataframe/frames.py| 51 -- sdks/python/apache_beam/dataframe/frames_test.py | 14 +- .../apache_beam/dataframe/pandas_doctests_test.py | 2 - 3 files changed, 61 insertions(+), 6 deletions(-)
[beam] branch master updated (b6aad41 -> 9ba2e06)
This is an automated email from the ASF dual-hosted git repository. bhulette pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from b6aad41 [BEAM-9547] Thorough testing for global aggregations (#14893) add 9ba2e06 [BEAM-9547] Add support for Series.repeat (#14909) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/dataframe/frames.py| 33 ++ .../apache_beam/dataframe/pandas_doctests_test.py | 4 ++- 2 files changed, 36 insertions(+), 1 deletion(-)
[beam] branch master updated (a571952 -> b6aad41)
This is an automated email from the ASF dual-hosted git repository. bhulette pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from a571952 [BEAM-12438] Add Regression test for issue around LP coding Row coders. (#14924) add b6aad41 [BEAM-9547] Thorough testing for global aggregations (#14893) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/dataframe/frames.py| 74 ++- sdks/python/apache_beam/dataframe/frames_test.py | 498 +++-- .../apache_beam/dataframe/pandas_doctests_test.py | 2 - 3 files changed, 312 insertions(+), 262 deletions(-)
[beam] branch master updated: [BEAM-12438] Add Regression test for issue around LP coding Row coders. (#14924)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a571952 [BEAM-12438] Add Regression test for issue around LP coding
Row coders. (#14924)
a571952 is described below
commit a571952e3ce470b3871ebd333eacb3d6368d2737
Author: Robert Burke
AuthorDate: Mon Jun 7 09:45:26 2021 -0700
[BEAM-12438] Add Regression test for issue around LP coding Row coders.
(#14924)
* [BEAM-12438] Run regression during integration
* [BEAM-12438] Add LP error repro
* [BEAM-12438] Ignore extra LP on injects.
* [BEAM-12438] Populate schema option types.
Co-authored-by: zelliott
---
model/pipeline/src/main/proto/schema.proto | 6 +--
sdks/go/pkg/beam/core/runtime/exec/translate.go| 9 +++-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 56 ++-
.../beam/core/runtime/graphx/schema/schema_test.go | 16 ++
sdks/go/test/regression/lperror.go | 63 ++
.../regression/{pardo_test.go => lperror_test.go} | 46 +---
sdks/go/test/regression/pardo_test.go | 35 ++--
.../{pardo_test.go => regression_test.go} | 38 ++---
sdks/go/test/run_validatesrunner_tests.sh | 4 +-
9 files changed, 158 insertions(+), 115 deletions(-)
diff --git a/model/pipeline/src/main/proto/schema.proto
b/model/pipeline/src/main/proto/schema.proto
index 837689f..bcab2e7 100644
--- a/model/pipeline/src/main/proto/schema.proto
+++ b/model/pipeline/src/main/proto/schema.proto
@@ -113,9 +113,9 @@ message LogicalType {
message Option {
// REQUIRED. Identifier for the option.
string name = 1;
- // OPTIONAL. Type specifer for the structure of value.
- // If not present, assumes no additional configuration is needed
- // for this option and value is ignored.
+ // REQUIRED. Type specifer for the structure of value.
+ // Conventionally, options that don't require additional configuration should
+ // use a boolean type, with the value set to true.
FieldType type = 2;
FieldValue value = 3;
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate.go
b/sdks/go/pkg/beam/core/runtime/exec/translate.go
index fbbdab3..105eb82 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate.go
@@ -472,7 +472,14 @@ func (b *builder) makeLink(from string, id linkID) (Node,
error) {
if !coder.IsKV(c) {
return nil, errors.Errorf("unexpected inject
coder: %v", c)
}
- u = {UID: b.idgen.New(), N: (int)(tp.Inject.N),
ValueEncoder: MakeElementEncoder(c.Components[1]), Out: out[0]}
+ valCoder := c.Components[1]
+ // JIRA BEAM-12438 - an extra LP coder can get added
here, but isn't added
+ // on decode. Strip them until we get a better fix.
+ if valCoder.Kind == coder.LP {
+ // strip unexpected length prefix coder.
+ valCoder = valCoder.Components[0]
+ }
+ u = {UID: b.idgen.New(), N: (int)(tp.Inject.N),
ValueEncoder: MakeElementEncoder(valCoder), Out: out[0]}
case graphx.URNExpand:
var pid string
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
index 087d8c1..2e3ea3f 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
@@ -356,9 +356,7 @@ func (r *Registry) fromType(ot reflect.Type)
(*pipepb.Schema, error) {
schm := ftype.GetRowType().GetSchema()
schm = proto.Clone(schm).(*pipepb.Schema)
if ot.Kind() == reflect.Ptr {
- schm.Options = append(schm.Options, {
- Name: optGoNillable,
- })
+ schm.Options = append(schm.Options, optGoNillable())
}
if lID != "" {
schm.Options = append(schm.Options, logicalOption(lID))
@@ -379,9 +377,7 @@ func (r *Registry) fromType(ot reflect.Type)
(*pipepb.Schema, error) {
pt := reflect.PtrTo(t)
schm = proto.Clone(schm).(*pipepb.Schema)
schm.Id = getUUID(pt)
- schm.Options = append(schm.Options, {
- Name: optGoNillable,
- })
+ schm.Options = append(schm.Options, optGoNillable())
r.idToType[schm.GetId()] = pt
r.typeToSchema[pt] = schm
@@ -392,14 +388,46 @@ func (r *Registry) fromType(ot reflect.Type)
(*pipepb.Schema, error) {
// Schema Option urns.
const (
// optGoNillable
