This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 1a7e213 (commit)
to fd3075f (commit)
from 1a7e213 [BEAM-11628] Add initial support for GroupBy.apply (#13843)
add bae76b0 fix google3 import warnings
add 7e746db Merge pull request #14363 from apache/ihji-patch-5
add e92d184 [BEAM-12069] Make mock a test-only dependency (#14328)
add 5a9260f Refactor Dataflow V2 sickbay lists to prepare for portable
job submission
add 20fbcf7 Produce portable pipeline proto before applying Dataflow V1
replacements
add c5546b2 Re-enable portable job submission by default
add a16bbf7 Merge pull request #14339: Roll forward PJS with Pubsub fix
add 4a1b0aa [BEAM-7093] Add Spark 3 and Java 11 info into the CHANGES file
add 7eff49f Merge pull request #14378: [BEAM-7093] Add Spark 3 and Java
11 info into the CHANGES file
add a0a8666 Merge pull request #13985 from [BEAM-11810] A trigger manager
for FnApiRunner
add f93d61d [BEAM-10943] Enable SqlTransform::registerUdf in ZetaSQL.
(#14263)
add 52c3b0b [BEAM-11797] Fix wrong assertion usage (#14377)
add 911771d [BEAM-12071] Don't re-use WriteToPandasSink instances across
windows (#14374)
add fd3075f Merge pull request #13995 from [BEAM-11322] Apache Beam
Example to tokenize sensitive data
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 2 +
build.gradle.kts | 1 +
examples/java/build.gradle | 13 +
.../datatokenization/DataTokenization.java | 343 +++++++++++++
.../examples/complete/datatokenization/README.md | 172 +++++++
.../options/DataTokenizationOptions.java | 67 +++
.../datatokenization/options/package-info.java | 18 +
.../complete/datatokenization/package-info.java | 18 +
.../transforms/DataProtectors.java | 290 +++++++++++
.../datatokenization/transforms/JsonToBeamRow.java | 76 +++
.../transforms/SerializableFunctions.java | 49 ++
.../transforms/io/TokenizationBigQueryIO.java | 98 ++++
.../transforms/io/TokenizationBigTableIO.java | 158 ++++++
.../transforms/io/TokenizationFileSystemIO.java | 261 ++++++++++
.../transforms/io/package-info.java | 18 +
.../datatokenization/transforms/package-info.java | 18 +
.../datatokenization/utils/CsvConverters.java | 537 +++++++++++++++++++++
.../datatokenization/utils/DurationUtils.java | 75 +++
.../datatokenization/utils/ErrorConverters.java | 311 ++++++++++++
.../datatokenization/utils/FailsafeElement.java | 97 ++++
.../utils/FailsafeElementCoder.java | 106 ++++
.../complete/datatokenization/utils/RowToCsv.java | 38 ++
.../datatokenization/utils/SchemasUtils.java | 206 ++++++++
.../datatokenization/utils/package-info.java | 18 +
.../datatokenization/DataTokenizationTest.java | 208 ++++++++
examples/java/src/test/resources/schema.txt | 19 +
examples/java/src/test/resources/testInput.csv | 3 +
examples/java/src/test/resources/testInput.txt | 3 +
runners/google-cloud-dataflow-java/build.gradle | 31 +-
.../beam/runners/dataflow/DataflowRunner.java | 120 +++--
.../src/main/resources/beam/suppressions.xml | 1 +
.../extensions/sql/zetasql/BeamZetaSqlCatalog.java | 64 ++-
.../zetasql/translation/ExpressionConverter.java | 5 +-
.../translation/UserFunctionDefinitions.java | 10 +-
.../sql/zetasql/BeamZetaSqlCatalogTest.java | 72 +++
.../extensions/sql/zetasql/ZetaSqlJavaUdfTest.java | 57 +++
sdks/python/apache_beam/dataframe/io.py | 2 +-
sdks/python/apache_beam/dataframe/io_test.py | 41 +-
.../runners/dataflow/internal/apiclient_test.py | 6 +-
.../apache_beam/runners/interactive/utils_test.py | 8 +-
.../portability/fn_api_runner/trigger_manager.py | 458 ++++++++++++++++++
.../fn_api_runner/trigger_manager_test.py | 250 ++++++++++
sdks/python/apache_beam/transforms/trigger.py | 11 +-
.../apache_beam/transforms/userstate_test.py | 39 ++
sdks/python/apache_beam/utils/timestamp.py | 4 +
sdks/python/setup.py | 2 +-
46 files changed, 4326 insertions(+), 78 deletions(-)
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/DataTokenization.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/README.md
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/options/DataTokenizationOptions.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/options/package-info.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/package-info.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/JsonToBeamRow.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/SerializableFunctions.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/io/TokenizationBigQueryIO.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/io/TokenizationBigTableIO.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/io/TokenizationFileSystemIO.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/io/package-info.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/package-info.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/CsvConverters.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/DurationUtils.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/ErrorConverters.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElement.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElementCoder.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/RowToCsv.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/SchemasUtils.java
create mode 100644
examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/package-info.java
create mode 100644
examples/java/src/test/java/org/apache/beam/examples/complete/datatokenization/DataTokenizationTest.java
create mode 100644 examples/java/src/test/resources/schema.txt
create mode 100644 examples/java/src/test/resources/testInput.csv
create mode 100644 examples/java/src/test/resources/testInput.txt
create mode 100644
sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalogTest.java
create mode 100644
sdks/python/apache_beam/runners/portability/fn_api_runner/trigger_manager.py
create mode 100644
sdks/python/apache_beam/runners/portability/fn_api_runner/trigger_manager_test.py
