This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch release-2.0
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-2.0 by this push:
new efcee81d5f5 [FLINK-37192][python] Replace deprecated avro-python3 with
avro
efcee81d5f5 is described below
commit efcee81d5f5628c3f3e4064f74743a1d3f027b1d
Author: Mina Asham <[email protected]>
AuthorDate: Fri Jan 17 13:59:56 2025 +0000
[FLINK-37192][python] Replace deprecated avro-python3 with avro
- avro-python3 was deprecated and replaced by avro, the first hasn't had a
release since March 17, 2021 while the second has had multiple fixes and
updated, latest is August 5, 2024
- Both libraries are the exact same (i.e. `avro-python3` was just renamed
to `avro` and had multiple updates since), but the problem is that they overlap
in package name, so using PyFlink with any updated library that relies on
`avro` fails starting the pipeline even if the pipeline doesn't actually do any
avro encoding/decoding
- This updates the library to the latest one, and fixes a few imports,
other than that the library's functionality is exactly the same
This closes #26008.
---
flink-python/dev/dev-requirements.txt | 2 +-
flink-python/pyflink/fn_execution/formats/avro.py | 9 ++++-----
flink-python/setup.py | 2 +-
3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/flink-python/dev/dev-requirements.txt
b/flink-python/dev/dev-requirements.txt
index 63649ab7e98..8c1eb18ffa5 100755
--- a/flink-python/dev/dev-requirements.txt
+++ b/flink-python/dev/dev-requirements.txt
@@ -20,7 +20,7 @@ cython>=0.29.24
py4j==0.10.9.7
python-dateutil>=2.8.0,<3
cloudpickle~=2.2.0
-avro-python3>=1.8.1,!=1.9.2
+avro>=1.12.0
pandas>=1.3.0
pyarrow>=5.0.0
pytz>=2018.3
diff --git a/flink-python/pyflink/fn_execution/formats/avro.py
b/flink-python/pyflink/fn_execution/formats/avro.py
index db8a9da3e7b..5c291304797 100644
--- a/flink-python/pyflink/fn_execution/formats/avro.py
+++ b/flink-python/pyflink/fn_execution/formats/avro.py
@@ -17,14 +17,13 @@
################################################################################
import struct
+from avro.errors import AvroTypeException, SchemaResolutionException
from avro.io import (
- AvroTypeException,
BinaryDecoder,
BinaryEncoder,
DatumReader,
DatumWriter,
- SchemaResolutionException,
- Validate,
+ validate,
)
STRUCT_FLOAT = struct.Struct('>f') # big-endian float
@@ -203,7 +202,7 @@ class FlinkAvroEncoder(BinaryEncoder):
class FlinkAvroDatumWriter(DatumWriter):
def __init__(self, writer_schema=None):
- super().__init__(writer_schema=writer_schema)
+ super().__init__(writers_schema=writer_schema)
def write_array(self, writer_schema, datum, encoder):
if len(datum) > 0:
@@ -224,7 +223,7 @@ class FlinkAvroDatumWriter(DatumWriter):
# resolve union
index_of_schema = -1
for i, candidate_schema in enumerate(writer_schema.schemas):
- if Validate(candidate_schema, datum):
+ if validate(candidate_schema, datum):
index_of_schema = i
if index_of_schema < 0:
raise AvroTypeException(writer_schema, datum)
diff --git a/flink-python/setup.py b/flink-python/setup.py
index 119f6ba4585..2703f8d931c 100644
--- a/flink-python/setup.py
+++ b/flink-python/setup.py
@@ -318,7 +318,7 @@ try:
install_requires = ['py4j==0.10.9.7', 'python-dateutil>=2.8.0,<3',
'apache-beam>=2.54.0,<=2.61.0',
- 'cloudpickle>=2.2.0', 'avro-python3>=1.8.1,!=1.9.2',
+ 'cloudpickle>=2.2.0', 'avro>=1.12.0',
'pytz>=2018.3', 'fastavro>=1.1.0,!=1.8.0',
'requests>=2.26.0',
'protobuf>=3.19.0',
'numpy>=1.22.4',
