[
https://issues.apache.org/jira/browse/BEAM-5624?focusedWorklogId=153657&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153657
]
ASF GitHub Bot logged work on BEAM-5624:
----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Oct/18 19:15
Start Date: 11/Oct/18 19:15
Worklog Time Spent: 10m
Work Description: aaltay closed pull request #6616: [BEAM-5624] Fix
avro.schema parser for py3
URL: https://github.com/apache/beam/pull/6616
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/examples/avro_bitcoin.py
b/sdks/python/apache_beam/examples/avro_bitcoin.py
index d05b73553b0..079d9d79fcb 100644
--- a/sdks/python/apache_beam/examples/avro_bitcoin.py
+++ b/sdks/python/apache_beam/examples/avro_bitcoin.py
@@ -29,8 +29,6 @@
import argparse
import logging
-import avro
-
import apache_beam as beam
from apache_beam.io.avroio import ReadFromAvro
from apache_beam.io.avroio import WriteToAvro
@@ -38,6 +36,13 @@
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
+# pylint: disable=wrong-import-order, wrong-import-position
+try:
+ from avro.schema import Parse # avro-python3 library for python3
+except ImportError:
+ from avro.schema import parse as Parse # avro library for python2
+# pylint: enable=wrong-import-order, wrong-import-position
+
class BitcoinTxnCountDoFn(beam.DoFn):
"""Count inputs and outputs per transaction"""
@@ -85,7 +90,7 @@ def process(self, elem):
]
-SCHEMA = avro.schema.parse('''
+SCHEMA = Parse('''
{
"namespace": "example.avro",
"type": "record",
diff --git a/sdks/python/apache_beam/examples/fastavro_it_test.py
b/sdks/python/apache_beam/examples/fastavro_it_test.py
index 1504b5b0456..327b10f7162 100644
--- a/sdks/python/apache_beam/examples/fastavro_it_test.py
+++ b/sdks/python/apache_beam/examples/fastavro_it_test.py
@@ -49,7 +49,6 @@
import unittest
import uuid
-import avro
from nose.plugins.attrib import attr
from apache_beam.io.avroio import ReadAllFromAvro
@@ -63,6 +62,13 @@
from apache_beam.transforms.core import Map
from apache_beam.transforms.util import CoGroupByKey
+# pylint: disable=wrong-import-order, wrong-import-position
+try:
+ from avro.schema import Parse # avro-python3 library for python3
+except ImportError:
+ from avro.schema import parse as Parse # avro library for python2
+# pylint: enable=wrong-import-order, wrong-import-position
+
LABELS = ['abc', 'def', 'ghi', 'jkl', 'mno', 'pqr', 'stu', 'vwx']
COLORS = ['RED', 'ORANGE', 'YELLOW', 'GREEN', 'BLUE', 'PURPLE', None]
@@ -78,7 +84,7 @@ def record(i):
class FastavroIT(unittest.TestCase):
- SCHEMA = avro.schema.parse('''
+ SCHEMA = Parse('''
{"namespace": "example.avro",
"type": "record",
"name": "User",
diff --git a/sdks/python/apache_beam/io/avroio.py
b/sdks/python/apache_beam/io/avroio.py
index f90dc3c6833..df3fbeac874 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -52,7 +52,6 @@
import avro
from avro import io as avroio
from avro import datafile
-from avro import schema
from fastavro.read import block_reader
from fastavro.write import Writer
@@ -64,6 +63,13 @@
from apache_beam.io.iobase import Read
from apache_beam.transforms import PTransform
+# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
+try:
+ from avro.schema import Parse # avro-python3 library for python3
+except ImportError:
+ from avro.schema import parse as Parse # avro library for python2
+# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
+
__all__ = ['ReadFromAvro', 'ReadAllFromAvro', 'WriteToAvro']
@@ -312,7 +318,7 @@ def __init__(self, block_bytes, num_records, codec,
schema_string,
# iteration.
self._decompressed_block_bytes = self._decompress_bytes(block_bytes, codec)
self._num_records = num_records
- self._schema = schema.parse(schema_string)
+ self._schema = Parse(schema_string)
self._offset = offset
self._size = size
@@ -481,7 +487,7 @@ def __init__(self,
end in a common extension, if given by file_name_suffix. In most cases,
only this argument is specified and num_shards, shard_name_template,
and
file_name_suffix use default values.
- schema: The schema to use, as returned by avro.schema.parse
+ schema: The schema to use, as returned by avro.schema.Parse
codec: The codec to use for block-level compression. Any string supported
by the Avro specification is accepted (for example 'null').
file_name_suffix: Suffix for the files written.
diff --git a/sdks/python/apache_beam/io/avroio_test.py
b/sdks/python/apache_beam/io/avroio_test.py
index daec98c38dd..6346c9b1b65 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -25,10 +25,15 @@
from builtins import range
import avro.datafile
-import avro.schema
from avro.datafile import DataFileWriter
from avro.io import DatumWriter
import hamcrest as hc
+# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
+try:
+ from avro.schema import Parse # avro-python3 library for python3
+except ImportError:
+ from avro.schema import parse as Parse # avro library for python2
+# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
import apache_beam as beam
from apache_beam import Create
@@ -93,7 +98,7 @@ def tearDown(self):
'favorite_number': 6,
'favorite_color': 'Green'}]
- SCHEMA = avro.schema.parse('''
+ SCHEMA = Parse('''
{"namespace": "example.avro",
"type": "record",
"name": "User",
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 153657)
Time Spent: 1h (was: 50m)
> Avro IO does not work with avro-python3 package out-of-the-box on Python 3,
> several tests fail with AttributeError (module 'avro.schema' has no attribute
> 'parse')
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: BEAM-5624
> URL: https://issues.apache.org/jira/browse/BEAM-5624
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-py-core
> Reporter: Valentyn Tymofieiev
> Assignee: Simon
> Priority: Major
> Time Spent: 1h
> Remaining Estimate: 0h
>
> ======================================================================
> ERROR: Failure: AttributeError (module 'avro.schema' has no attribute 'parse')
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/site-packages/nose/failure.py",
> line 39, in runTest
> raise self.exc_val.with_traceback(self.tb)
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/site-packages/nose/loader.py",
> line 418, in loadTestsFromName
> addr.filename, addr.module)
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/site-packages/nose/importer.py",
> line 47, in importFromPath
> return self.importFromDir(dir_path, fqname)
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/site-packages/nose/importer.py",
> line 94, in importFromDir
> mod = load_module(part_fqname, fh, filename, desc)
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/imp.py",
> line 234, in load_module
> return load_source(name, filename, file)
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/target/.tox/py3/lib/python3.5/imp.py",
> line 172, in load_source
> module = _load(spec)
> File "<frozen importlib._bootstrap>", line 693, in _load
> File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
> File "<frozen importlib._bootstrap_external>", line 673, in exec_module
> File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/apache_beam/io/avroio_test.py",
> line 54, in <module>
> class TestAvro(unittest.TestCase):
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/apache_beam/io/avroio_test.py",
> line 89, in TestAvro
> SCHEMA = avro.schema.parse('''
> AttributeError: module 'avro.schema' has no attribute 'parse'
> Note that we use a different implementation of avro/avro-python3 package
> depending on Python version. We are also evaluating potential replacement of
> avro with fastavro.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
