This is an automated email from the ASF dual-hosted git repository.
kojiromike pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 1d55d10 AVRO-2611: Use warnings for invalid logical types. (#708)
1d55d10 is described below
commit 1d55d10ed9200084d2603f2a6b136aede1f530b2
Author: RyanSkraba <[email protected]>
AuthorDate: Wed Nov 20 03:03:41 2019 +0100
AVRO-2611: Use warnings for invalid logical types. (#708)
* AVRO-2611: Use warnings for invalid logical types.
AVRO-2611: Add unit tests and restore warnings.
AVRO-2611: Add warnings to unit tests.
AVRO-2611: Update messages for missing precision.
* AVRO-2611: Typo before knowing the scale type.
* AVRO-2611: Clean up docstrings and format.
Fixes after code review.
* Simplify list sort.
* Update precision attribute to explicit fp number.
---
lang/py/src/avro/schema.py | 65 +++++++++++++++++----------
lang/py/test/test_schema.py | 107 ++++++++++++++++++++++++++++++++++----------
2 files changed, 124 insertions(+), 48 deletions(-)
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index 6e3ebfa..60a826d 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -43,6 +43,7 @@ from __future__ import absolute_import, division,
print_function
import json
import math
import sys
+import warnings
from avro import constants
@@ -112,6 +113,12 @@ class AvroException(Exception):
class SchemaParseException(AvroException):
pass
+class AvroWarning(UserWarning):
+ """Base class for warnings."""
+
+class IgnoredLogicalType(AvroWarning):
+ """Warnings for unknown or invalid logical types."""
+
#
# Base Classes
#
@@ -337,19 +344,20 @@ class LogicalSchema(object):
class DecimalLogicalSchema(LogicalSchema):
def __init__(self, precision, scale=0, max_precision=0):
if not isinstance(precision, int) or precision <= 0:
- raise SchemaParseException("""Precision is required for logical type
- DECIMAL and must be a positive integer but
- is %s.""" % precision)
- elif precision > max_precision:
- raise SchemaParseException("Cannot store precision digits. Max is %s"
- %(max_precision))
+ raise IgnoredLogicalType(
+ "Invalid decimal precision {}. Must be a positive
integer.".format(precision))
+
+ if precision > max_precision:
+ raise IgnoredLogicalType(
+ "Invalid decimal precision {}. Max is {}.".format(precision,
max_precision))
if not isinstance(scale, int) or scale < 0:
- raise SchemaParseException("Scale %s must be a positive Integer." %
scale)
+ raise IgnoredLogicalType(
+ "Invalid decimal scale {}. Must be a positive
integer.".format(scale))
- elif scale > precision:
- raise SchemaParseException("Invalid DECIMAL scale %s. Cannot be greater
than precision %s"
- %(scale, precision))
+ if scale > precision:
+ raise IgnoredLogicalType("Invalid decimal scale {}. Cannot be greater
than precision {}."
+ .format(scale, precision))
super(DecimalLogicalSchema, self).__init__('decimal')
@@ -506,7 +514,7 @@ class FixedSchema(NamedSchema):
class FixedDecimalSchema(FixedSchema, DecimalLogicalSchema):
def __init__(self, size, name, precision, scale=0, namespace=None,
names=None, other_props=None):
- max_precision = math.floor(math.log10(2) * (8 * size - 1))
+ max_precision = int(math.floor(math.log10(2) * (8 * size - 1)))
DecimalLogicalSchema.__init__(self, precision, scale, max_precision)
FixedSchema.__init__(self, name, namespace, size, names, other_props)
self.set_prop('precision', precision)
@@ -887,20 +895,29 @@ def make_bytes_decimal_schema(other_props):
def make_logical_schema(logical_type, type_, other_props):
"""Map the logical types to the appropriate literal type and schema class."""
logical_types = {
- constants.DATE: ('int', DateSchema),
- # Fixed decimal schema is handled before we get here.
- constants.DECIMAL: ('bytes', make_bytes_decimal_schema),
- constants.TIMESTAMP_MICROS: ('long', TimestampMicrosSchema),
- constants.TIMESTAMP_MILLIS: ('long', TimestampMillisSchema),
- constants.TIME_MICROS: ('long', TimeMicrosSchema),
- constants.TIME_MILLIS: ('int', TimeMillisSchema),
+ (constants.DATE, 'int'): DateSchema,
+ (constants.DECIMAL, 'bytes'): make_bytes_decimal_schema,
+ # The fixed decimal schema is handled later by returning None now.
+ (constants.DECIMAL, 'fixed'): lambda x: None,
+ (constants.TIMESTAMP_MICROS, 'long'): TimestampMicrosSchema,
+ (constants.TIMESTAMP_MILLIS, 'long'): TimestampMillisSchema,
+ (constants.TIME_MICROS, 'long'): TimeMicrosSchema,
+ (constants.TIME_MILLIS, 'int'): TimeMillisSchema,
}
- literal_type, schema_type = logical_types.get(logical_type, (None, None))
try:
- if literal_type == type_:
+ schema_type = logical_types.get((logical_type, type_), None)
+ if schema_type is not None:
return schema_type(other_props)
- except SchemaParseException:
- pass
+
+ expected_types = sorted(literal_type for lt, literal_type in logical_types
if lt == logical_type)
+ if expected_types:
+ warnings.warn(
+ IgnoredLogicalType("Logical type {} requires literal type {}, not
{}.".format(
+ logical_type, "/".join(expected_types), type_)))
+ else:
+ warnings.warn(IgnoredLogicalType("Unknown {}, using
{}.".format(logical_type, type_)))
+ except IgnoredLogicalType as warning:
+ warnings.warn(warning)
return None
def make_avsc_object(json_data, names=None):
@@ -931,8 +948,8 @@ def make_avsc_object(json_data, names=None):
scale = 0 if json_data.get('scale') is None else
json_data.get('scale')
try:
return FixedDecimalSchema(size, name, precision, scale, namespace,
names, other_props)
- except (AvroException, SchemaParseException):
- pass
+ except IgnoredLogicalType as warning:
+ warnings.warn(warning)
return FixedSchema(name, namespace, size, names, other_props)
elif type == 'enum':
symbols = json_data.get('symbols')
diff --git a/lang/py/test/test_schema.py b/lang/py/test/test_schema.py
index 6186ca2..ea6779d 100644
--- a/lang/py/test/test_schema.py
+++ b/lang/py/test/test_schema.py
@@ -23,21 +23,22 @@ from __future__ import absolute_import, division,
print_function
import json
import unittest
+import warnings
import set_avro_test_path
from avro import schema
-from avro.schema import AvroException, SchemaParseException
class TestSchema(object):
"""A proxy for a schema string that provides useful test metadata."""
- def __init__(self, data, name='', comment=''):
+ def __init__(self, data, name='', comment='', warnings=None):
if not isinstance(data, basestring):
data = json.dumps(data)
self.data = data
self.name = name or data # default to data for name
self.comment = comment
+ self.warnings = warnings
def parse(self):
return schema.parse(str(self))
@@ -217,23 +218,72 @@ TIMESTAMPMICROS_LOGICAL_TYPE = [
]
IGNORED_LOGICAL_TYPE = [
- ValidTestSchema({"type": "string", "logicalType": "uuid"}),
- ValidTestSchema({"type": "string", "logicalType": "unknown-logical-type"}),
- ValidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": 2,
"scale": -2}),
- ValidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": -2,
"scale": 2}),
- ValidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": 2,
"scale": 3}),
- ValidTestSchema({"type": "fixed", "logicalType": "decimal", "name":
"TestIgnored", "precision": -10, "scale": 2, "size": 5}),
- ValidTestSchema({"type": "fixed", "logicalType": "decimal", "name":
"TestIgnored2", "precision": 2, "scale": 3, "size": 2}),
- ValidTestSchema({"type": "int", "logicalType": "date1"}),
- ValidTestSchema({"type": "long", "logicalType": "date"}),
- ValidTestSchema({"type": "int", "logicalType": "time-milis"}),
- ValidTestSchema({"type": "long", "logicalType": "time-millis"}),
- ValidTestSchema({"type": "long", "logicalType": "time-micro"}),
- ValidTestSchema({"type": "int", "logicalType": "time-micros"}),
- ValidTestSchema({"type": "long", "logicalType": "timestamp-milis"}),
- ValidTestSchema({"type": "int", "logicalType": "timestamp-millis"}),
- ValidTestSchema({"type": "long", "logicalType": "timestamp-micro"}),
- ValidTestSchema({"type": "int", "logicalType": "timestamp-micros"})
+ ValidTestSchema(
+ {"type": "string", "logicalType": "uuid"},
+ warnings=[schema.IgnoredLogicalType('Unknown uuid, using string.')]),
+ ValidTestSchema(
+ {"type": "string", "logicalType": "unknown-logical-type"},
+ warnings=[schema.IgnoredLogicalType('Unknown unknown-logical-type, using
string.')]),
+ ValidTestSchema(
+ {"type": "bytes", "logicalType": "decimal", "scale": 0},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision None. Must
be a positive integer.')]),
+ ValidTestSchema(
+ {"type": "bytes", "logicalType": "decimal", "precision": 2.4, "scale": 0},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision 2.4. Must
be a positive integer.')]),
+ ValidTestSchema(
+ {"type": "bytes", "logicalType": "decimal", "precision": 2, "scale": -2},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal scale -2. Must be a
positive integer.')]),
+ ValidTestSchema(
+ {"type": "bytes", "logicalType": "decimal", "precision": -2, "scale": 2},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision -2. Must be
a positive integer.')]),
+ ValidTestSchema(
+ {"type": "bytes", "logicalType": "decimal", "precision": 2, "scale": 3},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal scale 3. Cannot be
greater than precision 2.')]),
+ ValidTestSchema(
+ {"type": "fixed", "logicalType": "decimal", "name": "TestIgnored",
"precision": -10, "scale": 2, "size": 5},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision -10. Must
be a positive integer.')]),
+ ValidTestSchema(
+ {"type": "fixed", "logicalType": "decimal", "name": "TestIgnored",
"scale": 2, "size": 5},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision None. Must
be a positive integer.')]),
+ ValidTestSchema(
+ {"type": "fixed", "logicalType": "decimal", "name": "TestIgnored",
"precision": 2, "scale": 3, "size": 2},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal scale 3. Cannot be
greater than precision 2.')]),
+ ValidTestSchema(
+ {"type": "fixed", "logicalType": "decimal", "name": "TestIgnored",
"precision": 311, "size": 129},
+ warnings=[schema.IgnoredLogicalType('Invalid decimal precision 311. Max is
310.')]),
+ ValidTestSchema(
+ {"type": "float", "logicalType": "decimal", "precision": 2, "scale": 0},
+ warnings=[schema.IgnoredLogicalType('Logical type decimal requires literal
type bytes/fixed, not float.')]),
+ ValidTestSchema(
+ {"type": "int", "logicalType": "date1"},
+ warnings=[schema.IgnoredLogicalType('Unknown date1, using int.')]),
+ ValidTestSchema(
+ {"type": "long", "logicalType": "date"},
+ warnings=[schema.IgnoredLogicalType('Logical type date requires literal
type int, not long.')]),
+ ValidTestSchema(
+ {"type": "int", "logicalType": "time-milis"},
+ warnings=[schema.IgnoredLogicalType('Unknown time-milis, using int.')]),
+ ValidTestSchema(
+ {"type": "long", "logicalType": "time-millis"},
+ warnings=[schema.IgnoredLogicalType('Logical type time-millis requires
literal type int, not long.')]),
+ ValidTestSchema(
+ {"type": "long", "logicalType": "time-micro"},
+ warnings=[schema.IgnoredLogicalType('Unknown time-micro, using long.')]),
+ ValidTestSchema(
+ {"type": "int", "logicalType": "time-micros"},
+ warnings=[schema.IgnoredLogicalType('Logical type time-micros requires
literal type long, not int.')]),
+ ValidTestSchema(
+ {"type": "long", "logicalType": "timestamp-milis"},
+ warnings=[schema.IgnoredLogicalType('Unknown timestamp-milis, using
long.')]),
+ ValidTestSchema(
+ {"type": "int", "logicalType": "timestamp-millis"},
+ warnings=[schema.IgnoredLogicalType('Logical type timestamp-millis
requires literal type long, not int.')]),
+ ValidTestSchema(
+ {"type": "long", "logicalType": "timestamp-micro"},
+ warnings=[schema.IgnoredLogicalType('Unknown timestamp-micro, using
long.')]),
+ ValidTestSchema(
+ {"type": "int", "logicalType": "timestamp-micros"},
+ warnings=[schema.IgnoredLogicalType('Logical type timestamp-micros
requires literal type long, not int.')])
]
EXAMPLES = PRIMITIVE_EXAMPLES
@@ -397,13 +447,22 @@ class SchemaParseTestCase(unittest.TestCase):
super(SchemaParseTestCase, self).__init__(
'parse_valid' if test_schema.valid else 'parse_invalid')
self.test_schema = test_schema
+ # Never hide repeated warnings when running this test case.
+ warnings.simplefilter("always")
def parse_valid(self):
- """Parsing a valid schema should not error."""
- try:
- self.test_schema.parse()
- except (schema.AvroException, schema.SchemaParseException):
- self.fail("Valid schema failed to parse: {!s}".format(self.test_schema))
+ """Parsing a valid schema should not error, but may contain warnings."""
+ with warnings.catch_warnings(record=True) as actual_warnings:
+ try:
+ self.test_schema.parse()
+ except (schema.AvroException, schema.SchemaParseException):
+ self.fail("Valid schema failed to parse:
{!s}".format(self.test_schema))
+ actual_messages = [str(wmsg.message) for wmsg in actual_warnings]
+ if self.test_schema.warnings:
+ expected_messages = [str(w) for w in self.test_schema.warnings]
+ self.assertItemsEqual(actual_messages, expected_messages)
+ else:
+ self.assertEqual(actual_messages, [])
def parse_invalid(self):
"""Parsing an invalid schema should error."""
