Hello community,
here is the log from the commit of package python-avro for openSUSE:Factory
checked in at 2019-09-16 10:50:25
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-avro (Old)
and /work/SRC/openSUSE:Factory/.python-avro.new.7948 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-avro"
Mon Sep 16 10:50:25 2019 rev:4 rq:730689 version:1.9.1
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-avro/python-avro.changes 2019-06-22
11:24:50.757427448 +0200
+++ /work/SRC/openSUSE:Factory/.python-avro.new.7948/python-avro.changes
2019-09-16 10:50:28.787171632 +0200
@@ -1,0 +2,6 @@
+Fri Sep 13 11:26:49 UTC 2019 - Tomáš Chvátal <tchva...@suse.com>
+
+- Update to 1.9.0:
+ * no upstream changelog
+
+-------------------------------------------------------------------
Old:
----
avro-1.9.0.tar.gz
New:
----
avro-1.9.1.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-avro.spec ++++++
--- /var/tmp/diff_new_pack.9KxWoZ/_old 2019-09-16 10:50:29.823171498 +0200
+++ /var/tmp/diff_new_pack.9KxWoZ/_new 2019-09-16 10:50:29.827171497 +0200
@@ -18,19 +18,19 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
Name: python-avro
-Version: 1.9.0
+Version: 1.9.1
Release: 0
Summary: A serialization and RPC framework for Python
License: Apache-2.0
Group: Development/Languages/Python
-Url: http://avro.apache.org/
+URL: https://avro.apache.org/
Source:
https://files.pythonhosted.org/packages/source/a/avro/avro-%{version}.tar.gz
+BuildRequires: %{python_module pytest}
BuildRequires: %{python_module setuptools}
BuildRequires: fdupes
BuildRequires: python-rpm-macros
Suggests: python-python-snappy
BuildArch: noarch
-
%python_subpackages
%description
@@ -47,7 +47,9 @@
%python_expand %fdupes %{buildroot}%{$python_sitelib}
%check
-%python_exec setup.py test
+# Dies on which is nowhere even in their VCS
+# E ImportError: No module named set_avro_test_path
+#%%pytest
%files %{python_files}
%python3_only %{_bindir}/avro
++++++ avro-1.9.0.tar.gz -> avro-1.9.1.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/PKG-INFO new/avro-1.9.1/PKG-INFO
--- old/avro-1.9.0/PKG-INFO 2019-05-21 16:59:01.000000000 +0200
+++ new/avro-1.9.1/PKG-INFO 2019-09-02 09:40:28.000000000 +0200
@@ -1,8 +1,8 @@
Metadata-Version: 2.1
Name: avro
-Version: 1.9.0
+Version: 1.9.1
Summary: Avro is a serialization and RPC framework.
-Home-page: http://avro.apache.org/
+Home-page: https://avro.apache.org/
Author: Apache Avro
Author-email: d...@avro.apache.org
License: Apache License 2.0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/avro.egg-info/PKG-INFO
new/avro-1.9.1/avro.egg-info/PKG-INFO
--- old/avro-1.9.0/avro.egg-info/PKG-INFO 2019-05-21 16:59:01.000000000
+0200
+++ new/avro-1.9.1/avro.egg-info/PKG-INFO 2019-09-02 09:40:28.000000000
+0200
@@ -1,8 +1,8 @@
Metadata-Version: 2.1
Name: avro
-Version: 1.9.0
+Version: 1.9.1
Summary: Avro is a serialization and RPC framework.
-Home-page: http://avro.apache.org/
+Home-page: https://avro.apache.org/
Author: Apache Avro
Author-email: d...@avro.apache.org
License: Apache License 2.0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/avro.egg-info/SOURCES.txt
new/avro-1.9.1/avro.egg-info/SOURCES.txt
--- old/avro-1.9.0/avro.egg-info/SOURCES.txt 2019-05-21 16:59:01.000000000
+0200
+++ new/avro-1.9.1/avro.egg-info/SOURCES.txt 2019-09-02 09:40:28.000000000
+0200
@@ -9,11 +9,13 @@
src/avro/LICENSE
src/avro/NOTICE
src/avro/__init__.py
+src/avro/constants.py
src/avro/datafile.py
src/avro/io.py
src/avro/ipc.py
src/avro/protocol.py
src/avro/schema.py
+src/avro/timezones.py
src/avro/tool.py
src/avro/txipc.py
test/test_datafile.py
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/scripts/avro new/avro-1.9.1/scripts/avro
--- old/avro-1.9.0/scripts/avro 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/scripts/avro 2019-08-28 11:35:18.000000000 +0200
@@ -8,7 +8,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -209,7 +209,7 @@
argv = argv or sys.argv
parser = OptionParser(description="Display/write for Avro files",
- version="1.9.0",
+ version="1.9.1",
usage="usage: %prog cat|write [options] FILE [FILE...]")
# cat options
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/setup.py new/avro-1.9.1/setup.py
--- old/avro-1.9.0/setup.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/setup.py 2019-08-28 11:35:18.000000000 +0200
@@ -8,7 +8,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,7 +27,7 @@
setup(
name = 'avro',
- version = '1.9.0',
+ version = '1.9.1',
packages = ['avro',],
package_dir = {'avro': 'src/avro'},
scripts = ["./scripts/avro"],
@@ -45,7 +45,7 @@
description = 'Avro is a serialization and RPC framework.',
license = 'Apache License 2.0',
keywords = 'avro serialization rpc',
- url = 'http://avro.apache.org/',
+ url = 'https://avro.apache.org/',
extras_require = {
'snappy': ['python-snappy'],
},
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/LICENSE
new/avro-1.9.1/src/avro/LICENSE
--- old/avro-1.9.0/src/avro/LICENSE 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/LICENSE 2019-08-28 11:35:17.000000000 +0200
@@ -1,7 +1,7 @@
Apache License
Version 2.0, January 2004
- http://www.apache.org/licenses/
+ https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
@@ -193,7 +193,7 @@
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/NOTICE
new/avro-1.9.1/src/avro/NOTICE
--- old/avro-1.9.0/src/avro/NOTICE 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/NOTICE 2019-08-28 11:35:17.000000000 +0200
@@ -2,5 +2,5 @@
Copyright 2010-2015 The Apache Software Foundation
This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The Apache Software Foundation (https://www.apache.org/).
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/__init__.py
new/avro-1.9.1/src/avro/__init__.py
--- old/avro-1.9.0/src/avro/__init__.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/__init__.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,5 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__all__ = ['schema', 'io', 'datafile', 'protocol', 'ipc']
+__all__ = ['schema', 'io', 'datafile', 'protocol', 'ipc', 'constants',
'timezones']
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/constants.py
new/avro-1.9.1/src/avro/constants.py
--- old/avro-1.9.0/src/avro/constants.py 1970-01-01 01:00:00.000000000
+0100
+++ new/avro-1.9.1/src/avro/constants.py 2019-08-28 11:35:17.000000000
+0200
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Contains Constants for Python Avro
+"""
+
+DATE = "date"
+DECIMAL = "decimal"
+TIMESTAMP_MICROS = "timestamp-micros"
+TIMESTAMP_MILLIS = "timestamp-millis"
+TIME_MICROS = "time-micros"
+TIME_MILLIS = "time-millis"
+
+SUPPORTED_LOGICAL_TYPE = [
+ DATE,
+ DECIMAL,
+ TIMESTAMP_MICROS,
+ TIMESTAMP_MILLIS,
+ TIME_MICROS,
+ TIME_MILLIS
+]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/datafile.py
new/avro-1.9.1/src/avro/datafile.py
--- old/avro-1.9.0/src/avro/datafile.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/datafile.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/io.py
new/avro-1.9.1/src/avro/io.py
--- old/avro-1.9.0/src/avro/io.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/io.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -38,8 +38,11 @@
"""
import struct
from avro import schema
+from avro import constants
+from avro import timezones
import sys
from binascii import crc32
+import datetime
from decimal import Decimal
from decimal import getcontext
@@ -104,6 +107,9 @@
# Validate
#
+def _is_timezone_aware_datetime(dt):
+ return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None
+
def validate(expected_schema, datum):
"""Determine if a python datum is an instance of a schema."""
schema_type = expected_schema.type
@@ -119,10 +125,20 @@
return isinstance(datum, Decimal)
return isinstance(datum, str)
elif schema_type == 'int':
- return ((isinstance(datum, int) or isinstance(datum, long))
+ if hasattr(expected_schema, 'logical_type'):
+ if expected_schema.logical_type == constants.DATE:
+ return isinstance(datum, datetime.date)
+ elif expected_schema.logical_type == constants.TIME_MILLIS:
+ return isinstance(datum, datetime.time)
+ return (isinstance(datum, (int, long))
and INT_MIN_VALUE <= datum <= INT_MAX_VALUE)
elif schema_type == 'long':
- return ((isinstance(datum, int) or isinstance(datum, long))
+ if hasattr(expected_schema, 'logical_type'):
+ if expected_schema.logical_type == constants.TIME_MICROS:
+ return isinstance(datum, datetime.time)
+ elif expected_schema.logical_type in [constants.TIMESTAMP_MILLIS,
constants.TIMESTAMP_MICROS]:
+ return isinstance(datum, datetime.datetime) and
_is_timezone_aware_datetime(datum)
+ return (isinstance(datum, (int, long))
and LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE)
elif schema_type in ['float', 'double']:
return (isinstance(datum, int) or isinstance(datum, long)
@@ -280,6 +296,66 @@
"""
return unicode(self.read_bytes(), "utf-8")
+ def read_date_from_int(self):
+ """
+ int is decoded as python date object.
+ int stores the number of days from
+ the unix epoch, 1 January 1970 (ISO calendar).
+ """
+ days_since_epoch = self.read_int()
+ return datetime.date(1970, 1, 1) + datetime.timedelta(days_since_epoch)
+
+ def _build_time_object(self, value, scale_to_micro):
+ value = value * scale_to_micro
+ value, microseconds = value // 1000000, value % 1000000
+ value, seconds = value // 60, value % 60
+ value, minutes = value // 60, value % 60
+ hours = value
+
+ return datetime.time(
+ hour=hours,
+ minute=minutes,
+ second=seconds,
+ microsecond=microseconds
+ )
+
+ def read_time_millis_from_int(self):
+ """
+ int is decoded as python time object which represents
+ the number of milliseconds after midnight, 00:00:00.000.
+ """
+ milliseconds = self.read_int()
+ return self._build_time_object(milliseconds, 1000)
+
+ def read_time_micros_from_long(self):
+ """
+ long is decoded as python time object which represents
+ the number of microseconds after midnight, 00:00:00.000000.
+ """
+ microseconds = self.read_long()
+ return self._build_time_object(microseconds, 1)
+
+ def read_timestamp_millis_from_long(self):
+ """
+ long is decoded as python datetime object which represents
+ the number of milliseconds from the unix epoch, 1 January 1970.
+ """
+ timestamp_millis = self.read_long()
+ timedelta = datetime.timedelta(microseconds=timestamp_millis * 1000)
+ unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0,
tzinfo=timezones.utc)
+ return unix_epoch_datetime + timedelta
+
+ def read_timestamp_micros_from_long(self):
+ """
+ long is decoded as python datetime object which represents
+ the number of microseconds from the unix epoch, 1 January 1970.
+ """
+ timestamp_micros = self.read_long()
+ timedelta = datetime.timedelta(microseconds=timestamp_micros)
+ unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0,
tzinfo=timezones.utc)
+ return unix_epoch_datetime + timedelta
+
+
def check_crc32(self, bytes):
checksum = STRUCT_CRC32.unpack(self.read(4))[0];
if crc32(bytes) & 0xffffffff != checksum:
@@ -478,6 +554,56 @@
"""
self.write(STRUCT_CRC32.pack(crc32(bytes) & 0xffffffff));
+ def write_date_int(self, datum):
+ """
+ Encode python date object as int.
+ It stores the number of days from
+ the unix epoch, 1 January 1970 (ISO calendar).
+ """
+ delta_date = datum - datetime.date(1970, 1, 1)
+ self.write_int(delta_date.days)
+
+ def write_time_millis_int(self, datum):
+ """
+ Encode python time object as int.
+ It stores the number of milliseconds from midnight, 00:00:00.000
+ """
+ milliseconds = datum.hour*3600000 + datum.minute * 60000 + datum.second *
1000 + datum.microsecond // 1000
+ self.write_int(milliseconds)
+
+ def write_time_micros_long(self, datum):
+ """
+ Encode python time object as long.
+ It stores the number of microseconds from midnight, 00:00:00.000000
+ """
+ microseconds = datum.hour*3600000000 + datum.minute * 60000000 +
datum.second * 1000000 + datum.microsecond
+ self.write_long(microseconds)
+
+ def _timedelta_total_microseconds(self, timedelta):
+ return (
+ timedelta.microseconds + (timedelta.seconds + timedelta.days * 24 *
3600) * 10 ** 6)
+
+ def write_timestamp_millis_long(self, datum):
+ """
+ Encode python datetime object as long.
+ It stores the number of milliseconds from midnight of unix epoch, 1
January 1970.
+ """
+ datum = datum.astimezone(tz=timezones.utc)
+ timedelta = datum - datetime.datetime(1970, 1, 1, 0, 0, 0, 0,
tzinfo=timezones.utc)
+ milliseconds = self._timedelta_total_microseconds(timedelta) / 1000
+ self.write_long(long(milliseconds))
+
+ def write_timestamp_micros_long(self, datum):
+ """
+ Encode python datetime object as long.
+ It stores the number of microseconds from midnight of unix epoch, 1
January 1970.
+ """
+ datum = datum.astimezone(tz=timezones.utc)
+ timedelta = datum - datetime.datetime(1970, 1, 1, 0, 0, 0, 0,
tzinfo=timezones.utc)
+ microseconds = self._timedelta_total_microseconds(timedelta)
+ self.write_long(long(microseconds))
+
+
#
# DatumReader/Writer
#
@@ -583,9 +709,26 @@
elif writers_schema.type == 'string':
return decoder.read_utf8()
elif writers_schema.type == 'int':
- return decoder.read_int()
+ if (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.DATE):
+ return decoder.read_date_from_int()
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIME_MILLIS):
+ return decoder.read_time_millis_from_int()
+ else:
+ return decoder.read_int()
elif writers_schema.type == 'long':
- return decoder.read_long()
+ if (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIME_MICROS):
+ return decoder.read_time_micros_from_long()
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIMESTAMP_MILLIS):
+ return decoder.read_timestamp_millis_from_long()
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIMESTAMP_MICROS):
+ return decoder.read_timestamp_micros_from_long()
+ else:
+ return decoder.read_long()
elif writers_schema.type == 'float':
return decoder.read_float()
elif writers_schema.type == 'double':
@@ -909,9 +1052,26 @@
elif writers_schema.type == 'string':
encoder.write_utf8(datum)
elif writers_schema.type == 'int':
- encoder.write_int(datum)
+ if (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.DATE):
+ encoder.write_date_int(datum)
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIME_MILLIS):
+ encoder.write_time_millis_int(datum)
+ else:
+ encoder.write_int(datum)
elif writers_schema.type == 'long':
- encoder.write_long(datum)
+ if (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIME_MICROS):
+ encoder.write_time_micros_long(datum)
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIMESTAMP_MILLIS):
+ encoder.write_timestamp_millis_long(datum)
+ elif (hasattr(writers_schema, 'logical_type') and
+ writers_schema.logical_type == constants.TIMESTAMP_MICROS):
+ encoder.write_timestamp_micros_long(datum)
+ else:
+ encoder.write_long(datum)
elif writers_schema.type == 'float':
encoder.write_float(datum)
elif writers_schema.type == 'double':
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/ipc.py
new/avro-1.9.1/src/avro/ipc.py
--- old/avro-1.9.0/src/avro/ipc.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/ipc.py 2019-08-28 11:35:18.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/protocol.py
new/avro-1.9.1/src/avro/protocol.py
--- old/avro-1.9.0/src/avro/protocol.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/protocol.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/schema.py
new/avro-1.9.1/src/avro/schema.py
--- old/avro-1.9.0/src/avro/schema.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/schema.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -41,6 +41,8 @@
except ImportError:
import simplejson as json
+from avro import constants
+
#
# Constants
#
@@ -347,7 +349,7 @@
raise SchemaParseException("Invalid DECIMAL scale %s. Cannot be greater
than precision %s"
%(scale, precision))
- LogicalSchema.__init__(self, 'decimal')
+ super(DecimalLogicalSchema, self).__init__('decimal')
def _max_precision(self):
raise NotImplementedError()
@@ -788,6 +790,91 @@
to_cmp = json.loads(str(self))
return to_cmp == json.loads(str(that))
+
+#
+# Logical Type
+#
+
+class LogicalSchema(object):
+ def __init__(self, logical_type):
+ self.logical_type = logical_type
+
+
+#
+# Date Type
+#
+
+class DateSchema(LogicalSchema, PrimitiveSchema):
+ def __init__(self, other_props=None):
+ LogicalSchema.__init__(self, constants.DATE)
+ PrimitiveSchema.__init__(self, 'int', other_props)
+
+ def to_json(self, names=None):
+ return self.props
+
+ def __eq__(self, that):
+ return self.props == that.props
+
+#
+# time-millis Type
+#
+
+class TimeMillisSchema(LogicalSchema, PrimitiveSchema):
+ def __init__(self, other_props=None):
+ LogicalSchema.__init__(self, constants.TIME_MILLIS)
+ PrimitiveSchema.__init__(self, 'int', other_props)
+
+ def to_json(self, names=None):
+ return self.props
+
+ def __eq__(self, that):
+ return self.props == that.props
+
+#
+# time-micros Type
+#
+
+class TimeMicrosSchema(LogicalSchema, PrimitiveSchema):
+ def __init__(self, other_props=None):
+ LogicalSchema.__init__(self, constants.TIME_MICROS)
+ PrimitiveSchema.__init__(self, 'long', other_props)
+
+ def to_json(self, names=None):
+ return self.props
+
+ def __eq__(self, that):
+ return self.props == that.props
+
+#
+# timestamp-millis Type
+#
+
+class TimestampMillisSchema(LogicalSchema, PrimitiveSchema):
+ def __init__(self, other_props=None):
+ LogicalSchema.__init__(self, constants.TIMESTAMP_MILLIS)
+ PrimitiveSchema.__init__(self, 'long', other_props)
+
+ def to_json(self, names=None):
+ return self.props
+
+ def __eq__(self, that):
+ return self.props == that.props
+
+#
+# timestamp-micros Type
+#
+
+class TimestampMicrosSchema(LogicalSchema, PrimitiveSchema):
+ def __init__(self, other_props=None):
+ LogicalSchema.__init__(self, constants.TIMESTAMP_MICROS)
+ PrimitiveSchema.__init__(self, 'long', other_props)
+
+ def to_json(self, names=None):
+ return self.props
+
+ def __eq__(self, that):
+ return self.props == that.props
+
#
# Module Methods
#
@@ -817,11 +904,20 @@
logical_type = None
if 'logicalType' in json_data:
logical_type = json_data.get('logicalType')
- if logical_type != 'decimal':
- raise SchemaParseException("Currently does not support %s logical type"
% logical_type)
+ if logical_type not in constants.SUPPORTED_LOGICAL_TYPE:
+ raise SchemaParseException("Currently does not support %s logical
type" % logical_type)
if type in PRIMITIVE_TYPES:
- if type == 'bytes':
- if logical_type == 'decimal':
+ if type == 'int' and logical_type == constants.DATE:
+ return DateSchema(other_props)
+ if type == 'int' and logical_type == constants.TIME_MILLIS:
+ return TimeMillisSchema(other_props=other_props)
+ if type == 'long' and logical_type == constants.TIME_MICROS:
+ return TimeMicrosSchema(other_props=other_props)
+ if type == 'long' and logical_type == constants.TIMESTAMP_MILLIS:
+ return TimestampMillisSchema(other_props=other_props)
+ if type == 'long' and logical_type == constants.TIMESTAMP_MICROS:
+ return TimestampMicrosSchema(other_props=other_props)
+ if type == 'bytes' and logical_type == constants.DECIMAL:
precision = json_data.get('precision')
scale = 0 if json_data.get('scale') is None else
json_data.get('scale')
return BytesDecimalSchema(precision, scale, other_props)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/timezones.py
new/avro-1.9.1/src/avro/timezones.py
--- old/avro-1.9.0/src/avro/timezones.py 1970-01-01 01:00:00.000000000
+0100
+++ new/avro-1.9.1/src/avro/timezones.py 2019-08-28 11:35:17.000000000
+0200
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from datetime import timedelta
+from datetime import tzinfo
+
+
+class UTCTzinfo(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(0)
+
+ def tzname(self, dt):
+ return "UTC"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+utc = UTCTzinfo()
+
+
+# Test Time Zone with fixed offset and no DST
+class TSTTzinfo(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=10)
+
+ def tzname(self, dt):
+ return "TST"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+tst = TSTTzinfo()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/tool.py
new/avro-1.9.1/src/avro/tool.py
--- old/avro-1.9.0/src/avro/tool.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/tool.py 2019-08-28 11:35:17.000000000 +0200
@@ -7,7 +7,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/src/avro/txipc.py
new/avro-1.9.1/src/avro/txipc.py
--- old/avro-1.9.0/src/avro/txipc.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/src/avro/txipc.py 2019-08-28 11:35:17.000000000 +0200
@@ -8,7 +8,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_datafile.py
new/avro-1.9.1/test/test_datafile.py
--- old/avro-1.9.0/test/test_datafile.py 2019-05-21 15:54:34.000000000
+0200
+++ new/avro-1.9.1/test/test_datafile.py 2019-08-28 11:35:17.000000000
+0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_datafile_interop.py
new/avro-1.9.1/test/test_datafile_interop.py
--- old/avro-1.9.0/test/test_datafile_interop.py 2019-05-21
15:54:35.000000000 +0200
+++ new/avro-1.9.1/test/test_datafile_interop.py 2019-08-28
11:35:18.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_io.py
new/avro-1.9.1/test/test_io.py
--- old/avro-1.9.0/test/test_io.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/test/test_io.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,11 +22,13 @@
except ImportError:
from StringIO import StringIO
from binascii import hexlify
+import datetime
import set_avro_test_path
from avro import schema
from avro import io
+from avro import timezones
SCHEMAS_TO_VALIDATE = (
('"null"', None),
@@ -48,6 +50,35 @@
('{"type": "array", "items": "long"}', [1, 3, 2]),
('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}),
('["string", "null", "long"]', None),
+ ('{"type": "int", "logicalType": "date"}', datetime.date(2000, 1, 1)),
+ ('{"type": "int", "logicalType": "time-millis"}', datetime.time(23, 59, 59,
999000)),
+ ('{"type": "int", "logicalType": "time-millis"}', datetime.time(0, 0, 0,
000000)),
+ ('{"type": "long", "logicalType": "time-micros"}', datetime.time(23, 59, 59,
999999)),
+ ('{"type": "long", "logicalType": "time-micros"}', datetime.time(0, 0, 0,
000000)),
+ (
+ '{"type": "long", "logicalType": "timestamp-millis"}',
+ datetime.datetime(1000, 1, 1, 0, 0, 0, 000000, tzinfo=timezones.utc)
+ ),
+ (
+ '{"type": "long", "logicalType": "timestamp-millis"}',
+ datetime.datetime(9999, 12, 31, 23, 59, 59, 999000, tzinfo=timezones.utc)
+ ),
+ (
+ '{"type": "long", "logicalType": "timestamp-millis"}',
+ datetime.datetime(2000, 1, 18, 2, 2, 1, 100000, tzinfo=timezones.tst)
+ ),
+ (
+ '{"type": "long", "logicalType": "timestamp-micros"}',
+ datetime.datetime(1000, 1, 1, 0, 0, 0, 000000, tzinfo=timezones.utc)
+ ),
+ (
+ '{"type": "long", "logicalType": "timestamp-micros"}',
+ datetime.datetime(9999, 12, 31, 23, 59, 59, 999999, tzinfo=timezones.utc)
+ ),
+ (
+ '{"type": "long", "logicalType": "timestamp-micros"}',
+ datetime.datetime(2000, 1, 18, 2, 2, 1, 123499, tzinfo=timezones.tst)
+ ),
("""\
{"type": "record",
"name": "Test",
@@ -211,7 +242,10 @@
if isinstance(round_trip_datum, Decimal):
round_trip_datum = round_trip_datum.to_eng_string()
datum = str(datum)
- if datum == round_trip_datum: correct += 1
+ elif isinstance(round_trip_datum, datetime.datetime):
+ datum = datum.astimezone(tz=timezones.utc)
+ if datum == round_trip_datum:
+ correct += 1
self.assertEquals(correct, len(SCHEMAS_TO_VALIDATE))
#
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_ipc.py
new/avro-1.9.1/test/test_ipc.py
--- old/avro-1.9.0/test/test_ipc.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/test/test_ipc.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_protocol.py
new/avro-1.9.1/test/test_protocol.py
--- old/avro-1.9.0/test/test_protocol.py 2019-05-21 15:54:34.000000000
+0200
+++ new/avro-1.9.1/test/test_protocol.py 2019-08-28 11:35:17.000000000
+0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_schema.py
new/avro-1.9.1/test/test_schema.py
--- old/avro-1.9.0/test/test_schema.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/test/test_schema.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -313,6 +313,67 @@
"scale": 2}""", True)
]
+DATE_LOGICAL_TYPE = [
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "date"} """, True),
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "date1"} """, False),
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "date"} """, False),
+]
+
+TIMEMILLIS_LOGICAL_TYPE = [
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "time-millis"} """, True),
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "time-milis"} """, False),
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "time-millis"} """, False),
+]
+
+TIMEMICROS_LOGICAL_TYPE = [
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "time-micros"} """, True),
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "time-micro"} """, False),
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "time-micros"} """, False),
+]
+
+TIMESTAMPMILLIS_LOGICAL_TYPE = [
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "timestamp-millis"} """, True),
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "timestamp-milis"} """, False),
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "timestamp-millis"} """, False),
+]
+
+TIMESTAMPMICROS_LOGICAL_TYPE = [
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "timestamp-micros"} """, True),
+ ExampleSchema("""{
+ "type": "long",
+ "logicalType": "timestamp-micro"} """, False),
+ ExampleSchema("""{
+ "type": "int",
+ "logicalType": "timestamp-micros"} """, False),
+]
+
+
EXAMPLES = PRIMITIVE_EXAMPLES
EXAMPLES += FIXED_EXAMPLES
EXAMPLES += ENUM_EXAMPLES
@@ -322,6 +383,11 @@
EXAMPLES += RECORD_EXAMPLES
EXAMPLES += DOC_EXAMPLES
EXAMPLES += DECIMAL_LOGICAL_TYPE
+EXAMPLES += DATE_LOGICAL_TYPE
+EXAMPLES += TIMEMILLIS_LOGICAL_TYPE
+EXAMPLES += TIMEMICROS_LOGICAL_TYPE
+EXAMPLES += TIMESTAMPMILLIS_LOGICAL_TYPE
+EXAMPLES += TIMESTAMPMICROS_LOGICAL_TYPE
VALID_EXAMPLES = [e for e in EXAMPLES if e.valid]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_script.py
new/avro-1.9.1/test/test_script.py
--- old/avro-1.9.0/test/test_script.py 2019-05-21 15:54:34.000000000 +0200
+++ new/avro-1.9.1/test/test_script.py 2019-08-28 11:35:17.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_tether_task.py
new/avro-1.9.1/test/test_tether_task.py
--- old/avro-1.9.0/test/test_tether_task.py 2019-05-21 15:54:34.000000000
+0200
+++ new/avro-1.9.1/test/test_tether_task.py 2019-08-28 11:35:17.000000000
+0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_tether_task_runner.py
new/avro-1.9.1/test/test_tether_task_runner.py
--- old/avro-1.9.0/test/test_tether_task_runner.py 2019-05-21
15:54:34.000000000 +0200
+++ new/avro-1.9.1/test/test_tether_task_runner.py 2019-08-28
11:35:18.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/avro-1.9.0/test/test_tether_word_count.py
new/avro-1.9.1/test/test_tether_word_count.py
--- old/avro-1.9.0/test/test_tether_word_count.py 2019-05-21
15:54:36.000000000 +0200
+++ new/avro-1.9.1/test/test_tether_word_count.py 2019-08-28
11:35:21.000000000 +0200
@@ -6,7 +6,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -146,7 +146,7 @@
args.append("java")
args.append("-jar")
-
args.append(os.path.abspath("/avro/lang/py/../java/tools/target/avro-tools-1.9.0.jar"))
+
args.append(os.path.abspath("/avro/lang/py/../java/tools/target/avro-tools-1.9.1.jar"))
args.append("tether")
