IGNITE-10358: Added collections data type specification for python thin client
This closes #5470 Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04fae6d2 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04fae6d2 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04fae6d2 Branch: refs/heads/ignite-9720 Commit: 04fae6d2cbbd83b15a43cf17da9011c0eccec41b Parents: acc1024 Author: Dmitry Melnichuk <dmitry.melnic...@nobitlost.com> Authored: Mon Nov 26 17:14:59 2018 +0300 Committer: Igor Sapego <isap...@apache.org> Committed: Mon Nov 26 17:34:19 2018 +0300 ---------------------------------------------------------------------- .../docs/source/pyignite.datatypes.base.rst | 7 ++ .../python/docs/source/pyignite.datatypes.rst | 1 + .../platforms/python/pyignite/datatypes/base.py | 24 +++++++ .../python/pyignite/datatypes/complex.py | 23 +++--- .../python/pyignite/datatypes/internal.py | 19 +++-- .../python/pyignite/datatypes/null_object.py | 3 +- .../python/pyignite/datatypes/primitive.py | 3 +- .../pyignite/datatypes/primitive_arrays.py | 3 +- .../pyignite/datatypes/primitive_objects.py | 3 +- .../python/pyignite/datatypes/standard.py | 9 +-- modules/platforms/python/pyignite/utils.py | 6 +- modules/platforms/python/setup.py | 2 +- .../platforms/python/tests/test_key_value.py | 75 +++++++++++++++++++- 13 files changed, 149 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/docs/source/pyignite.datatypes.base.rst ---------------------------------------------------------------------- diff --git a/modules/platforms/python/docs/source/pyignite.datatypes.base.rst b/modules/platforms/python/docs/source/pyignite.datatypes.base.rst new file mode 100644 index 0000000..849a028 --- /dev/null +++ b/modules/platforms/python/docs/source/pyignite.datatypes.base.rst @@ -0,0 +1,7 @@ +pyignite.datatypes.base module +============================== + +.. automodule:: pyignite.datatypes.base + :members: + :undoc-members: + :show-inheritance: http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/docs/source/pyignite.datatypes.rst ---------------------------------------------------------------------- diff --git a/modules/platforms/python/docs/source/pyignite.datatypes.rst b/modules/platforms/python/docs/source/pyignite.datatypes.rst index 77e7183..d72f844 100644 --- a/modules/platforms/python/docs/source/pyignite.datatypes.rst +++ b/modules/platforms/python/docs/source/pyignite.datatypes.rst @@ -11,6 +11,7 @@ Submodules .. toctree:: + pyignite.datatypes.base pyignite.datatypes.binary pyignite.datatypes.cache_config pyignite.datatypes.cache_properties http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/base.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/base.py b/modules/platforms/python/pyignite/datatypes/base.py new file mode 100644 index 0000000..a0522c0 --- /dev/null +++ b/modules/platforms/python/pyignite/datatypes/base.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC + + +class IgniteDataType(ABC): + """ + This is a base class for all Ignite data types, a.k.a. parser/constructor + classes, both object and payload varieties. + """ + pass http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/complex.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/complex.py b/modules/platforms/python/pyignite/datatypes/complex.py index 9a5664c..87e5130 100644 --- a/modules/platforms/python/pyignite/datatypes/complex.py +++ b/modules/platforms/python/pyignite/datatypes/complex.py @@ -20,7 +20,8 @@ import inspect from pyignite.constants import * from pyignite.exceptions import ParseError from pyignite.utils import entity_id, hashcode, is_hinted -from .internal import AnyDataObject +from .base import IgniteDataType +from .internal import AnyDataObject, infer_from_python from .type_codes import * @@ -30,7 +31,7 @@ __all__ = [ ] -class ObjectArrayObject: +class ObjectArrayObject(IgniteDataType): """ Array of objects of any type. Its Python representation is tuple(type_id, iterable of any type). @@ -106,11 +107,11 @@ class ObjectArrayObject: buffer = bytes(header) for x in value: - buffer += AnyDataObject.from_python(x) + buffer += infer_from_python(x) return buffer -class WrappedDataObject: +class WrappedDataObject(IgniteDataType): """ One or more binary objects can be wrapped in an array. This allows reading, storing, passing and writing objects efficiently without understanding @@ -195,7 +196,7 @@ class CollectionObject(ObjectArrayObject): ) -class Map: +class Map(IgniteDataType): """ Dictionary type, payload-only. @@ -273,14 +274,8 @@ class Map: buffer = bytes(header) for k, v in value.items(): - if is_hinted(k): - buffer += k[1].from_python(k[0]) - else: - buffer += AnyDataObject.from_python(k) - if is_hinted(v): - buffer += v[1].from_python(v[0]) - else: - buffer += AnyDataObject.from_python(v) + buffer += infer_from_python(k) + buffer += infer_from_python(v) return buffer @@ -323,7 +318,7 @@ class MapObject(Map): return super().from_python(value, type_id) -class BinaryObject: +class BinaryObject(IgniteDataType): type_code = TC_COMPLEX_OBJECT USER_TYPE = 0x0001 http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/internal.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/internal.py b/modules/platforms/python/pyignite/datatypes/internal.py index a363a5f..844e0ef 100644 --- a/modules/platforms/python/pyignite/datatypes/internal.py +++ b/modules/platforms/python/pyignite/datatypes/internal.py @@ -389,6 +389,20 @@ class AnyDataObject: return cls.map_python_type(value).from_python(value) +def infer_from_python(value: Any): + """ + Convert pythonic value to ctypes buffer, type hint-aware. + + :param value: pythonic value or (value, type_hint) tuple, + :return: bytes. + """ + if is_hinted(value): + value, data_type = value + else: + data_type = AnyDataObject + return data_type.from_python(value) + + @attr.s class AnyDataArray(AnyDataObject): """ @@ -454,8 +468,5 @@ class AnyDataArray(AnyDataObject): buffer = bytes(header) for x in value: - if is_hinted(x): - buffer += x[1].from_python(x[0]) - else: - buffer += super().from_python(x) + buffer += infer_from_python(x) return buffer http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/null_object.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/null_object.py b/modules/platforms/python/pyignite/datatypes/null_object.py index 9fa1e8f..a648e30 100644 --- a/modules/platforms/python/pyignite/datatypes/null_object.py +++ b/modules/platforms/python/pyignite/datatypes/null_object.py @@ -21,13 +21,14 @@ There can't be null type, because null payload takes exactly 0 bytes. import ctypes +from .base import IgniteDataType from .type_codes import TC_NULL __all__ = ['Null'] -class Null: +class Null(IgniteDataType): default = None pythonic = type(None) _object_c_type = None http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/primitive.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/primitive.py b/modules/platforms/python/pyignite/datatypes/primitive.py index 94c8fe3..d1e9f4e 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive.py +++ b/modules/platforms/python/pyignite/datatypes/primitive.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType __all__ = [ @@ -24,7 +25,7 @@ __all__ = [ ] -class Primitive: +class Primitive(IgniteDataType): """ Ignite primitive type. Base type for the following types: http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/primitive_arrays.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/primitive_arrays.py b/modules/platforms/python/pyignite/datatypes/primitive_arrays.py index 83a2b4c..6a93191 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive_arrays.py +++ b/modules/platforms/python/pyignite/datatypes/primitive_arrays.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType from .primitive import * from .type_codes import * @@ -28,7 +29,7 @@ __all__ = [ ] -class PrimitiveArray: +class PrimitiveArray(IgniteDataType): """ Base class for array of primitives. Payload-only. """ http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/primitive_objects.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/primitive_objects.py b/modules/platforms/python/pyignite/datatypes/primitive_objects.py index 4e37ce1..105acee 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive_objects.py +++ b/modules/platforms/python/pyignite/datatypes/primitive_objects.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType from .type_codes import * @@ -25,7 +26,7 @@ __all__ = [ ] -class DataObject: +class DataObject(IgniteDataType): """ Base class for primitive data objects. http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/datatypes/standard.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/datatypes/standard.py b/modules/platforms/python/pyignite/datatypes/standard.py index 5f3af74..cc5b955 100644 --- a/modules/platforms/python/pyignite/datatypes/standard.py +++ b/modules/platforms/python/pyignite/datatypes/standard.py @@ -20,6 +20,7 @@ from math import ceil import uuid from pyignite.constants import * +from .base import IgniteDataType from .type_codes import * from .null_object import Null @@ -39,7 +40,7 @@ __all__ = [ ] -class StandardObject: +class StandardObject(IgniteDataType): type_code = None @classmethod @@ -58,7 +59,7 @@ class StandardObject: return c_type, buffer -class String: +class String(IgniteDataType): """ Pascal-style string: `c_int` counter, followed by count*bytes. UTF-8-encoded, so that one character may take 1 to 4 bytes. @@ -125,7 +126,7 @@ class String: return bytes(data_object) -class DecimalObject: +class DecimalObject(IgniteDataType): type_code = TC_DECIMAL pythonic = decimal.Decimal default = decimal.Decimal('0.00') @@ -511,7 +512,7 @@ class BinaryEnumObject(EnumObject): type_code = TC_BINARY_ENUM -class StandardArray: +class StandardArray(IgniteDataType): """ Base class for array of primitives. Payload-only. """ http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/pyignite/utils.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/pyignite/utils.py b/modules/platforms/python/pyignite/utils.py index a08bc9b..1d4298e 100644 --- a/modules/platforms/python/pyignite/utils.py +++ b/modules/platforms/python/pyignite/utils.py @@ -16,6 +16,7 @@ from functools import wraps from typing import Any, Type, Union +from pyignite.datatypes.base import IgniteDataType from .constants import * @@ -47,11 +48,14 @@ def is_hinted(value): return ( isinstance(value, tuple) and len(value) == 2 - and isinstance(value[1], object) + and issubclass(value[1], IgniteDataType) ) def is_wrapped(value: Any) -> bool: + """ + Check if a value is of WrappedDataObject type. + """ return ( type(value) is tuple and len(value) == 2 http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/setup.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/setup.py b/modules/platforms/python/setup.py index 7419c97..583eaa3 100644 --- a/modules/platforms/python/setup.py +++ b/modules/platforms/python/setup.py @@ -70,7 +70,7 @@ with open('README.md', 'r', encoding='utf-8') as readme_file: setuptools.setup( name='pyignite', - version='0.3.1', + version='0.3.4', python_requires='>={}.{}'.format(*PYTHON_REQUIRED), author='Dmitry Melnichuk', author_email='dmitry.melnic...@nobitlost.com', http://git-wip-us.apache.org/repos/asf/ignite/blob/04fae6d2/modules/platforms/python/tests/test_key_value.py ---------------------------------------------------------------------- diff --git a/modules/platforms/python/tests/test_key_value.py b/modules/platforms/python/tests/test_key_value.py index c569c77..6b4fb0e 100644 --- a/modules/platforms/python/tests/test_key_value.py +++ b/modules/platforms/python/tests/test_key_value.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime + from pyignite.api import * -from pyignite.datatypes import IntObject +from pyignite.datatypes import ( + CollectionObject, IntObject, MapObject, TimestampObject, +) def test_put_get(client, cache): @@ -325,3 +329,72 @@ def test_cache_get_size(client, cache): result = cache_get_size(client, cache) assert result.status == 0 assert result.value == 1 + + +def test_put_get_collection(client): + + test_datetime = datetime(year=1996, month=3, day=1) + + cache = client.get_or_create_cache('test_coll_cache') + cache.put( + 'simple', + ( + 1, + [ + (123, IntObject), + 678, + None, + 55.2, + ((test_datetime, 0), TimestampObject), + ] + ), + value_hint=CollectionObject + ) + value = cache.get('simple') + assert value == (1, [123, 678, None, 55.2, (test_datetime, 0)]) + + cache.put( + 'nested', + ( + 1, + [ + 123, + ((1, [456, 'inner_test_string', 789]), CollectionObject), + 'outer_test_string', + ] + ), + value_hint=CollectionObject + ) + value = cache.get('nested') + assert value == ( + 1, + [ + 123, + (1, [456, 'inner_test_string', 789]), + 'outer_test_string' + ] + ) + + +def test_put_get_map(client): + + cache = client.get_or_create_cache('test_map_cache') + + cache.put( + 'test_map', + ( + MapObject.HASH_MAP, + { + (123, IntObject): 'test_data', + 456: ((1, [456, 'inner_test_string', 789]), CollectionObject), + 'test_key': 32.4, + } + ), + value_hint=MapObject + ) + value = cache.get('test_map') + assert value == (MapObject.HASH_MAP, { + 123: 'test_data', + 456: (1, [456, 'inner_test_string', 789]), + 'test_key': 32.4, + })