dramaticlly commented on a change in pull request #4262: URL: https://github.com/apache/iceberg/pull/4262#discussion_r819999310
########## File path: python/src/iceberg/literals.py ########## @@ -0,0 +1,595 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime +import uuid +from decimal import ROUND_HALF_UP, Decimal + +import pytz + +from .types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + + +class Literals(object): + + EPOCH = datetime.datetime.utcfromtimestamp(0) + EPOCH_DAY = EPOCH.date() + + @staticmethod + def above_max(): + return ABOVE_MAX + + @staticmethod + def below_min(): + return BELOW_MIN + + +def from_(value): # noqa: C901 + if value is None: + raise RuntimeError("Cannot create an expression literal from None") + if isinstance(value, bool): + return BooleanLiteral(value) + elif isinstance(value, int): + if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT: + return IntegerLiteral(value) + return LongLiteral(value) + elif isinstance(value, float): + if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT: + return FloatLiteral(value) + return DoubleLiteral(value) + elif isinstance(value, str): + return StringLiteral(value) + elif isinstance(value, uuid.UUID): + return UUIDLiteral(value) + elif isinstance(value, bytearray): + return BinaryLiteral(value) + elif isinstance(value, bytes): + return FixedLiteral(value) + elif isinstance(value, Decimal): + return DecimalLiteral(value) + else: + raise NotImplementedError("Unimplemented Type Literal for value: %s" % value) + + +class Literal(object): + JAVA_MAX_INT = 2147483647 + JAVA_MIN_INT = -2147483648 + JAVA_MAX_FLOAT = 3.4028235e38 + JAVA_MIN_FLOAT = -3.4028235e38 + + def to(self, type_var): + raise NotImplementedError() + + def to_byte_buffer(self): + raise NotImplementedError() + + +def of(value): # noqa: C901 + if isinstance(value, bool): + return BooleanLiteral(value) + elif isinstance(value, int): + if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT: + return LongLiteral(value) + return IntegerLiteral(value) + elif isinstance(value, float): + if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT: + return DoubleLiteral(value) + return FloatLiteral(value) + elif isinstance(value, str): + return StringLiteral(value) + elif isinstance(value, uuid.UUID): + return UUIDLiteral(value) + elif isinstance(value, bytes): + return FixedLiteral(value) + elif isinstance(value, bytearray): + return BinaryLiteral(value) + elif isinstance(value, Decimal): + return DecimalLiteral(value) + + +class BaseLiteral(Literal): + """Base literal for all""" + + def __init__(self, value): + self.value = value + self.byte_buffer = None + + def to(self, type_var): + raise NotImplementedError() + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, BaseLiteral): + return False + + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "BaseLiteral(%s)" % str(self.value) + + def __str__(self): + return str(self.value) + + def to_byte_buffer(self): + if self.byte_buffer is None: + # TODO need byte buffer conversion + raise NotImplementedError() + + return self.byte_buffer + + +class ComparableLiteral(BaseLiteral): + def __init__(self, value): + super(ComparableLiteral, self).__init__(value) + + def to(self, type_var): + raise NotImplementedError() + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if self.value is None: + return True + + if other is None or other.value is None: + return False + + return self.value < other.value + + def __gt__(self, other): + if self.value is None: + return False + + if other is None or other.value is None: + return True + + return self.value > other.value + + def __le__(self, other): + if self.value is None: + return True + + if other is None or other.value is None: + return False + + return self.value <= other.value + + def __ge__(self, other): + if self.value is None: + return False + + if other is None or other.value is None: + return True + + return self.value >= other.value + + +class AboveMax(Literal): + def __init__(self): + super(AboveMax, self).__init__() + + def value(self): + raise RuntimeError("AboveMax has no value") + + def to(self, type): + raise RuntimeError("Cannot change the type of AboveMax") + + def __str__(self): + return "aboveMax" + + +class BelowMin(Literal): + def __init__(self): + super(BelowMin, self).__init__() + + def value(self): + raise RuntimeError("BelowMin has no value") + + def to(self, type): + raise RuntimeError("Cannot change the type of BelowMin") + + def __str__(self): + return "belowMin" + + +class BooleanLiteral(ComparableLiteral): + def __init__(self, value): + super(BooleanLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, BooleanType): + return self + + +class IntegerLiteral(ComparableLiteral): + def __init__(self, value): + super(IntegerLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, IntegerType): + return self + elif isinstance(type_var, LongType): + return LongLiteral(self.value) + elif isinstance(type_var, FloatType): + return FloatLiteral(float(self.value)) + elif isinstance(type_var, DoubleType): + return DoubleLiteral(float(self.value)) + elif isinstance(type_var, DateType): + return DateLiteral(self.value) + elif isinstance(type_var, DecimalType): + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral( + Decimal(self.value).quantize( + Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), rounding=ROUND_HALF_UP + ) + ) + + +class LongLiteral(ComparableLiteral): + def __init__(self, value): + super(LongLiteral, self).__init__(value) + + def to(self, type_var): # noqa: C901 + if isinstance(type_var, IntegerType): + if Literal.JAVA_MAX_INT < self.value: + return ABOVE_MAX + elif Literal.JAVA_MIN_INT > self.value: + return BELOW_MIN + + return IntegerLiteral(self.value) + elif isinstance(type_var, LongType): + return self + elif isinstance(type_var, FloatType): + return FloatLiteral(float(self.value)) + elif isinstance(type_var, DoubleType): + return DoubleLiteral(float(self.value)) + elif isinstance(type_var, TimeType): + return TimeLiteral(self.value) + elif isinstance(type_var, TimestampType): + return TimestampLiteral(self.value) + elif isinstance(type_var, DecimalType): + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral( + Decimal(self.value).quantize( + Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), rounding=ROUND_HALF_UP + ) + ) + + +class FloatLiteral(ComparableLiteral): + def __init__(self, value): + super(FloatLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, FloatType): + return self + elif isinstance(type_var, DoubleType): + return DoubleLiteral(self.value) + elif isinstance(type_var, DecimalType): + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value).quantize(Decimal("1."), rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral( + Decimal(self.value).quantize( + Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), rounding=ROUND_HALF_UP + ) + ) + + +class DoubleLiteral(ComparableLiteral): + def __init__(self, value): + super(DoubleLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, FloatType): + if Literal.JAVA_MAX_FLOAT < self.value: + return ABOVE_MAX + elif Literal.JAVA_MIN_FLOAT > self.value: + return BELOW_MIN + + return FloatLiteral(self.value) + elif isinstance(type_var, DoubleType): + return self + elif isinstance(type_var, DecimalType): + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value).quantize(Decimal("1."), rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral( + Decimal(self.value).quantize( + Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), rounding=ROUND_HALF_UP + ) + ) + + +class DateLiteral(ComparableLiteral): + def __init__(self, value): + super(DateLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, DateType): + return self + + +class TimeLiteral(ComparableLiteral): + def __init__(self, value): + super(TimeLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, TimeType): + return self + + +class TimestampLiteral(ComparableLiteral): + def __init__(self, value): + super(TimestampLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, TimestampType): + return self + elif isinstance(type_var, DateType): + return DateLiteral((datetime.datetime.fromtimestamp(self.value / 1000000) - Literals.EPOCH).days) + + +class DecimalLiteral(ComparableLiteral): + def __init__(self, value): + super(DecimalLiteral, self).__init__(value) + + def to(self, type_var): + if isinstance(type_var, DecimalType) and type_var.scale == abs(self.value.as_tuple().exponent): + return self + + +class StringLiteral(BaseLiteral): + def __init__(self, value): + super(StringLiteral, self).__init__(value) + + def to(self, type_var): # noqa: C901 + import dateutil.parser + + if isinstance(type_var, DateType): + return DateLiteral((dateutil.parser.parse(self.value) - Literals.EPOCH).days) + elif isinstance(type_var, TimeType): + return TimeLiteral( + int( + (dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d ") + self.value) - Literals.EPOCH).total_seconds() + * 1000000 + ) + ) + elif isinstance(type_var, TimestampType): + timestamp = dateutil.parser.parse(self.value) + if bool(timestamp.tzinfo): + raise RuntimeError(f"Cannot convert StringLiteral to {type_var} when timezone is included in {self.value}") + EPOCH = Literals.EPOCH + return TimestampLiteral(int((timestamp - EPOCH).total_seconds() * 1_000_000)) + elif isinstance(type_var, TimestamptzType): + timestamp = dateutil.parser.parse(self.value) + if not bool(timestamp.tzinfo): + raise RuntimeError(f"Cannot convert StringLiteral to {type_var} when string {self.value} miss timezones") + EPOCH = Literals.EPOCH.replace(tzinfo=pytz.UTC) + return TimestampLiteral(int((timestamp - EPOCH).total_seconds() * 1_000_000)) + elif isinstance(type_var, StringType): + return self + elif isinstance(type_var, UUIDType): + return UUIDLiteral(uuid.UUID(self.value)) + elif isinstance(type_var, DecimalType): Review comment: Thanks @samredai , Looks like the [@singledispatchmethod](https://docs.python.org/3/library/functools.html#functools.singledispatchmethod) helps but only available in python 3.8. So this actually make our tox CI on 3.7 fails. As much as I want to keep this neat way of dynamic overloading, do we know if we want to support minimally for 3.7? If so, I guess we cannot use it within the class method -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
