samredai commented on a change in pull request #4262:
URL: https://github.com/apache/iceberg/pull/4262#discussion_r819176811
##########
File path: python/tests/test_misc_literal_conversions.py
##########
@@ -0,0 +1,308 @@
+# Licensed to the Apache Software Foundation (ASF) under one
Review comment:
The convention we've been using is to create a test file named after the
source code file (so `test_literals.py` in this case) with tests localized to
contents of that particular file.
Once the individual pieces come together we'll have to think through a
structure for adding tests based more on use-cases that aren't tied to
individual files but I don't think we're there yet.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
+
+ def __str__(self):
+ return str(self.value)
+
+ def to_byte_buffer(self):
+ if self.byte_buffer is None:
+ # TODO need byte buffer conversion
+ raise NotImplementedError()
+
+ return self.byte_buffer
+
+
+class ComparableLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(ComparableLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value >= other.value
+
+
+class AboveMax(Literal):
+ def __init__(self):
+ super(AboveMax, self).__init__()
+
+ def value(self):
+ raise RuntimeError("AboveMax has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of AboveMax")
+
+ def __str__(self):
+ return "aboveMax"
+
+
+class BelowMin(Literal):
+ def __init__(self):
+ super(BelowMin, self).__init__()
+
+ def value(self):
+ raise RuntimeError("BelowMin has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of BelowMin")
+
+ def __str__(self):
+ return "belowMin"
+
+
+class BooleanLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(BooleanLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, BooleanType):
+ return self
+
+
+class IntegerLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(IntegerLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, IntegerType):
+ return self
+ elif isinstance(type_var, LongType):
+ return LongLiteral(self.value)
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, DateType):
+ return DateLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class LongLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(LongLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ if isinstance(type_var, IntegerType):
+ if Literal.JAVA_MAX_INT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_INT > self.value:
+ return BELOW_MIN
+
+ return IntegerLiteral(self.value)
+ elif isinstance(type_var, LongType):
+ return self
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(self.value)
+ elif isinstance(type_var, TimestampType):
+ return TimestampLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class FloatLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(FloatLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ return self
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DoubleLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DoubleLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ if Literal.JAVA_MAX_FLOAT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_FLOAT > self.value:
+ return BELOW_MIN
+
+ return FloatLiteral(self.value)
+ elif isinstance(type_var, DoubleType):
+ return self
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DateLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DateLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DateType):
+ return self
+
+
+class TimeLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimeLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimeType):
+ return self
+
+
+class TimestampLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimestampLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimestampType):
+ return self
+ elif isinstance(type_var, DateType):
+ return DateLiteral((datetime.datetime.fromtimestamp(self.value /
1000000) - Literals.EPOCH).days)
+
+
+class DecimalLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DecimalLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DecimalType) and type_var.scale ==
abs(self.value.as_tuple().exponent):
+ return self
+
+
+class StringLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(StringLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ import dateutil.parser
+
+ if isinstance(type_var, DateType):
+ return DateLiteral((dateutil.parser.parse(self.value) -
Literals.EPOCH).days)
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(
+ int(
+ (dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d
") + self.value) - Literals.EPOCH).total_seconds()
+ * 1000000
+ )
+ )
+ elif isinstance(type_var, TimestampType):
+ timestamp = dateutil.parser.parse(self.value)
+ if bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when timezone is included in {self.value}")
+ EPOCH = Literals.EPOCH
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, TimestamptzType):
+ timestamp = dateutil.parser.parse(self.value)
+ if not bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when string {self.value} miss timezones")
+ EPOCH = Literals.EPOCH.replace(tzinfo=pytz.UTC)
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, StringType):
+ return self
+ elif isinstance(type_var, UUIDType):
+ return UUIDLiteral(uuid.UUID(self.value))
+ elif isinstance(type_var, DecimalType):
+ dec_val = Decimal(str(self.value))
+ if abs(dec_val.as_tuple().exponent) == type_var.scale:
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(str(self.value)).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(str(self.value)).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+
+ if other is None or not isinstance(other, StringLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+ def __str__(self):
+ return '"' + self.value + '"'
+
+
+class UUIDLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(UUIDLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, UUIDType):
+ return self
+
+
+class FixedLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(FixedLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if len(self.value) == type_var.length:
+ return self
+ elif isinstance(type_var, BinaryType):
+ return BinaryLiteral(self.value)
+
+ def write_replace(self):
+ return FixedLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class BinaryLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(BinaryLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if type_var.length == len(self.value):
+ return FixedLiteral(self.value)
+ return None
+ elif isinstance(type_var, BinaryType):
+ return self
+
+ def write_replace(self):
+ return BinaryLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class FixedLiteralProxy(object):
Review comment:
`(object)` can be removed here and in a few other places.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
Review comment:
Is there any reason why these need to be class attributes? Can we set
these as module level constants at the top of the file?
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
Review comment:
I'd suggest `@singledispatch` here to enable overloading. If you need an
example I just opened PR #4263 which uses that.
##########
File path: python/tests/test_string_literal_conversions.py
##########
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid
+from datetime import datetime
+from decimal import Decimal
+
+import dateutil.parser
+from fastavro.write import LOGICAL_WRITERS as avro_conversion
Review comment:
Do we gain anything by aliasing this? I think it would be clearer to
just keep it as LOGICAL_WRITERS which is also subtly informative based on [PEP
8 - Constants](https://www.python.org/dev/peps/pep-0008/#constants) that
[LOGICAL_WRITERS](https://github.com/fastavro/fastavro/blob/master/fastavro/_logical_writers_py.py#L250)
is a constant.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
Review comment:
I'm going to spend more time looking at this but is there a reason why
we need both `Literal` and `BaseLiteral`? Could what's here be collapsed into
`BaseLiteral`?
##########
File path: python/setup.py
##########
@@ -19,7 +19,7 @@
setup(
name="py-iceberg",
- install_requires=[],
+ install_requires=["pytz", "python-dateutil", "fastavro>=1.3.2<1.4.0",
"python-dateutil"],
Review comment:
I believe these should be added to the `pyproject.toml` file instead of
here.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
+
+ def __str__(self):
+ return str(self.value)
+
+ def to_byte_buffer(self):
+ if self.byte_buffer is None:
+ # TODO need byte buffer conversion
Review comment:
I just opened PR #4263 for this!
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
+
+ def __str__(self):
+ return str(self.value)
+
+ def to_byte_buffer(self):
+ if self.byte_buffer is None:
+ # TODO need byte buffer conversion
+ raise NotImplementedError()
+
+ return self.byte_buffer
+
+
+class ComparableLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(ComparableLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value >= other.value
+
+
+class AboveMax(Literal):
+ def __init__(self):
+ super(AboveMax, self).__init__()
+
+ def value(self):
+ raise RuntimeError("AboveMax has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of AboveMax")
+
+ def __str__(self):
+ return "aboveMax"
+
+
+class BelowMin(Literal):
+ def __init__(self):
+ super(BelowMin, self).__init__()
+
+ def value(self):
+ raise RuntimeError("BelowMin has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of BelowMin")
+
+ def __str__(self):
+ return "belowMin"
+
+
+class BooleanLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(BooleanLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, BooleanType):
+ return self
+
+
+class IntegerLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(IntegerLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, IntegerType):
+ return self
+ elif isinstance(type_var, LongType):
+ return LongLiteral(self.value)
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, DateType):
+ return DateLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class LongLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(LongLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ if isinstance(type_var, IntegerType):
+ if Literal.JAVA_MAX_INT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_INT > self.value:
+ return BELOW_MIN
+
+ return IntegerLiteral(self.value)
+ elif isinstance(type_var, LongType):
+ return self
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(self.value)
+ elif isinstance(type_var, TimestampType):
+ return TimestampLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class FloatLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(FloatLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ return self
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DoubleLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DoubleLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ if Literal.JAVA_MAX_FLOAT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_FLOAT > self.value:
+ return BELOW_MIN
+
+ return FloatLiteral(self.value)
+ elif isinstance(type_var, DoubleType):
+ return self
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DateLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DateLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DateType):
+ return self
+
+
+class TimeLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimeLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimeType):
+ return self
+
+
+class TimestampLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimestampLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimestampType):
+ return self
+ elif isinstance(type_var, DateType):
+ return DateLiteral((datetime.datetime.fromtimestamp(self.value /
1000000) - Literals.EPOCH).days)
+
+
+class DecimalLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DecimalLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DecimalType) and type_var.scale ==
abs(self.value.as_tuple().exponent):
+ return self
+
+
+class StringLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(StringLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ import dateutil.parser
+
+ if isinstance(type_var, DateType):
+ return DateLiteral((dateutil.parser.parse(self.value) -
Literals.EPOCH).days)
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(
+ int(
+ (dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d
") + self.value) - Literals.EPOCH).total_seconds()
+ * 1000000
+ )
+ )
+ elif isinstance(type_var, TimestampType):
+ timestamp = dateutil.parser.parse(self.value)
+ if bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when timezone is included in {self.value}")
+ EPOCH = Literals.EPOCH
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, TimestamptzType):
+ timestamp = dateutil.parser.parse(self.value)
+ if not bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when string {self.value} miss timezones")
+ EPOCH = Literals.EPOCH.replace(tzinfo=pytz.UTC)
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, StringType):
+ return self
+ elif isinstance(type_var, UUIDType):
+ return UUIDLiteral(uuid.UUID(self.value))
+ elif isinstance(type_var, DecimalType):
+ dec_val = Decimal(str(self.value))
+ if abs(dec_val.as_tuple().exponent) == type_var.scale:
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(str(self.value)).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(str(self.value)).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+
+ if other is None or not isinstance(other, StringLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+ def __str__(self):
+ return '"' + self.value + '"'
+
+
+class UUIDLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(UUIDLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, UUIDType):
+ return self
+
+
+class FixedLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(FixedLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if len(self.value) == type_var.length:
+ return self
+ elif isinstance(type_var, BinaryType):
+ return BinaryLiteral(self.value)
+
+ def write_replace(self):
+ return FixedLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class BinaryLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(BinaryLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if type_var.length == len(self.value):
+ return FixedLiteral(self.value)
+ return None
+ elif isinstance(type_var, BinaryType):
+ return self
+
+ def write_replace(self):
+ return BinaryLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class FixedLiteralProxy(object):
+ def __init__(self, buffer=None):
+ if buffer is not None:
+ self.bytes = list(buffer)
+
+ def read_resolve(self):
+ return FixedLiteral(self.bytes)
+
+
+class ConstantExpressionProxy(object):
+ def __init__(self, true_or_false=None):
+ if true_or_false is not None:
+ self.true_or_false = true_or_false
+
+ def read_resolve(self):
+ # TODO needs expression
+ raise NotImplementedError()
+
+
+class BinaryLiteralProxy(FixedLiteralProxy):
+ def __init__(self, buffer=None):
+ super(BinaryLiteralProxy, self).__init__(buffer)
+
+ def read_resolve(self):
+ return BinaryLiteral(self.bytes)
+
+
+ABOVE_MAX = AboveMax()
+BELOW_MIN = BelowMin()
Review comment:
For consistency should we stick with what we ended up with in `types.py`
and just make the class a singleton? You can probably do something like:
```py
from iceberg.types import Singleton
class AboveMax(Literal, Singleton):
_instance = None
def __init__(self):
self._initialized = True
super(AboveMax, self).__init__()
def value(self):
raise RuntimeError("AboveMax has no value")
def to(self, type):
raise RuntimeError("Cannot change the type of AboveMax")
def __str__(self):
return "aboveMax"
```
(same for `BelowMin`)
Then where `ABOVE_MAX` is used you can just use `AboveMax()`. Let me know
what you think.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
+
+ def __str__(self):
+ return str(self.value)
+
+ def to_byte_buffer(self):
+ if self.byte_buffer is None:
+ # TODO need byte buffer conversion
+ raise NotImplementedError()
+
+ return self.byte_buffer
+
+
+class ComparableLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(ComparableLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value >= other.value
+
+
+class AboveMax(Literal):
+ def __init__(self):
+ super(AboveMax, self).__init__()
+
+ def value(self):
+ raise RuntimeError("AboveMax has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of AboveMax")
+
+ def __str__(self):
+ return "aboveMax"
+
+
+class BelowMin(Literal):
+ def __init__(self):
+ super(BelowMin, self).__init__()
+
+ def value(self):
+ raise RuntimeError("BelowMin has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of BelowMin")
+
+ def __str__(self):
+ return "belowMin"
+
+
+class BooleanLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(BooleanLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, BooleanType):
+ return self
+
+
+class IntegerLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(IntegerLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, IntegerType):
+ return self
+ elif isinstance(type_var, LongType):
+ return LongLiteral(self.value)
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, DateType):
+ return DateLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class LongLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(LongLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ if isinstance(type_var, IntegerType):
+ if Literal.JAVA_MAX_INT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_INT > self.value:
+ return BELOW_MIN
+
+ return IntegerLiteral(self.value)
+ elif isinstance(type_var, LongType):
+ return self
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(self.value)
+ elif isinstance(type_var, TimestampType):
+ return TimestampLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class FloatLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(FloatLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ return self
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DoubleLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DoubleLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ if Literal.JAVA_MAX_FLOAT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_FLOAT > self.value:
+ return BELOW_MIN
+
+ return FloatLiteral(self.value)
+ elif isinstance(type_var, DoubleType):
+ return self
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DateLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DateLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DateType):
+ return self
+
+
+class TimeLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimeLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimeType):
+ return self
+
+
+class TimestampLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimestampLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimestampType):
+ return self
+ elif isinstance(type_var, DateType):
+ return DateLiteral((datetime.datetime.fromtimestamp(self.value /
1000000) - Literals.EPOCH).days)
+
+
+class DecimalLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DecimalLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DecimalType) and type_var.scale ==
abs(self.value.as_tuple().exponent):
+ return self
+
+
+class StringLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(StringLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ import dateutil.parser
+
+ if isinstance(type_var, DateType):
+ return DateLiteral((dateutil.parser.parse(self.value) -
Literals.EPOCH).days)
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(
+ int(
+ (dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d
") + self.value) - Literals.EPOCH).total_seconds()
+ * 1000000
+ )
+ )
+ elif isinstance(type_var, TimestampType):
+ timestamp = dateutil.parser.parse(self.value)
+ if bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when timezone is included in {self.value}")
+ EPOCH = Literals.EPOCH
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, TimestamptzType):
+ timestamp = dateutil.parser.parse(self.value)
+ if not bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when string {self.value} miss timezones")
+ EPOCH = Literals.EPOCH.replace(tzinfo=pytz.UTC)
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, StringType):
+ return self
+ elif isinstance(type_var, UUIDType):
+ return UUIDLiteral(uuid.UUID(self.value))
+ elif isinstance(type_var, DecimalType):
Review comment:
I think this would be clearer if you use dynamic overloading via
[@singledispatchmethod](https://docs.python.org/3/library/functools.html#functools.singledispatchmethod)
to achieve polymorphism instead of the many if statements. It's generally
considered an anti-pattern in python to inspect the type of an argument to
determine what to do with it.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
Review comment:
Is it true that we don't expect BaseLiteral to ever be instantiated
directly? If so maybe we should remove this `__repr__` and instead add
`__repr__` methods directly to the subclasses like `BooleanLiteral` and
`IntegerLiteral`. Or you can take the same approach as
[types.py](https://github.com/apache/iceberg/blob/master/python/src/iceberg/types.py#L49-#L58)
where the repr and str methods are defined as part of the super init.
##########
File path: python/src/iceberg/literals.py
##########
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+from decimal import ROUND_HALF_UP, Decimal
+
+import pytz
+
+from .types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ LongType,
+ StringType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+class Literals(object):
+
+ EPOCH = datetime.datetime.utcfromtimestamp(0)
+ EPOCH_DAY = EPOCH.date()
+
+ @staticmethod
+ def above_max():
+ return ABOVE_MAX
+
+ @staticmethod
+ def below_min():
+ return BELOW_MIN
+
+
+def from_(value): # noqa: C901
+ if value is None:
+ raise RuntimeError("Cannot create an expression literal from None")
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT:
+ return IntegerLiteral(value)
+ return LongLiteral(value)
+ elif isinstance(value, float):
+ if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
+ return FloatLiteral(value)
+ return DoubleLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+ else:
+ raise NotImplementedError("Unimplemented Type Literal for value: %s" %
value)
+
+
+class Literal(object):
+ JAVA_MAX_INT = 2147483647
+ JAVA_MIN_INT = -2147483648
+ JAVA_MAX_FLOAT = 3.4028235e38
+ JAVA_MIN_FLOAT = -3.4028235e38
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def to_byte_buffer(self):
+ raise NotImplementedError()
+
+
+def of(value): # noqa: C901
+ if isinstance(value, bool):
+ return BooleanLiteral(value)
+ elif isinstance(value, int):
+ if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT:
+ return LongLiteral(value)
+ return IntegerLiteral(value)
+ elif isinstance(value, float):
+ if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
+ return DoubleLiteral(value)
+ return FloatLiteral(value)
+ elif isinstance(value, str):
+ return StringLiteral(value)
+ elif isinstance(value, uuid.UUID):
+ return UUIDLiteral(value)
+ elif isinstance(value, bytes):
+ return FixedLiteral(value)
+ elif isinstance(value, bytearray):
+ return BinaryLiteral(value)
+ elif isinstance(value, Decimal):
+ return DecimalLiteral(value)
+
+
+class BaseLiteral(Literal):
+ """Base literal for all"""
+
+ def __init__(self, value):
+ self.value = value
+ self.byte_buffer = None
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+ elif other is None or not isinstance(other, BaseLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "BaseLiteral(%s)" % str(self.value)
+
+ def __str__(self):
+ return str(self.value)
+
+ def to_byte_buffer(self):
+ if self.byte_buffer is None:
+ # TODO need byte buffer conversion
+ raise NotImplementedError()
+
+ return self.byte_buffer
+
+
+class ComparableLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(ComparableLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if self.value is None:
+ return True
+
+ if other is None or other.value is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if self.value is None:
+ return False
+
+ if other is None or other.value is None:
+ return True
+
+ return self.value >= other.value
+
+
+class AboveMax(Literal):
+ def __init__(self):
+ super(AboveMax, self).__init__()
+
+ def value(self):
+ raise RuntimeError("AboveMax has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of AboveMax")
+
+ def __str__(self):
+ return "aboveMax"
+
+
+class BelowMin(Literal):
+ def __init__(self):
+ super(BelowMin, self).__init__()
+
+ def value(self):
+ raise RuntimeError("BelowMin has no value")
+
+ def to(self, type):
+ raise RuntimeError("Cannot change the type of BelowMin")
+
+ def __str__(self):
+ return "belowMin"
+
+
+class BooleanLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(BooleanLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, BooleanType):
+ return self
+
+
+class IntegerLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(IntegerLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, IntegerType):
+ return self
+ elif isinstance(type_var, LongType):
+ return LongLiteral(self.value)
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, DateType):
+ return DateLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class LongLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(LongLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ if isinstance(type_var, IntegerType):
+ if Literal.JAVA_MAX_INT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_INT > self.value:
+ return BELOW_MIN
+
+ return IntegerLiteral(self.value)
+ elif isinstance(type_var, LongType):
+ return self
+ elif isinstance(type_var, FloatType):
+ return FloatLiteral(float(self.value))
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(float(self.value))
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(self.value)
+ elif isinstance(type_var, TimestampType):
+ return TimestampLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return DecimalLiteral(Decimal(self.value))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class FloatLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(FloatLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ return self
+ elif isinstance(type_var, DoubleType):
+ return DoubleLiteral(self.value)
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DoubleLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DoubleLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FloatType):
+ if Literal.JAVA_MAX_FLOAT < self.value:
+ return ABOVE_MAX
+ elif Literal.JAVA_MIN_FLOAT > self.value:
+ return BELOW_MIN
+
+ return FloatLiteral(self.value)
+ elif isinstance(type_var, DoubleType):
+ return self
+ elif isinstance(type_var, DecimalType):
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(self.value).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(self.value).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+
+class DateLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DateLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DateType):
+ return self
+
+
+class TimeLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimeLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimeType):
+ return self
+
+
+class TimestampLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(TimestampLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, TimestampType):
+ return self
+ elif isinstance(type_var, DateType):
+ return DateLiteral((datetime.datetime.fromtimestamp(self.value /
1000000) - Literals.EPOCH).days)
+
+
+class DecimalLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(DecimalLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, DecimalType) and type_var.scale ==
abs(self.value.as_tuple().exponent):
+ return self
+
+
+class StringLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(StringLiteral, self).__init__(value)
+
+ def to(self, type_var): # noqa: C901
+ import dateutil.parser
+
+ if isinstance(type_var, DateType):
+ return DateLiteral((dateutil.parser.parse(self.value) -
Literals.EPOCH).days)
+ elif isinstance(type_var, TimeType):
+ return TimeLiteral(
+ int(
+ (dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d
") + self.value) - Literals.EPOCH).total_seconds()
+ * 1000000
+ )
+ )
+ elif isinstance(type_var, TimestampType):
+ timestamp = dateutil.parser.parse(self.value)
+ if bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when timezone is included in {self.value}")
+ EPOCH = Literals.EPOCH
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, TimestamptzType):
+ timestamp = dateutil.parser.parse(self.value)
+ if not bool(timestamp.tzinfo):
+ raise RuntimeError(f"Cannot convert StringLiteral to
{type_var} when string {self.value} miss timezones")
+ EPOCH = Literals.EPOCH.replace(tzinfo=pytz.UTC)
+ return TimestampLiteral(int((timestamp - EPOCH).total_seconds() *
1_000_000))
+ elif isinstance(type_var, StringType):
+ return self
+ elif isinstance(type_var, UUIDType):
+ return UUIDLiteral(uuid.UUID(self.value))
+ elif isinstance(type_var, DecimalType):
+ dec_val = Decimal(str(self.value))
+ if abs(dec_val.as_tuple().exponent) == type_var.scale:
+ if type_var.scale == 0:
+ return
DecimalLiteral(Decimal(str(self.value)).quantize(Decimal("1."),
rounding=ROUND_HALF_UP))
+ else:
+ return DecimalLiteral(
+ Decimal(str(self.value)).quantize(
+ Decimal("." + "".join(["0" for i in range(1,
type_var.scale)]) + "1"), rounding=ROUND_HALF_UP
+ )
+ )
+
+ def __eq__(self, other):
+ if id(self) == id(other):
+ return True
+
+ if other is None or not isinstance(other, StringLiteral):
+ return False
+
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+ def __str__(self):
+ return '"' + self.value + '"'
+
+
+class UUIDLiteral(ComparableLiteral):
+ def __init__(self, value):
+ super(UUIDLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, UUIDType):
+ return self
+
+
+class FixedLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(FixedLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if len(self.value) == type_var.length:
+ return self
+ elif isinstance(type_var, BinaryType):
+ return BinaryLiteral(self.value)
+
+ def write_replace(self):
+ return FixedLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class BinaryLiteral(BaseLiteral):
+ def __init__(self, value):
+ super(BinaryLiteral, self).__init__(value)
+
+ def to(self, type_var):
+ if isinstance(type_var, FixedType):
+ if type_var.length == len(self.value):
+ return FixedLiteral(self.value)
+ return None
+ elif isinstance(type_var, BinaryType):
+ return self
+
+ def write_replace(self):
+ return BinaryLiteralProxy(self.value)
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ if other is None:
+ return False
+
+ return self.value < other.value
+
+ def __gt__(self, other):
+ if other is None:
+ return True
+
+ return self.value > other.value
+
+ def __le__(self, other):
+ if other is None:
+ return False
+
+ return self.value <= other.value
+
+ def __ge__(self, other):
+ if other is None:
+ return True
+
+ return self.value >= other.value
+
+
+class FixedLiteralProxy(object):
+ def __init__(self, buffer=None):
+ if buffer is not None:
+ self.bytes = list(buffer)
+
+ def read_resolve(self):
+ return FixedLiteral(self.bytes)
+
+
+class ConstantExpressionProxy(object):
Review comment:
@TGooch44 (or anyone else)--sorry if this is obvious but what's the
purpose of these "Proxy" classes?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
