This is an automated email from the ASF dual-hosted git repository.
gooch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 765ec12 [python] Adding Unknown and Void transforms (#2697)
765ec12 is described below
commit 765ec12476e1c25fa270660b56f1ea063910831d
Author: Ted Gooch <[email protected]>
AuthorDate: Sat Jun 19 16:51:18 2021 -0700
[python] Adding Unknown and Void transforms (#2697)
Co-authored-by: tgooch <[email protected]>
---
python/iceberg/api/transforms/transforms.py | 11 +++-
python/iceberg/api/transforms/unknown_transform.py | 61 ++++++++++++++++++++++
python/iceberg/api/transforms/void_transform.py | 52 ++++++++++++++++++
3 files changed, 123 insertions(+), 1 deletion(-)
diff --git a/python/iceberg/api/transforms/transforms.py
b/python/iceberg/api/transforms/transforms.py
index 0cf243e..205a9de 100644
--- a/python/iceberg/api/transforms/transforms.py
+++ b/python/iceberg/api/transforms/transforms.py
@@ -22,6 +22,8 @@ from .dates import Dates
from .identity import Identity
from .timestamps import Timestamps
from .truncate import Truncate
+from .unknown_transform import UnknownTransform
+from .void_transform import VoidTransform
from ..types import (TypeID)
@@ -60,7 +62,10 @@ class Transforms(object):
elif type_var.type_id == TypeID.DATE:
return Dates(transform.lower(), transform.lower())
- raise RuntimeError("Unknown transform: %s" % transform)
+ if transform.lower() == "void":
+ return VoidTransform.get()
+
+ return UnknownTransform(type_var, transform)
@staticmethod
def identity(type_var):
@@ -109,3 +114,7 @@ class Transforms(object):
@staticmethod
def truncate(type_var, width):
return Truncate.get(type_var, width)
+
+ @staticmethod
+ def always_null():
+ return VoidTransform.get()
diff --git a/python/iceberg/api/transforms/unknown_transform.py
b/python/iceberg/api/transforms/unknown_transform.py
new file mode 100644
index 0000000..de32644
--- /dev/null
+++ b/python/iceberg/api/transforms/unknown_transform.py
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Union
+
+from iceberg.api.types import StringType, Type
+
+from .transform import Transform
+
+
+class UnknownTransform(Transform):
+
+ def __init__(self, source_type: Type, transform: str):
+ self.source_type = source_type
+ self.transform = transform
+
+ def apply(self, value):
+ raise AttributeError(f"Cannot apply unsupported transform:
{self.transform}")
+
+ def can_transform(self, type_var) -> bool:
+ # assume the transform function can be applied for this type because
unknown transform is only used when parsing
+ # a transform in an existing table. a different Iceberg version must
have already validated it.
+ return self.source_type == type_var
+
+ def get_result_type(self, source_type):
+ # the actual result type is not known
+ return StringType.get()
+
+ def project(self, name, predicate):
+ return None
+
+ def project_strict(self, name, predicate):
+ return None
+
+ def __str__(self):
+ return self.transform
+
+ def __eq__(self, other: Union['UnknownTransform', Transform, object]):
+ if id(self) == id(other):
+ return True
+ elif not isinstance(other, UnknownTransform):
+ return False
+
+ return self.source_type == other.source_type and self.transform ==
other.transform
+
+ def __hash__(self):
+ return hash((self.source_type, self.transform))
diff --git a/python/iceberg/api/transforms/void_transform.py
b/python/iceberg/api/transforms/void_transform.py
new file mode 100644
index 0000000..ea85964
--- /dev/null
+++ b/python/iceberg/api/transforms/void_transform.py
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .transform import Transform
+
+
+class VoidTransform(Transform):
+ _INSTANCE = None
+
+ @staticmethod
+ def get():
+ if VoidTransform._INSTANCE is None:
+ VoidTransform._INSTANCE = VoidTransform()
+ return VoidTransform._INSTANCE
+
+ def __init__(self):
+ pass
+
+ def apply(self, value):
+ return None
+
+ def can_transform(self, type_var):
+ return True
+
+ def get_result_type(self, source_type):
+ return source_type
+
+ def project(self, name, predicate):
+ return None
+
+ def project_strict(self, name, predicate):
+ return None
+
+ def to_human_string(self, value):
+ return "null"
+
+ def __str__(self):
+ return "void"
