This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new fabdbc9aa [#5200]Update the Column data type for python client (#5354)
fabdbc9aa is described below
commit fabdbc9aa424aa76bf6109244b64c40e7acad934
Author: SophieTech88 <[email protected]>
AuthorDate: Wed Nov 6 08:04:06 2024 -0600
[#5200]Update the Column data type for python client (#5354)
### What changes were proposed in this pull request?
1. Implement type.java as type.py in Python client
2. Implement types.java as types.py in Python client
3. Create a test_types.py in unit tests.
### Why are the changes needed?
We need to support the column data type in python client
Fix: #5200
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Need to add the unit tests
---------
Co-authored-by: Xun <[email protected]>
---
clients/client-python/gravitino/api/type.py | 180 ++++
clients/client-python/gravitino/api/types.py | 1106 ++++++++++++++++++++
.../tests/unittests/test_gvfs_with_local.py | 1 +
.../client-python/tests/unittests/test_types.py | 195 ++++
docs/manage-relational-metadata-using-gravitino.md | 2 +-
5 files changed, 1483 insertions(+), 1 deletion(-)
diff --git a/clients/client-python/gravitino/api/type.py
b/clients/client-python/gravitino/api/type.py
new file mode 100644
index 000000000..9b089ea18
--- /dev/null
+++ b/clients/client-python/gravitino/api/type.py
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC, abstractmethod
+from enum import Enum
+
+
+class Name(Enum):
+ """
+ The root type name of this type, representing all data types supported.
+ """
+
+ BOOLEAN = "BOOLEAN"
+ """ The boolean type. """
+
+ BYTE = "BYTE"
+ """ The byte type. """
+
+ SHORT = "SHORT"
+ """ The short type. """
+
+ INTEGER = "INTEGER"
+ """ The integer type. """
+
+ LONG = "LONG"
+ """ The long type. """
+
+ FLOAT = "FLOAT"
+ """ The float type. """
+
+ DOUBLE = "DOUBLE"
+ """ The double type. """
+
+ DECIMAL = "DECIMAL"
+ """ The decimal type. """
+
+ DATE = "DATE"
+ """ The date type. """
+
+ TIME = "TIME"
+ """ The time type. """
+
+ TIMESTAMP = "TIMESTAMP"
+ """ The timestamp type. """
+
+ INTERVAL_YEAR = "INTERVAL_YEAR"
+ """ The interval year type. """
+
+ INTERVAL_DAY = "INTERVAL_DAY"
+ """ The interval day type. """
+
+ STRING = "STRING"
+ """ The string type. """
+
+ VARCHAR = "VARCHAR"
+ """ The varchar type. """
+
+ FIXEDCHAR = "FIXEDCHAR"
+ """ The char type with fixed length. """
+
+ UUID = "UUID"
+ """ The UUID type. """
+
+ FIXED = "FIXED"
+ """ The binary type with fixed length. """
+
+ BINARY = "BINARY"
+ """ The binary type with variable length. The length is specified in the
type itself. """
+
+ STRUCT = "STRUCT"
+ """
+ The struct type.
+ A struct type is a complex type that contains a set of named fields, each
with a type,
+ and optionally a comment.
+ """
+
+ LIST = "LIST"
+ """
+ The list type.
+ A list type is a complex type that contains a set of elements, each with
the same type.
+ """
+
+ MAP = "MAP"
+ """
+ The map type.
+ A map type is a complex type that contains a set of key-value pairs, each
with a key type
+ and a value type.
+ """
+
+ UNION = "UNION"
+ """
+ The union type.
+ A union type is a complex type that contains a set of types.
+ """
+
+ NULL = "NULL"
+ """ The null type. A null type represents a value that is null. """
+
+ UNPARSED = "UNPARSED"
+ """ The unparsed type. An unparsed type represents an unresolvable type.
"""
+
+ EXTERNAL = "EXTERNAL"
+ """ The external type. An external type represents a type that is not
supported. """
+
+
+# Define the Type interface (abstract base class)
+class Type(ABC):
+ @abstractmethod
+ def name(self) -> Name:
+ """Returns the generic name of the type."""
+ pass
+
+ @abstractmethod
+ def simple_string(self) -> str:
+ """Returns a readable string representation of the type."""
+ pass
+
+
+# Define base classes
+class PrimitiveType(Type, ABC):
+ """Base class for all primitive types."""
+
+ pass
+
+
+class NumericType(PrimitiveType, ABC):
+ """Base class for all numeric types."""
+
+ pass
+
+
+class DateTimeType(PrimitiveType, ABC):
+ """Base class for all date/time types."""
+
+ pass
+
+
+class IntervalType(PrimitiveType, ABC):
+ """Base class for all interval types."""
+
+ pass
+
+
+class ComplexType(Type, ABC):
+ """Base class for all complex types, including struct, list, map, and
union."""
+
+ pass
+
+
+class IntegralType(NumericType, ABC):
+ """Base class for all integral types."""
+
+ _signed: bool
+
+ def __init__(self, signed: bool):
+ self._signed = signed
+
+ def signed(self) -> bool:
+ """Returns True if the integer type is signed, False otherwise."""
+ return self._signed
+
+
+class FractionType(NumericType, ABC):
+ """Base class for all fractional types."""
+
+ pass
diff --git a/clients/client-python/gravitino/api/types.py
b/clients/client-python/gravitino/api/types.py
new file mode 100644
index 000000000..b82ac2b68
--- /dev/null
+++ b/clients/client-python/gravitino/api/types.py
@@ -0,0 +1,1106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=C0302
+from __future__ import annotations
+from typing import List
+from .type import (
+ Type,
+ Name,
+ PrimitiveType,
+ IntegralType,
+ FractionType,
+ DateTimeType,
+ IntervalType,
+ ComplexType,
+)
+
+
+class Types:
+ """The helper class for Type. It contains all built-in types and provides
utility methods."""
+
+ class NullType(Type):
+ """The data type representing `NULL` values."""
+
+ _instance: "NullType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.NullType, cls).__new__(cls)
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "NullType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.NULL
+
+ def simple_string(self) -> str:
+ return "null"
+
+ class BooleanType(PrimitiveType):
+ """The boolean type in Gravitino."""
+
+ _instance: "BooleanType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.BooleanType, cls).__new__(cls)
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "BooleanType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.BOOLEAN
+
+ def simple_string(self) -> str:
+ return "boolean"
+
+ class ByteType(IntegralType):
+ """The byte type in Gravitino."""
+
+ _instance: "ByteType" = None
+ _unsigned_instance: "ByteType" = None
+
+ def __new__(cls, signed: bool = True):
+ if signed:
+ if cls._instance is None:
+ cls._instance = super(Types.ByteType, cls).__new__(cls)
+ cls._instance.__init__(signed)
+ return cls._instance
+ if cls._unsigned_instance is None:
+ cls._unsigned_instance = super(Types.ByteType,
cls).__new__(cls)
+ cls._unsigned_instance.__init__(signed)
+ return cls._unsigned_instance
+
+ @classmethod
+ def get(cls) -> "ByteType":
+ return cls(True)
+
+ @classmethod
+ def unsigned(cls) -> "ByteType":
+ return cls(False)
+
+ def name(self) -> Name:
+ return Name.BYTE
+
+ def simple_string(self) -> str:
+ return "byte" if self.signed() else "byte unsigned"
+
+ class ShortType(IntegralType):
+ _instance: "ShortType" = None
+ _unsigned_instance: "ShortType" = None
+
+ def __new__(cls, signed=True):
+ if signed:
+ if cls._instance is None:
+ cls._instance = super(Types.ShortType, cls).__new__(cls)
+ cls._instance.__init__(signed)
+ return cls._instance
+ if cls._unsigned_instance is None:
+ cls._unsigned_instance = super(Types.ShortType,
cls).__new__(cls)
+ cls._unsigned_instance.__init__(signed)
+ return cls._unsigned_instance
+
+ @classmethod
+ def get(cls) -> "ShortType":
+ return cls(True)
+
+ @classmethod
+ def unsigned(cls):
+ return cls(False)
+
+ def name(self) -> Name:
+ return Name.SHORT
+
+ def simple_string(self) -> str:
+ return "short" if self.signed() else "short unsigned"
+
+ class IntegerType(IntegralType):
+ _instance: "IntegerType" = None
+ _unsigned_instance: "IntegerType" = None
+
+ def __new__(cls, signed=True):
+ if signed:
+ if cls._instance is None:
+ cls._instance = super(Types.IntegerType, cls).__new__(cls)
+ cls._instance.__init__(signed)
+ return cls._instance
+ if cls._unsigned_instance is None:
+ cls._unsigned_instance = super(Types.IntegerType,
cls).__new__(cls)
+ cls._unsigned_instance.__init__(signed)
+ return cls._unsigned_instance
+
+ @classmethod
+ def get(cls) -> "IntegerType":
+ return cls(True)
+
+ @classmethod
+ def unsigned(cls):
+ return cls(False)
+
+ def name(self) -> Name:
+ return Name.INTEGER
+
+ def simple_string(self) -> str:
+ return "integer" if self.signed() else "integer unsigned"
+
+ class LongType(IntegralType):
+ _instance: "LongType" = None
+ _unsigned_instance: "LongType" = None
+
+ def __new__(cls, signed=True):
+ if signed:
+ if cls._instance is None:
+ cls._instance = super(Types.LongType, cls).__new__(cls)
+ cls._instance.__init__(signed)
+ return cls._instance
+ if cls._unsigned_instance is None:
+ cls._unsigned_instance = super(Types.LongType,
cls).__new__(cls)
+ cls._unsigned_instance.__init__(signed)
+ return cls._unsigned_instance
+
+ @classmethod
+ def get(cls) -> "LongType":
+ return cls(True)
+
+ @classmethod
+ def unsigned(cls):
+ return cls(False)
+
+ def name(self) -> Name:
+ return Name.LONG
+
+ def simple_string(self) -> str:
+ return "long" if self.signed() else "long unsigned"
+
+ class FloatType(FractionType):
+ _instance: "FloatType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.FloatType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "FloatType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.FLOAT
+
+ def simple_string(self) -> str:
+ return "float"
+
+ class DoubleType(FractionType):
+ _instance: "DoubleType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.DoubleType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "DoubleType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.DOUBLE
+
+ def simple_string(self) -> str:
+ return "double"
+
+ class DecimalType(FractionType):
+ """The decimal type in Gravitino."""
+
+ MAX_PRECISION = 38
+ _precision: int
+ _scale: int
+
+ def __init__(self, precision: int, scale: int):
+ """
+ Args:
+ precision: The precision of the decimal type.
+ scale: The scale of the decimal type.
+ """
+ super().__init__()
+ self.check_precision_scale(precision, scale)
+ self._precision = precision
+ self._scale = scale
+
+ @staticmethod
+ def check_precision_scale(precision: int, scale: int):
+ """
+ Ensures the precision and scale values are within valid range.
+
+ Args:
+ precision: The precision of the decimal.
+ scale: The scale of the decimal.
+ """
+ if not 1 <= precision <= Types.DecimalType.MAX_PRECISION:
+ raise ValueError(
+ f"Decimal precision must be in range [1, 38]: {precision}"
+ )
+ if not 0 <= scale <= precision:
+ raise ValueError(
+ f"Decimal scale must be in range [0, precision
({precision})]: {scale}"
+ )
+
+ @classmethod
+ def of(cls, precision: int, scale: int) -> "DecimalType":
+ return cls(precision, scale)
+
+ def name(self) -> Name:
+ return Name.DECIMAL
+
+ def precision(self) -> int:
+ return self._precision
+
+ def scale(self) -> int:
+ return self._scale
+
+ def simple_string(self) -> str:
+ return f"decimal({self._precision},{self._scale})"
+
+ def __eq__(self, other):
+ """
+ Compares two DecimalType objects for equality.
+
+ Args:
+ other: The other DecimalType to compare with.
+
+ Returns:
+ True if both objects have the same precision and scale, False
otherwise.
+ """
+ if not isinstance(other, Types.DecimalType):
+ return False
+ return self._precision == other._precision and self._scale ==
other._scale
+
+ def __hash__(self):
+ return hash((self._precision, self._scale))
+
+ class DateType(DateTimeType):
+ """The date time type in Gravitino."""
+
+ _instance: "DateType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.DateType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "DateType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.DATE
+
+ def simple_string(self) -> str:
+ return "date"
+
+ class TimeType(DateTimeType):
+ _instance: "TimeType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.TimeType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "TimeType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.TIME
+
+ def simple_string(self) -> str:
+ return "time"
+
+ class TimestampType(DateTimeType):
+ _instance_with_tz: "TimestampType" = None
+ _instance_without_tz: "TimestampType" = None
+ _with_time_zone: bool
+
+ def __new__(cls, with_time_zone: bool):
+ if with_time_zone:
+ if cls._instance_with_tz is None:
+ cls._instance_with_tz = super(Types.TimestampType,
cls).__new__(cls)
+ cls._instance_with_tz.__init__(with_time_zone)
+ return cls._instance_with_tz
+ if cls._instance_without_tz is None:
+ cls._instance_without_tz = super(Types.TimestampType,
cls).__new__(cls)
+ cls._instance_without_tz.__init__(with_time_zone)
+ return cls._instance_without_tz
+
+ @classmethod
+ def with_time_zone(cls) -> "TimestampType":
+ return cls(True)
+
+ @classmethod
+ def without_time_zone(cls) -> "TimestampType":
+ return cls(False)
+
+ def __init__(self, with_time_zone: bool):
+ self._with_time_zone = with_time_zone
+ super().__init__()
+
+ def has_time_zone(self) -> bool:
+ return self._with_time_zone
+
+ def name(self) -> Name:
+ return Name.TIMESTAMP
+
+ def simple_string(self) -> str:
+ return "timestamp_tz" if self._with_time_zone else "timestamp"
+
+ class IntervalYearType(IntervalType):
+ """The interval year type in Gravitino."""
+
+ _instance: "IntervalYearType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.IntervalYearType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "IntervalYearType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.INTERVAL_YEAR
+
+ def simple_string(self) -> str:
+ return "interval_year"
+
+ class IntervalDayType(IntervalType):
+ """The interval day type in Gravitino."""
+
+ _instance: "IntervalDayType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.IntervalDayType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "IntervalDayType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.INTERVAL_DAY
+
+ def simple_string(self) -> str:
+ return "interval_day"
+
+ class StringType(PrimitiveType):
+ """The string type in Gravitino, equivalent to varchar(MAX),
+ which the MAX is determined by the underlying catalog."""
+
+ _instance: "StringType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.StringType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "StringType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.STRING
+
+ def simple_string(self) -> str:
+ return "string"
+
+ class UUIDType(PrimitiveType):
+ """The uuid type in Gravitino."""
+
+ _instance: "UUIDType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.UUIDType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "UUIDType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.UUID
+
+ def simple_string(self) -> str:
+ return "uuid"
+
+ class FixedType(PrimitiveType):
+ """Fixed-length byte array type, if you want to use variable-length
+ byte array, use BinaryType instead."""
+
+ _length: int
+
+ def __init__(self, length: int):
+ """
+ Initializes the FixedType with the given length.
+
+ Args:
+ length: The length of the fixed type.
+ """
+ self._length = length
+
+ @classmethod
+ def of(cls, length: int) -> "FixedType":
+ """
+ Args:
+ length: The length of the fixed type.
+
+ Returns:
+ A FixedType instance with the given length.
+ """
+ return cls(length)
+
+ def name(self) -> Name:
+ return Name.FIXED
+
+ def length(self) -> int:
+ return self._length
+
+ def simple_string(self) -> str:
+ return f"fixed({self._length})"
+
+ def __eq__(self, other):
+ """
+ Compares two FixedType objects for equality.
+
+ Args:
+ other: The other FixedType object to compare with.
+
+ Returns:
+ True if both FixedType objects have the same length, False
otherwise.
+ """
+ if not isinstance(other, Types.FixedType):
+ return False
+ return self._length == other._length
+
+ def __hash__(self):
+ return hash(self._length)
+
+ class VarCharType(PrimitiveType):
+ """The varchar type in Gravitino."""
+
+ _length: int
+
+ def __init__(self, length: int):
+ self._length = length
+
+ @classmethod
+ def of(cls, length: int) -> "VarCharType":
+ return cls(length)
+
+ def name(self) -> Name:
+ return Name.VARCHAR
+
+ def length(self) -> int:
+ return self._length
+
+ def simple_string(self) -> str:
+ return f"varchar({self._length})"
+
+ def __eq__(self, other):
+ """
+ Compares two VarCharType objects for equality.
+
+ Args:
+ other: The other VarCharType object to compare with.
+
+ Returns:
+ True if both VarCharType objects have the same length, False
otherwise.
+ """
+ if isinstance(other, Types.VarCharType):
+ return self._length == other._length
+ return False
+
+ def __hash__(self):
+ return hash(self._length)
+
+ class FixedCharType(PrimitiveType):
+ """The fixed char type in Gravitino."""
+
+ _length: int
+
+ def __init__(self, length: int):
+ self._length = length
+
+ @classmethod
+ def of(cls, length: int) -> "FixedCharType":
+ return cls(length)
+
+ def name(self) -> Name:
+ return Name.FIXEDCHAR
+
+ def length(self) -> int:
+ return self._length
+
+ def simple_string(self) -> str:
+ return f"char({self._length})"
+
+ def __eq__(self, other):
+ if not isinstance(other, Types.FixedCharType):
+ return False
+ return self._length == other._length
+
+ def __hash__(self):
+ return hash(self._length)
+
+ class BinaryType(PrimitiveType):
+ _instance: "BinaryType" = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super(Types.BinaryType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+
+ @classmethod
+ def get(cls) -> "BinaryType":
+ return cls()
+
+ def name(self) -> Name:
+ return Name.BINARY
+
+ def simple_string(self) -> str:
+ return "binary"
+
+ class StructType(ComplexType):
+ """The struct type in Gravitino.
+ Note, this type is not supported in the current version of
Gravitino."""
+
+ _fields: List["Field"]
+
+ def __init__(self, fields: List["Field"]):
+ if not fields or len(fields) == 0:
+ raise ValueError("fields cannot be null or empty")
+ self._fields = fields
+
+ @classmethod
+ def of(cls, *fields) -> "StructType":
+ """
+ Args:
+ fields: The fields of the struct type.
+
+ Returns:
+ A StructType instance with the given fields.
+ """
+ return cls(fields)
+
+ def fields(self) -> List["Field"]:
+ return self._fields
+
+ def name(self) -> Name:
+ return Name.STRUCT
+
+ def simple_string(self) -> str:
+ return (
+ f"struct<{', '.join(field.simple_string() for field in
self._fields)}>"
+ )
+
+ def __eq__(self, other):
+ """
+ Compares two StructType objects for equality.
+
+ Args:
+ other: The other StructType object to compare with.
+
+ Returns:
+ True if both StructType objects have the same fields, False
otherwise.
+ """
+ if isinstance(other, Types.StructType):
+ return self._fields == other._fields
+ return False
+
+ def __hash__(self):
+ return hash(tuple(self._fields))
+
+ class Field:
+ _name: str
+ _type: Type
+ _nullable: bool
+ _comment: str
+
+ def __init__(
+ self, name: str, field_type: Type, nullable: bool, comment: str
+ ) -> None:
+ """
+ Initializes the Field with the given name, type, nullable
flag, and comment.
+
+ Args:
+ name: The name of the field.
+ field_type: The type of the field.
+ nullable: Whether the field is nullable.
+ comment: The comment of the field (optional).
+ """
+ if name is None:
+ raise ValueError("name cannot be null")
+ if type is None:
+ raise ValueError("type cannot be null")
+ self._name = name
+ self._type = field_type
+ self._nullable = nullable
+ self._comment = comment
+
+ @classmethod
+ def not_null_field(
+ cls, name: str, field_type: Type, comment: str = None
+ ) -> "Field":
+ """
+ Args:
+ name: The name of the field.
+ field_type: The type of the field.
+ comment: The comment of the field (optional).
+ """
+ return cls(name, field_type, False, comment)
+
+ @classmethod
+ def nullable_field(
+ cls, name: str, field_type: Type, comment: str = None
+ ) -> "Field":
+ """
+ Args:
+ name: The name of the field.
+ field_type: The type of the field.
+ comment: The comment of the field (optional).
+
+ Returns:
+ A nullable Field instance with the given name, field_type,
and comment.
+ """
+ return cls(name, field_type, True, comment)
+
+ def name(self):
+ return self._name
+
+ def type(self):
+ return self._type
+
+ def nullable(self):
+ return self._nullable
+
+ def comment(self):
+ return self._comment
+
+ def __eq__(self, other):
+ """
+ Compares two Field objects for equality.
+
+ Args:
+ other: The other Field object to compare with.
+
+ Returns:
+ True if both Field objects have the same attributes, False
otherwise.
+ """
+ if isinstance(other, Types.StructType.Field):
+ return (
+ self._name == other._name
+ and self._type == other._type
+ and self._nullable == other._nullable
+ and self._comment == other._comment
+ )
+ return False
+
+ def __hash__(self):
+ return hash((self._name, self._type, self._nullable))
+
+ def simple_string(self) -> str:
+ nullable_str = "NULL" if self._nullable else "NOT NULL"
+ comment_str = f" COMMENT '{self._comment}'" if self._comment
else ""
+ return f"{self._name}: {self._type.simple_string()}
{nullable_str}{comment_str}"
+
+ class ListType(ComplexType):
+ """A list type. Note, this type is not supported in the current
version of Gravitino."""
+
+ _element_type: Type
+ _element_nullable: bool
+
+ def __init__(self, element_type: Type, element_nullable: bool):
+ """
+ Create a new ListType with the given element type and the type is
nullable.
+
+ Args:
+ element_type: The element type of the list.
+ element_nullable: Whether the element of the list is nullable.
+ """
+ if element_type is None:
+ raise ValueError("element_type cannot be null")
+ self._element_type = element_type
+ self._element_nullable = element_nullable
+
+ @classmethod
+ def nullable(cls, element_type: Type) -> "ListType":
+ """
+ Create a new ListType with the given element type and the type is
nullable.
+
+ Args:
+ element_type: The element type of the list.
+
+ Returns:
+ A new ListType instance.
+ """
+ return cls.of(element_type, True)
+
+ @classmethod
+ def not_null(cls, element_type: Type) -> "ListType":
+ """
+ Create a new ListType with the given element type.
+
+ Args:
+ element_type: The element type of the list.
+
+ Returns:
+ A new ListType instance.
+ """
+ return cls.of(element_type, False)
+
+ @classmethod
+ def of(cls, element_type: Type, element_nullable: bool) -> "ListType":
+ """
+ Create a new ListType with the given element type and whether the
element is nullable.
+
+ Args:
+ element_type: The element type of the list.
+ element_nullable: Whether the element of the list is nullable.
+
+ Returns
+ A new ListType instance.
+ """
+ return cls(element_type, element_nullable)
+
+ def element_type(self) -> Type:
+ return self._element_type
+
+ def element_nullable(self) -> bool:
+ return self._element_nullable
+
+ def name(self) -> Name:
+ return Name.LIST
+
+ def simple_string(self) -> str:
+ return (
+ f"list<{self._element_type.simple_string()}>"
+ if self._element_nullable
+ else f"list<{self._element_type.simple_string()}, NOT NULL>"
+ )
+
+ def __eq__(self, other):
+ if not isinstance(other, Types.ListType):
+ return (
+ self._element_nullable == other.element_nullable()
+ and self._element_type == other.element_type()
+ )
+ return False
+
+ def __hash__(self):
+ return hash((self._element_type, self._element_nullable))
+
+ class MapType(ComplexType):
+ """The map type in Gravitino. Note, this type is not supported in the
current version of Gravitino."""
+
+ _key_type: Type
+ _value_type: Type
+ _value_nullable: bool
+
+ def __init__(self, key_type: Type, value_type: Type, value_nullable:
bool):
+ """
+ Create a new MapType with the given key type, value type and the
value is nullable.
+
+ Args:
+ key_type: The key type of the map.
+ value_type: The value type of the map.
+ value_nullable: Whether the value of the map is nullable.
+ """
+ self._key_type = key_type
+ self._value_type = value_type
+ self._value_nullable = value_nullable
+
+ @classmethod
+ def value_nullable(cls, key_type: Type, value_type: Type) -> "MapType":
+ """
+ Create a new MapType with the given key type, value type, and the
value is nullable.
+
+ Args:
+ key_type: The key type of the map.
+ value_type: The value type of the map.
+
+ Returns:
+ A new MapType instance.
+ """
+ return cls.of(key_type, value_type, True)
+
+ @classmethod
+ def value_not_null(cls, key_type: Type, value_type: Type) -> "MapType":
+ """
+ Create a new MapType with the given key type, value type, and the
value is not nullable.
+
+ Args:
+ key_type: The key type of the map.
+ value_type: The value type of the map.
+
+ Returns:
+ A new MapType instance.
+ """
+ return cls.of(key_type, value_type, False)
+
+ @classmethod
+ def of(
+ cls, key_type: Type, value_type: Type, value_nullable: bool
+ ) -> "MapType":
+ """
+ Create a new MapType with the given key type, value type, and
whether the value is nullable.
+
+ Args:
+ key_type: The key type of the map.
+ value_type: The value type of the map.
+ value_nullable: Whether the value of the map is nullable.
+
+ Returns:
+ A new MapType instance.
+ """
+ return cls(key_type, value_type, value_nullable)
+
+ def key_type(self) -> Type:
+ return self._key_type
+
+ def value_type(self) -> Type:
+ return self._value_type
+
+ def is_value_nullable(self) -> bool:
+ return self._value_nullable
+
+ def name(self) -> Name:
+ return Name.MAP
+
+ def simple_string(self) -> str:
+ return f"map<{self._key_type.simple_string()},
{self._value_type.simple_string()}>"
+
+ def __eq__(self, other):
+ """
+ Compares two MapType objects for equality.
+
+ Args:
+ other The other MapType object to compare with.
+
+ Returns:
+ True if both MapType objects have the same key type, value
type, and nullability, False otherwise.
+ """
+ if isinstance(other, Types.MapType):
+ return (
+ self._value_nullable == other._value_nullable
+ and self._key_type == other._key_type
+ and self._value_type == other._value_type
+ )
+ return False
+
+ def __hash__(self):
+ return hash((self._key_type, self._value_type,
self._value_nullable))
+
+ class UnionType(ComplexType):
+ """The union type in Gravitino. Note, this type is not supported in
the current version of Gravitino."""
+
+ _types: list[Type]
+
+ def __init__(self, types: list[Type]):
+ """
+ Create a new UnionType with the given types.
+
+ Args:
+ types The types of the union.
+ """
+ self._types = types
+
+ @classmethod
+ def of(cls, *types: Type) -> "UnionType":
+ """
+ Create a new UnionType with the given types.
+
+ Args:
+ types: The types of the union.
+
+ Returns:
+ A new UnionType instance.
+ """
+ return Types.UnionType(list(types))
+
+ def types(self) -> list:
+ return self._types
+
+ def name(self) -> Name:
+ return Name.UNION
+
+ def simple_string(self) -> str:
+ return f"union<{', '.join(t.simple_string() for t in
self._types)}>"
+
+ def __eq__(self, other):
+ """
+ Compares two UnionType objects for equality.
+
+ Args:
+ other The other UnionType object to compare with.
+
+ Returns:
+ True if both UnionType objects have the same types, False
otherwise.
+ """
+ if not isinstance(other, Types.UnionType):
+ return self._types == other.types()
+ return False
+
+ def __hash__(self):
+ return hash(tuple(self._types))
+
+ class UnparsedType(Type):
+ """Represents a type that is not parsed yet. The parsed type is
represented by other types of types."""
+
+ _unparsed_type: str
+
+ def __init__(self, unparsed_type: str):
+ """
+ Initializes an unparsed_type instance.
+
+ Args:
+ unparsed_type: The unparsed type as a string.
+ """
+ self._unparsed_type = unparsed_type
+
+ @classmethod
+ def of(cls, unparsed_type: str) -> "UnparsedType":
+ """
+ Creates a new unparsed_type with the given unparsed type.
+
+ Args:
+ unparsed_type The unparsed type.
+
+ Returns
+ A new unparsed_type instance.
+ """
+ return cls(unparsed_type)
+
+ def unparsed_type(self) -> str:
+ return self._unparsed_type
+
+ def name(self) -> Name:
+ return Name.UNPARSED
+
+ def simple_string(self) -> str:
+ return f"unparsed({self._unparsed_type})"
+
+ def __eq__(self, other):
+ """
+ Compares two unparsed_type objects for equality.
+
+ Args:
+ other: The other unparsed_type object to compare with.
+
+ Returns:
+ True if both unparsed_type objects have the same unparsed type
string, False otherwise.
+ """
+ if not isinstance(other, Types.UnparsedType):
+ return self._unparsed_type == other.unparsed_type()
+ return False
+
+ def __hash__(self):
+ return hash(self._unparsed_type)
+
+ def __str__(self):
+ return self._unparsed_type
+
+ class ExternalType(Type):
+ """Represents a type that is defined in an external catalog."""
+
+ _catalog_string: str
+
+ def __init__(self, catalog_string: str):
+ """
+ Initializes an ExternalType instance.
+
+ Args:
+ catalog_string The string representation of this type in the
catalog.
+ """
+ self._catalog_string = catalog_string
+
+ @classmethod
+ def of(cls, catalog_string: str) -> "ExternalType":
+ """
+ Creates a new ExternalType with the given catalog string.
+
+ Args:
+ catalog_string The string representation of this type in the
catalog.
+
+ Returns:
+ A new ExternalType instance.
+ """
+ return cls(catalog_string)
+
+ def catalog_string(self) -> str:
+ return self._catalog_string
+
+ def name(self) -> Name:
+ return Name.EXTERNAL
+
+ def simple_string(self) -> str:
+ return f"external({self._catalog_string})"
+
+ def __eq__(self, other):
+ """
+ Compares two ExternalType objects for equality.
+
+ Args:
+ other: The other ExternalType object to compare with.
+
+ Returns:
+ True if both ExternalType objects have the same catalog
string, False otherwise.
+ """
+ if not isinstance(other, Types.ExternalType):
+ return False
+ return self._catalog_string == other._catalog_string
+
+ def __hash__(self):
+ return hash(self._catalog_string)
+
+ def __str__(self):
+ return self.simple_string()
+
+ @staticmethod
+ def allow_auto_increment(data_type: Type) -> bool:
+ """
+ Checks if the given data type is allowed to be an auto-increment
column.
+
+ Args:
+ data_type The data type to check.
+
+ Returns:
+ True if the given data type is allowed to be an auto-increment
column, False otherwise.
+ """
+ return isinstance(data_type, (Types.IntegerType, Types.LongType))
diff --git a/clients/client-python/tests/unittests/test_gvfs_with_local.py
b/clients/client-python/tests/unittests/test_gvfs_with_local.py
index b4ce39e57..6e8e20502 100644
--- a/clients/client-python/tests/unittests/test_gvfs_with_local.py
+++ b/clients/client-python/tests/unittests/test_gvfs_with_local.py
@@ -49,6 +49,7 @@ from tests.unittests.auth.mock_base import (
)
+# pylint: disable=C0302
def generate_unique_random_string(length):
characters = string.ascii_letters + string.digits
random_string = "".join(random.sample(characters, length))
diff --git a/clients/client-python/tests/unittests/test_types.py
b/clients/client-python/tests/unittests/test_types.py
new file mode 100644
index 000000000..e241b420a
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_types.py
@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from gravitino.api.types import Types, Name
+
+
+class TestTypes(unittest.TestCase):
+
+ def test_null_type(self):
+ instance: Types.NullType = Types.NullType.get()
+ self.assertIsInstance(instance, Types.NullType)
+ self.assertEqual(instance.name(), Name.NULL)
+ self.assertEqual(instance.simple_string(), "null")
+ self.assertIs(instance, Types.NullType.get()) # Singleton check
+
+ def test_boolean_type(self):
+ instance: Types.BooleanType = Types.BooleanType.get()
+ self.assertIsInstance(instance, Types.BooleanType)
+ self.assertEqual(instance.name(), Name.BOOLEAN)
+ self.assertEqual(instance.simple_string(), "boolean")
+ self.assertIs(instance, Types.BooleanType.get()) # Singleton check
+
+ def test_byte_type(self):
+ signed_instance: Types.ByteType = Types.ByteType.get()
+ unsigned_instance = Types.ByteType.unsigned()
+ self.assertIsInstance(signed_instance, Types.ByteType)
+ self.assertEqual(signed_instance.name(), Name.BYTE)
+ self.assertEqual(signed_instance.simple_string(), "byte")
+ self.assertEqual(unsigned_instance.simple_string(), "byte unsigned")
+
+ def test_short_type(self):
+ signed_instance: Types.ShortType = Types.ShortType.get()
+ unsigned_instance = Types.ShortType.unsigned()
+ self.assertIsInstance(signed_instance, Types.ShortType)
+ self.assertEqual(signed_instance.simple_string(), "short")
+ self.assertEqual(unsigned_instance.simple_string(), "short unsigned")
+
+ def test_integer_type(self):
+ signed_instance: Types.IntegerType = Types.IntegerType.get()
+ unsigned_instance = Types.IntegerType.unsigned()
+ self.assertIsInstance(signed_instance, Types.IntegerType)
+ self.assertEqual(signed_instance.simple_string(), "integer")
+ self.assertEqual(unsigned_instance.simple_string(), "integer unsigned")
+
+ def test_long_type(self):
+ signed_instance: Types.LongType = Types.LongType.get()
+ unsigned_instance = Types.LongType.unsigned()
+ self.assertIsInstance(signed_instance, Types.LongType)
+ self.assertEqual(signed_instance.simple_string(), "long")
+ self.assertEqual(unsigned_instance.simple_string(), "long unsigned")
+
+ def test_float_type(self):
+ instance: Types.FloatType = Types.FloatType.get()
+ self.assertEqual(instance.name(), Name.FLOAT)
+ self.assertEqual(instance.simple_string(), "float")
+
+ def test_double_type(self):
+ instance: Types.DoubleType = Types.DoubleType.get()
+ self.assertEqual(instance.name(), Name.DOUBLE)
+ self.assertEqual(instance.simple_string(), "double")
+
+ def test_decimal_type(self):
+ instance: Types.DecimalType = Types.DecimalType.of(10, 2)
+ self.assertEqual(instance.name(), Name.DECIMAL)
+ self.assertEqual(instance.precision(), 10)
+ self.assertEqual(instance.scale(), 2)
+ self.assertEqual(instance.simple_string(), "decimal(10,2)")
+ with self.assertRaises(ValueError):
+ Types.DecimalType.of(39, 2) # Precision out of range
+ with self.assertRaises(ValueError):
+ Types.DecimalType.of(10, 11) # Scale out of range
+
+ def test_date_type(self):
+ instance: Types.DateType = Types.DateType.get()
+ self.assertEqual(instance.name(), Name.DATE)
+ self.assertEqual(instance.simple_string(), "date")
+
+ def test_time_type(self):
+ instance: Types.TimeType = Types.TimeType.get()
+ self.assertEqual(instance.name(), Name.TIME)
+ self.assertEqual(instance.simple_string(), "time")
+
+ def test_timestamp_type(self):
+ instance_with_tz = Types.TimestampType.with_time_zone()
+ instance_without_tz = Types.TimestampType.without_time_zone()
+ self.assertTrue(instance_with_tz.has_time_zone())
+ self.assertFalse(instance_without_tz.has_time_zone())
+ self.assertEqual(instance_with_tz.simple_string(), "timestamp_tz")
+ self.assertEqual(instance_without_tz.simple_string(), "timestamp")
+
+ def test_interval_types(self):
+ year_instance: Types.IntervalYearType = Types.IntervalYearType.get()
+ day_instance: Types.IntervalDayType = Types.IntervalDayType.get()
+ self.assertEqual(year_instance.name(), Name.INTERVAL_YEAR)
+ self.assertEqual(day_instance.name(), Name.INTERVAL_DAY)
+ self.assertEqual(year_instance.simple_string(), "interval_year")
+ self.assertEqual(day_instance.simple_string(), "interval_day")
+
+ def test_string_type(self):
+ instance: Types.StringType = Types.StringType.get()
+ self.assertEqual(instance.name(), Name.STRING)
+ self.assertEqual(instance.simple_string(), "string")
+
+ def test_uuid_type(self):
+ instance: Types.UUIDType = Types.UUIDType.get()
+ self.assertEqual(instance.name(), Name.UUID)
+ self.assertEqual(instance.simple_string(), "uuid")
+
+ def test_fixed_type(self):
+ instance: Types.FixedType = Types.FixedType.of(5)
+ self.assertEqual(instance.name(), Name.FIXED)
+ self.assertEqual(instance.length(), 5)
+ self.assertEqual(instance.simple_string(), "fixed(5)")
+
+ def test_varchar_type(self):
+ instance: Types.VarCharType = Types.VarCharType.of(10)
+ self.assertEqual(instance.name(), Name.VARCHAR)
+ self.assertEqual(instance.length(), 10)
+ self.assertEqual(instance.simple_string(), "varchar(10)")
+
+ def test_fixed_char_type(self):
+ instance: Types.FixedCharType = Types.FixedCharType.of(3)
+ self.assertEqual(instance.name(), Name.FIXEDCHAR)
+ self.assertEqual(instance.length(), 3)
+ self.assertEqual(instance.simple_string(), "char(3)")
+
+ def test_binary_type(self):
+ instance: Types.BinaryType = Types.BinaryType.get()
+ self.assertEqual(instance.name(), Name.BINARY)
+ self.assertEqual(instance.simple_string(), "binary")
+
+ def test_struct_type(self):
+ field1: Types.StructType.Field = Types.StructType.Field(
+ "name", Types.StringType.get(), True, "User's name"
+ )
+ field2: Types.StructType.Field = Types.StructType.Field(
+ "age", Types.IntegerType.get(), False, "User's age"
+ )
+ struct: Types.StructType = Types.StructType.of(field1, field2)
+ self.assertEqual(
+ struct.simple_string(),
+ "struct<name: string NULL COMMENT 'User's name', age: integer NOT
NULL COMMENT 'User's age'>",
+ )
+
+ def test_list_type(self):
+ instance: Types.ListType = Types.ListType.of(Types.StringType.get(),
True)
+ self.assertEqual(instance.name(), Name.LIST)
+ self.assertTrue(instance.element_nullable())
+ self.assertEqual(instance.simple_string(), "list<string>")
+
+ def test_map_type(self):
+ instance: Types.MapType = Types.MapType.of(
+ Types.StringType.get(), Types.IntegerType.get(), True
+ )
+ self.assertEqual(instance.name(), Name.MAP)
+ self.assertTrue(instance.is_value_nullable())
+ self.assertEqual(instance.simple_string(), "map<string, integer>")
+
+ def test_union_type(self):
+ instance: Types.UnionType = Types.UnionType.of(
+ Types.StringType.get(), Types.IntegerType.get()
+ )
+ self.assertEqual(instance.name(), Name.UNION)
+ self.assertEqual(instance.simple_string(), "union<string, integer>")
+
+ def test_unparsed_type(self):
+ instance: Types.UnparsedType = Types.UnparsedType.of("custom_type")
+ self.assertEqual(instance.name(), Name.UNPARSED)
+ self.assertEqual(instance.simple_string(), "unparsed(custom_type)")
+
+ def test_external_type(self):
+ instance: Types.ExternalType = Types.ExternalType.of("external_type")
+ self.assertEqual(instance.name(), Name.EXTERNAL)
+ self.assertEqual(instance.simple_string(), "external(external_type)")
+
+ def test_auto_increment_check(self):
+ self.assertTrue(Types.allow_auto_increment(Types.IntegerType.get()))
+ self.assertTrue(Types.allow_auto_increment(Types.LongType.get()))
+ self.assertFalse(Types.allow_auto_increment(Types.StringType.get()))
diff --git a/docs/manage-relational-metadata-using-gravitino.md
b/docs/manage-relational-metadata-using-gravitino.md
index ef35e182e..280793e69 100644
--- a/docs/manage-relational-metadata-using-gravitino.md
+++ b/docs/manage-relational-metadata-using-gravitino.md
@@ -873,7 +873,7 @@ In order to create a table, you need to provide the
following information:
The following types that Gravitino supports:
-| Type | Java
| JSON
| Description
|
+| Type | Java / Python
| JSON
| Description
|
|---------------------------|--------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Boolean | `Types.BooleanType.get()`
| `boolean`
| Boolean type
|
| Byte | `Types.ByteType.get()`
| `byte`
| Byte type, indicates a numerical value of 1 byte
|
