This is an automated email from the ASF dual-hosted git repository.
sungwy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new eca98707 Deprecate Redundant Identifier Support in TableIdentifier,
and row_filter (#994)
eca98707 is described below
commit eca98707f30bdaf1ffd19f039058cc7e18c5f9cb
Author: Sung Yun <[email protected]>
AuthorDate: Thu Aug 8 10:56:13 2024 -0400
Deprecate Redundant Identifier Support in TableIdentifier, and row_filter
(#994)
---
pyiceberg/catalog/__init__.py | 30 +++++++++++--
pyiceberg/catalog/dynamodb.py | 8 ++--
pyiceberg/catalog/glue.py | 10 ++---
pyiceberg/catalog/hive.py | 14 +++---
pyiceberg/catalog/rest.py | 12 +++---
pyiceberg/catalog/sql.py | 16 +++----
pyiceberg/expressions/parser.py | 9 ++++
pyiceberg/table/__init__.py | 26 +++++++----
tests/catalog/test_base.py | 42 +++++++++---------
tests/catalog/test_rest.py | 12 +++---
tests/catalog/test_sql.py | 64 ++++++++++++++--------------
tests/integration/test_writes/test_writes.py | 14 ++++++
12 files changed, 156 insertions(+), 101 deletions(-)
diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py
index 6469c0d6..7b15a60c 100644
--- a/pyiceberg/catalog/__init__.py
+++ b/pyiceberg/catalog/__init__.py
@@ -67,7 +67,7 @@ from pyiceberg.typedef import (
RecursiveDict,
)
from pyiceberg.utils.config import Config, merge_config
-from pyiceberg.utils.deprecated import deprecation_message
+from pyiceberg.utils.deprecated import deprecated, deprecation_message
if TYPE_CHECKING:
import pyarrow as pa
@@ -613,6 +613,11 @@ class Catalog(ABC):
ValueError: If removals and updates have overlapping keys.
"""
+ @deprecated(
+ deprecated_in="0.8.0",
+ removed_in="0.9.0",
+ help_message="Support for parsing catalog level identifier in Catalog
identifiers is deprecated. Please refer to the table using only its namespace
and its table name.",
+ )
def identifier_to_tuple_without_catalog(self, identifier: Union[str,
Identifier]) -> Identifier:
"""Convert an identifier to a tuple and drop this catalog's name from
the first element.
@@ -627,6 +632,25 @@ class Catalog(ABC):
identifier_tuple = identifier_tuple[1:]
return identifier_tuple
+ def _identifier_to_tuple_without_catalog(self, identifier: Union[str,
Identifier]) -> Identifier:
+ """Convert an identifier to a tuple and drop this catalog's name from
the first element.
+
+ Args:
+ identifier (str | Identifier): Table identifier.
+
+ Returns:
+ Identifier: a tuple of strings with this catalog's name removed
+ """
+ identifier_tuple = Catalog.identifier_to_tuple(identifier)
+ if len(identifier_tuple) >= 3 and identifier_tuple[0] == self.name:
+ deprecation_message(
+ deprecated_in="0.8.0",
+ removed_in="0.9.0",
+ help_message="Support for parsing catalog level identifier in
Catalog identifiers is deprecated. Please refer to the table using only its
namespace and its table name.",
+ )
+ identifier_tuple = identifier_tuple[1:]
+ return identifier_tuple
+
@staticmethod
def identifier_to_tuple(identifier: Union[str, Identifier]) -> Identifier:
"""Parse an identifier to a tuple.
@@ -769,7 +793,7 @@ class MetastoreCatalog(Catalog, ABC):
return False
def purge_table(self, identifier: Union[str, Identifier]) -> None:
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
table = self.load_table(identifier_tuple)
self.drop_table(identifier_tuple)
io = load_file_io(self.properties, table.metadata_location)
@@ -823,7 +847,7 @@ class MetastoreCatalog(Catalog, ABC):
)
io = self._load_file_io(properties=properties,
location=metadata_location)
return StagedTable(
- identifier=(self.name, database_name, table_name),
+ identifier=(database_name, table_name),
metadata=metadata,
metadata_location=metadata_location,
io=io,
diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py
index 40d873cd..f0ef7c6b 100644
--- a/pyiceberg/catalog/dynamodb.py
+++ b/pyiceberg/catalog/dynamodb.py
@@ -246,7 +246,7 @@ class DynamoDbCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
dynamo_table_item =
self._get_iceberg_table_item(database_name=database_name, table_name=table_name)
return
self._convert_dynamo_table_item_to_iceberg_table(dynamo_table_item=dynamo_table_item)
@@ -260,7 +260,7 @@ class DynamoDbCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
try:
@@ -291,7 +291,7 @@ class DynamoDbCatalog(MetastoreCatalog):
NoSuchPropertyException: When from table miss some required
properties.
NoSuchNamespaceError: When the destination namespace doesn't exist.
"""
- from_identifier_tuple =
self.identifier_to_tuple_without_catalog(from_identifier)
+ from_identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
from_database_name, from_table_name =
self.identifier_to_database_and_table(from_identifier_tuple, NoSuchTableError)
to_database_name, to_table_name =
self.identifier_to_database_and_table(to_identifier)
@@ -638,7 +638,7 @@ class DynamoDbCatalog(MetastoreCatalog):
file = io.new_input(metadata_location)
metadata = FromInputFile.table_metadata(file)
return Table(
- identifier=(self.name, database_name, table_name),
+ identifier=(database_name, table_name),
metadata=metadata,
metadata_location=metadata_location,
io=self._load_file_io(metadata.properties, metadata_location),
diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py
index f9d84834..148b02f9 100644
--- a/pyiceberg/catalog/glue.py
+++ b/pyiceberg/catalog/glue.py
@@ -346,7 +346,7 @@ class GlueCatalog(MetastoreCatalog):
file = io.new_input(metadata_location)
metadata = FromInputFile.table_metadata(file)
return Table(
- identifier=(self.name, database_name, table_name),
+ identifier=(database_name, table_name),
metadata=metadata,
metadata_location=metadata_location,
io=self._load_file_io(metadata.properties, metadata_location),
@@ -462,7 +462,7 @@ class GlueCatalog(MetastoreCatalog):
NoSuchTableError: If a table with the given identifier does not
exist.
CommitFailedException: Requirement not met, or a conflict with a
concurrent commit.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(
+ identifier_tuple = self._identifier_to_tuple_without_catalog(
tuple(table_request.identifier.namespace.root +
[table_request.identifier.name])
)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple)
@@ -541,7 +541,7 @@ class GlueCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
return
self._convert_glue_to_iceberg(self._get_glue_table(database_name=database_name,
table_name=table_name))
@@ -555,7 +555,7 @@ class GlueCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
try:
self.glue.delete_table(DatabaseName=database_name, Name=table_name)
@@ -581,7 +581,7 @@ class GlueCatalog(MetastoreCatalog):
NoSuchPropertyException: When from table miss some required
properties.
NoSuchNamespaceError: When the destination namespace doesn't exist.
"""
- from_identifier_tuple =
self.identifier_to_tuple_without_catalog(from_identifier)
+ from_identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
from_database_name, from_table_name =
self.identifier_to_database_and_table(from_identifier_tuple, NoSuchTableError)
to_database_name, to_table_name =
self.identifier_to_database_and_table(to_identifier)
try:
diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py
index 2b9c2265..755cd34c 100644
--- a/pyiceberg/catalog/hive.py
+++ b/pyiceberg/catalog/hive.py
@@ -289,7 +289,7 @@ class HiveCatalog(MetastoreCatalog):
file = io.new_input(metadata_location)
metadata = FromInputFile.table_metadata(file)
return Table(
- identifier=(self.name, table.dbName, table.tableName),
+ identifier=(table.dbName, table.tableName),
metadata=metadata,
metadata_location=metadata_location,
io=self._load_file_io(metadata.properties, metadata_location),
@@ -297,7 +297,7 @@ class HiveCatalog(MetastoreCatalog):
)
def _convert_iceberg_into_hive(self, table: Table) -> HiveTable:
- identifier_tuple =
self.identifier_to_tuple_without_catalog(table.identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(table.identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
current_time_millis = int(time.time() * 1000)
@@ -431,7 +431,7 @@ class HiveCatalog(MetastoreCatalog):
NoSuchTableError: If a table with the given identifier does not
exist.
CommitFailedException: Requirement not met, or a conflict with a
concurrent commit.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(
+ identifier_tuple = self._identifier_to_tuple_without_catalog(
tuple(table_request.identifier.namespace.root +
[table_request.identifier.name])
)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
@@ -477,7 +477,7 @@ class HiveCatalog(MetastoreCatalog):
# Table does not exist, create it.
hive_table = self._convert_iceberg_into_hive(
StagedTable(
- identifier=(self.name, database_name, table_name),
+ identifier=(database_name, table_name),
metadata=updated_staged_table.metadata,
metadata_location=updated_staged_table.metadata_location,
io=updated_staged_table.io,
@@ -509,7 +509,7 @@ class HiveCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
with self._client as open_client:
@@ -526,7 +526,7 @@ class HiveCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist, or the
identifier is invalid.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
database_name, table_name =
self.identifier_to_database_and_table(identifier_tuple, NoSuchTableError)
try:
with self._client as open_client:
@@ -554,7 +554,7 @@ class HiveCatalog(MetastoreCatalog):
NoSuchTableError: When a table with the name does not exist.
NoSuchNamespaceError: When the destination namespace doesn't exist.
"""
- from_identifier_tuple =
self.identifier_to_tuple_without_catalog(from_identifier)
+ from_identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
from_database_name, from_table_name =
self.identifier_to_database_and_table(from_identifier_tuple, NoSuchTableError)
to_database_name, to_table_name =
self.identifier_to_database_and_table(to_identifier)
try:
diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py
index 639d732b..c22b614d 100644
--- a/pyiceberg/catalog/rest.py
+++ b/pyiceberg/catalog/rest.py
@@ -495,7 +495,7 @@ class RestCatalog(Catalog):
def _response_to_table(self, identifier_tuple: Tuple[str, ...],
table_response: TableResponse) -> Table:
return Table(
- identifier=(self.name,) + identifier_tuple if self.name else
identifier_tuple,
+ identifier=identifier_tuple,
metadata_location=table_response.metadata_location, # type: ignore
metadata=table_response.metadata,
io=self._load_file_io(
@@ -506,7 +506,7 @@ class RestCatalog(Catalog):
def _response_to_staged_table(self, identifier_tuple: Tuple[str, ...],
table_response: TableResponse) -> StagedTable:
return StagedTable(
- identifier=(self.name,) + identifier_tuple if self.name else
identifier_tuple,
+ identifier=identifier_tuple if self.name else identifier_tuple,
metadata_location=table_response.metadata_location, # type: ignore
metadata=table_response.metadata,
io=self._load_file_io(
@@ -664,7 +664,7 @@ class RestCatalog(Catalog):
@retry(**_RETRY_ARGS)
def load_table(self, identifier: Union[str, Identifier]) -> Table:
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
response = self._session.get(
self.url(Endpoints.load_table, prefixed=True,
**self._split_identifier_for_path(identifier_tuple))
)
@@ -678,7 +678,7 @@ class RestCatalog(Catalog):
@retry(**_RETRY_ARGS)
def drop_table(self, identifier: Union[str, Identifier], purge_requested:
bool = False) -> None:
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
response = self._session.delete(
self.url(
Endpoints.drop_table, prefixed=True, purge=purge_requested,
**self._split_identifier_for_path(identifier_tuple)
@@ -695,7 +695,7 @@ class RestCatalog(Catalog):
@retry(**_RETRY_ARGS)
def rename_table(self, from_identifier: Union[str, Identifier],
to_identifier: Union[str, Identifier]) -> Table:
- from_identifier_tuple =
self.identifier_to_tuple_without_catalog(from_identifier)
+ from_identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
payload = {
"source": self._split_identifier_for_json(from_identifier_tuple),
"destination": self._split_identifier_for_json(to_identifier),
@@ -830,7 +830,7 @@ class RestCatalog(Catalog):
Returns:
bool: True if the table exists, False otherwise.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
response = self._session.head(
self.url(Endpoints.load_table, prefixed=True,
**self._split_identifier_for_path(identifier_tuple))
)
diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py
index a8a36032..229124ad 100644
--- a/pyiceberg/catalog/sql.py
+++ b/pyiceberg/catalog/sql.py
@@ -155,7 +155,7 @@ class SqlCatalog(MetastoreCatalog):
file = io.new_input(metadata_location)
metadata = FromInputFile.table_metadata(file)
return Table(
- identifier=(self.name,) +
Catalog.identifier_to_tuple(table_namespace) + (table_name,),
+ identifier=Catalog.identifier_to_tuple(table_namespace) +
(table_name,),
metadata=metadata,
metadata_location=metadata_location,
io=self._load_file_io(metadata.properties, metadata_location),
@@ -192,7 +192,7 @@ class SqlCatalog(MetastoreCatalog):
"""
schema: Schema = self._convert_schema_if_needed(schema) # type: ignore
- identifier_nocatalog =
self.identifier_to_tuple_without_catalog(identifier)
+ identifier_nocatalog =
self._identifier_to_tuple_without_catalog(identifier)
namespace_identifier = Catalog.namespace_from(identifier_nocatalog)
table_name = Catalog.table_name_from(identifier_nocatalog)
if not self._namespace_exists(namespace_identifier):
@@ -238,7 +238,7 @@ class SqlCatalog(MetastoreCatalog):
TableAlreadyExistsError: If the table already exists
NoSuchNamespaceError: If namespace does not exist
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
namespace_tuple = Catalog.namespace_from(identifier_tuple)
namespace = Catalog.namespace_to_string(namespace_tuple)
table_name = Catalog.table_name_from(identifier_tuple)
@@ -277,7 +277,7 @@ class SqlCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
namespace_tuple = Catalog.namespace_from(identifier_tuple)
namespace = Catalog.namespace_to_string(namespace_tuple)
table_name = Catalog.table_name_from(identifier_tuple)
@@ -301,7 +301,7 @@ class SqlCatalog(MetastoreCatalog):
Raises:
NoSuchTableError: If a table with the name does not exist.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
namespace_tuple = Catalog.namespace_from(identifier_tuple)
namespace = Catalog.namespace_to_string(namespace_tuple)
table_name = Catalog.table_name_from(identifier_tuple)
@@ -348,8 +348,8 @@ class SqlCatalog(MetastoreCatalog):
TableAlreadyExistsError: If a table with the new name already
exist.
NoSuchNamespaceError: If the target namespace does not exist.
"""
- from_identifier_tuple =
self.identifier_to_tuple_without_catalog(from_identifier)
- to_identifier_tuple =
self.identifier_to_tuple_without_catalog(to_identifier)
+ from_identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
+ to_identifier_tuple =
self._identifier_to_tuple_without_catalog(to_identifier)
from_namespace_tuple = Catalog.namespace_from(from_identifier_tuple)
from_namespace = Catalog.namespace_to_string(from_namespace_tuple)
from_table_name = Catalog.table_name_from(from_identifier_tuple)
@@ -407,7 +407,7 @@ class SqlCatalog(MetastoreCatalog):
NoSuchTableError: If a table with the given identifier does not
exist.
CommitFailedException: Requirement not met, or a conflict with a
concurrent commit.
"""
- identifier_tuple = self.identifier_to_tuple_without_catalog(
+ identifier_tuple = self._identifier_to_tuple_without_catalog(
tuple(table_request.identifier.namespace.root +
[table_request.identifier.name])
)
namespace_tuple = Catalog.namespace_from(identifier_tuple)
diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py
index d99f9227..61aa1647 100644
--- a/pyiceberg/expressions/parser.py
+++ b/pyiceberg/expressions/parser.py
@@ -64,6 +64,7 @@ from pyiceberg.expressions.literals import (
)
from pyiceberg.typedef import L
from pyiceberg.types import strtobool
+from pyiceberg.utils.deprecated import deprecation_message
ParserElement.enablePackrat()
@@ -84,6 +85,14 @@ like_regex =
r"(?P<valid_wildcard>(?<!\\)%$)|(?P<invalid_wildcard>(?<!\\)%)"
@column.set_parse_action
def _(result: ParseResults) -> Reference:
+ if len(result.column) > 1:
+ deprecation_message(
+ deprecated_in="0.8.0",
+ removed_in="0.9.0",
+ help_message="Parsing expressions with table name is deprecated.
Only provide field names in the row_filter.",
+ )
+ # TODO: Once this is removed, we will no longer take just the last index
of parsed column result
+ # And introduce support for parsing filter expressions with nested fields.
return Reference(result.column[-1])
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index 626f3535..662c43d5 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -152,7 +152,7 @@ from pyiceberg.utils.bin_packing import ListPacker
from pyiceberg.utils.concurrent import ExecutorFactory
from pyiceberg.utils.config import Config
from pyiceberg.utils.datetime import datetime_to_millis
-from pyiceberg.utils.deprecated import deprecated
+from pyiceberg.utils.deprecated import deprecated, deprecation_message
from pyiceberg.utils.properties import property_as_bool, property_as_int
from pyiceberg.utils.singleton import _convert_to_hashable_type
@@ -1392,7 +1392,7 @@ class CommitTableResponse(IcebergBaseModel):
class Table:
- identifier: Identifier = Field()
+ _identifier: Identifier = Field()
metadata: TableMetadata
metadata_location: str = Field()
io: FileIO
@@ -1401,7 +1401,7 @@ class Table:
def __init__(
self, identifier: Identifier, metadata: TableMetadata,
metadata_location: str, io: FileIO, catalog: Catalog
) -> None:
- self.identifier = identifier
+ self._identifier = identifier
self.metadata = metadata
self.metadata_location = metadata_location
self.io = io
@@ -1422,12 +1422,22 @@ class Table:
def refresh(self) -> Table:
"""Refresh the current table metadata."""
- fresh = self.catalog.load_table(self.identifier[1:])
+ fresh = self.catalog.load_table(self._identifier)
self.metadata = fresh.metadata
self.io = fresh.io
self.metadata_location = fresh.metadata_location
return self
+ @property
+ def identifier(self) -> Identifier:
+ """Return the identifier of this table."""
+ deprecation_message(
+ deprecated_in="0.8.0",
+ removed_in="0.9.0",
+ help_message="Table.identifier property is deprecated. Please use
Table.name() function instead.",
+ )
+ return (self.catalog.name,) + self._identifier
+
def name(self) -> Identifier:
"""Return the identifier of this table."""
return self.identifier
@@ -1643,7 +1653,7 @@ class Table:
def _do_commit(self, updates: Tuple[TableUpdate, ...], requirements:
Tuple[TableRequirement, ...]) -> None:
response = self.catalog._commit_table( # pylint: disable=W0212
CommitTableRequest(
- identifier=TableIdentifier(namespace=self.identifier[:-1],
name=self.identifier[-1]),
+ identifier=TableIdentifier(namespace=self._identifier[:-1],
name=self._identifier[-1]),
updates=updates,
requirements=requirements,
)
@@ -1654,16 +1664,14 @@ class Table:
def __eq__(self, other: Any) -> bool:
"""Return the equality of two instances of the Table class."""
return (
- self.identifier == other.identifier
- and self.metadata == other.metadata
- and self.metadata_location == other.metadata_location
+ self.name() == other.name() and self.metadata == other.metadata
and self.metadata_location == other.metadata_location
if isinstance(other, Table)
else False
)
def __repr__(self) -> str:
"""Return the string representation of the Table class."""
- table_name = self.catalog.table_name_from(self.identifier)
+ table_name = self.catalog.table_name_from(self._identifier)
schema_str = ",\n ".join(str(column) for column in
self.schema().columns if self.schema())
partition_str = f"partition by: [{', '.join(field.name for field in
self.spec().fields if self.spec())}]"
sort_order_str = f"sort order: [{', '.join(str(field) for field in
self.sort_order().fields if self.sort_order())}]"
diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py
index 2f53a6c2..af5c67a9 100644
--- a/tests/catalog/test_base.py
+++ b/tests/catalog/test_base.py
@@ -129,7 +129,7 @@ class InMemoryCatalog(MetastoreCatalog):
raise NotImplementedError
def _commit_table(self, table_request: CommitTableRequest) ->
CommitTableResponse:
- identifier_tuple = self.identifier_to_tuple_without_catalog(
+ identifier_tuple = self._identifier_to_tuple_without_catalog(
tuple(table_request.identifier.namespace.root +
[table_request.identifier.name])
)
current_table = self.load_table(identifier_tuple)
@@ -154,28 +154,28 @@ class InMemoryCatalog(MetastoreCatalog):
return CommitTableResponse(metadata=updated_metadata,
metadata_location=new_metadata_location)
def load_table(self, identifier: Union[str, Identifier]) -> Table:
- identifier = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
try:
- return self.__tables[identifier]
+ return self.__tables[identifier_tuple]
except KeyError as error:
- raise NoSuchTableError(f"Table does not exist: {identifier}") from
error
+ raise NoSuchTableError(f"Table does not exist:
{identifier_tuple}") from error
def drop_table(self, identifier: Union[str, Identifier]) -> None:
- identifier = self.identifier_to_tuple_without_catalog(identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(identifier)
try:
- self.__tables.pop(identifier)
+ self.__tables.pop(identifier_tuple)
except KeyError as error:
- raise NoSuchTableError(f"Table does not exist: {identifier}") from
error
+ raise NoSuchTableError(f"Table does not exist:
{identifier_tuple}") from error
def purge_table(self, identifier: Union[str, Identifier]) -> None:
self.drop_table(identifier)
def rename_table(self, from_identifier: Union[str, Identifier],
to_identifier: Union[str, Identifier]) -> Table:
- from_identifier =
self.identifier_to_tuple_without_catalog(from_identifier)
+ identifier_tuple =
self._identifier_to_tuple_without_catalog(from_identifier)
try:
- table = self.__tables.pop(from_identifier)
+ table = self.__tables.pop(identifier_tuple)
except KeyError as error:
- raise NoSuchTableError(f"Table does not exist: {from_identifier}")
from error
+ raise NoSuchTableError(f"Table does not exist:
{identifier_tuple}") from error
to_identifier = Catalog.identifier_to_tuple(to_identifier)
to_namespace = Catalog.namespace_from(to_identifier)
@@ -438,7 +438,7 @@ def test_load_table_from_self_identifier(catalog:
InMemoryCatalog) -> None:
given_table = given_catalog_has_a_table(catalog)
# When
intermediate = catalog.load_table(TEST_TABLE_IDENTIFIER)
- table = catalog.load_table(intermediate.identifier)
+ table = catalog.load_table(intermediate._identifier)
# Then
assert table == given_table
@@ -473,10 +473,10 @@ def test_drop_table_from_self_identifier(catalog:
InMemoryCatalog) -> None:
# Given
table = given_catalog_has_a_table(catalog)
# When
- catalog.drop_table(table.identifier)
+ catalog.drop_table(table._identifier)
# Then
with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR):
- catalog.load_table(table.identifier)
+ catalog.load_table(table._identifier)
with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR):
catalog.load_table(TEST_TABLE_IDENTIFIER)
@@ -505,11 +505,11 @@ def test_rename_table(catalog: InMemoryCatalog) -> None:
table = catalog.rename_table(TEST_TABLE_IDENTIFIER, new_table)
# Then
- assert table.identifier == Catalog.identifier_to_tuple(new_table)
+ assert table._identifier == Catalog.identifier_to_tuple(new_table)
# And
table = catalog.load_table(new_table)
- assert table.identifier == Catalog.identifier_to_tuple(new_table)
+ assert table._identifier == Catalog.identifier_to_tuple(new_table)
# And
assert ("new", "namespace") in catalog.list_namespaces()
@@ -525,21 +525,21 @@ def test_rename_table_from_self_identifier(catalog:
InMemoryCatalog) -> None:
# When
new_table_name = "new.namespace.new_table"
- new_table = catalog.rename_table(table.identifier, new_table_name)
+ new_table = catalog.rename_table(table._identifier, new_table_name)
# Then
- assert new_table.identifier == Catalog.identifier_to_tuple(new_table_name)
+ assert new_table._identifier == Catalog.identifier_to_tuple(new_table_name)
# And
- new_table = catalog.load_table(new_table.identifier)
- assert new_table.identifier == Catalog.identifier_to_tuple(new_table_name)
+ new_table = catalog.load_table(new_table._identifier)
+ assert new_table._identifier == Catalog.identifier_to_tuple(new_table_name)
# And
assert ("new", "namespace") in catalog.list_namespaces()
# And
with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR):
- catalog.load_table(table.identifier)
+ catalog.load_table(table._identifier)
with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR):
catalog.load_table(TEST_TABLE_IDENTIFIER)
@@ -669,7 +669,7 @@ def test_commit_table(catalog: InMemoryCatalog) -> None:
# When
response = given_table.catalog._commit_table( # pylint: disable=W0212
CommitTableRequest(
-
identifier=TableIdentifier(namespace=Namespace(given_table.identifier[:-1]),
name=given_table.identifier[-1]),
+
identifier=TableIdentifier(namespace=Namespace(given_table._identifier[:-1]),
name=given_table._identifier[-1]),
updates=[
AddSchemaUpdate(schema=new_schema,
last_column_id=new_schema.highest_field_id),
SetCurrentSchemaUpdate(schema_id=-1),
diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py
index 174fff5d..54239ce3 100644
--- a/tests/catalog/test_rest.py
+++ b/tests/catalog/test_rest.py
@@ -648,7 +648,7 @@ def test_load_table_200(rest_mock: Mocker,
example_table_metadata_with_snapshot_
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
actual = catalog.load_table(("fokko", "table"))
expected = Table(
- identifier=("rest", "fokko", "table"),
+ identifier=("fokko", "table"),
metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
@@ -672,7 +672,7 @@ def test_load_table_from_self_identifier_200(
table = catalog.load_table(("pdames", "table"))
actual = catalog.load_table(table.identifier)
expected = Table(
- identifier=("rest", "pdames", "table"),
+ identifier=("pdames", "table"),
metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
@@ -771,7 +771,7 @@ def test_create_table_200(
properties={"owner": "fokko"},
)
expected = Table(
- identifier=("rest", "fokko", "fokko2"),
+ identifier=("fokko", "fokko2"),
metadata_location=example_table_metadata_no_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_no_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
@@ -934,7 +934,7 @@ def test_register_table_200(
identifier=("default", "registered_table"),
metadata_location="s3://warehouse/database/table/metadata.json"
)
expected = Table(
- identifier=("rest", "default", "registered_table"),
+ identifier=("default", "registered_table"),
metadata_location=example_table_metadata_no_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_no_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
@@ -1029,7 +1029,7 @@ def test_rename_table_200(rest_mock: Mocker,
example_table_metadata_with_snapsho
to_identifier = ("pdames", "destination")
actual = catalog.rename_table(from_identifier, to_identifier)
expected = Table(
- identifier=("rest", "pdames", "destination"),
+ identifier=("pdames", "destination"),
metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
@@ -1069,7 +1069,7 @@ def test_rename_table_from_self_identifier_200(
)
actual = catalog.rename_table(table.identifier, to_identifier)
expected = Table(
- identifier=("rest", "pdames", "destination"),
+ identifier=("pdames", "destination"),
metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"],
metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]),
io=load_file_io(),
diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py
index 9b05a1fc..e6c9a5b0 100644
--- a/tests/catalog/test_sql.py
+++ b/tests/catalog/test_sql.py
@@ -254,7 +254,7 @@ def test_create_tables_idempotency(catalog: SqlCatalog) ->
None:
],
)
def test_create_table_default_sort_order(catalog: SqlCatalog,
table_schema_nested: Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -279,7 +279,7 @@ def test_create_table_default_sort_order(catalog:
SqlCatalog, table_schema_neste
],
)
def test_create_v1_table(catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested,
properties={"format-version": "1"})
@@ -311,7 +311,7 @@ def test_create_table_with_pyarrow_schema(
iceberg_table_schema_simple: Schema,
table_identifier: Identifier,
) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier,
pyarrow_schema_simple_without_ids)
@@ -351,7 +351,7 @@ def test_write_pyarrow_schema(catalog: SqlCatalog,
table_identifier: Identifier)
pa.field("large", pa.large_string(), nullable=True),
]),
)
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, pyarrow_table.schema)
@@ -374,7 +374,7 @@ def test_write_pyarrow_schema(catalog: SqlCatalog,
table_identifier: Identifier)
],
)
def test_create_table_custom_sort_order(catalog: SqlCatalog,
table_schema_nested: Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
order = SortOrder(SortField(source_id=2, transform=IdentityTransform(),
null_order=NullOrder.NULLS_FIRST))
@@ -406,7 +406,7 @@ def test_create_table_custom_sort_order(catalog:
SqlCatalog, table_schema_nested
def test_create_table_with_default_warehouse_location(
warehouse: Path, catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier
) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
catalog.create_table(table_identifier, table_schema_nested)
@@ -435,7 +435,7 @@ def test_create_table_with_default_warehouse_location(
def test_create_table_with_given_location_removes_trailing_slash(
warehouse: Path, catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier
) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
table_name = Catalog.table_name_from(table_identifier_nocatalog)
location = f"file://{warehouse}/{catalog.name}.db/{table_name}-given"
@@ -465,7 +465,7 @@ def
test_create_table_with_given_location_removes_trailing_slash(
],
)
def test_create_duplicated_table(catalog: SqlCatalog, table_schema_nested:
Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
catalog.create_table(table_identifier, table_schema_nested)
@@ -491,7 +491,7 @@ def test_create_duplicated_table(catalog: SqlCatalog,
table_schema_nested: Schem
def test_create_table_if_not_exists_duplicated_table(
catalog: SqlCatalog, table_schema_nested: Schema, table_identifier:
Identifier
) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table1 = catalog.create_table(table_identifier, table_schema_nested)
@@ -540,7 +540,7 @@ def test_create_table_without_namespace(catalog:
SqlCatalog, table_schema_nested
],
)
def test_register_table(catalog: SqlCatalog, table_identifier: Identifier,
metadata_location: str) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.register_table(table_identifier, metadata_location)
@@ -566,7 +566,7 @@ def test_register_table(catalog: SqlCatalog,
table_identifier: Identifier, metad
],
)
def test_register_existing_table(catalog: SqlCatalog, table_identifier:
Identifier, metadata_location: str) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
catalog.register_table(table_identifier, metadata_location)
@@ -615,7 +615,7 @@ def test_register_table_without_namespace(catalog:
SqlCatalog, metadata_location
],
)
def test_load_table(catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -641,7 +641,7 @@ def test_load_table(catalog: SqlCatalog,
table_schema_nested: Schema, table_iden
],
)
def test_load_table_from_self_identifier(catalog: SqlCatalog,
table_schema_nested: Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -670,7 +670,7 @@ def test_load_table_from_self_identifier(catalog:
SqlCatalog, table_schema_neste
],
)
def test_drop_table(catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -697,7 +697,7 @@ def test_drop_table(catalog: SqlCatalog,
table_schema_nested: Schema, table_iden
],
)
def test_drop_table_from_self_identifier(catalog: SqlCatalog,
table_schema_nested: Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -757,8 +757,8 @@ def test_drop_table_that_does_not_exist(catalog:
SqlCatalog, table_identifier: I
def test_rename_table(
catalog: SqlCatalog, table_schema_nested: Schema, from_table_identifier:
Identifier, to_table_identifier: Identifier
) -> None:
- from_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(from_table_identifier)
- to_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(to_table_identifier)
+ from_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(from_table_identifier)
+ to_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(to_table_identifier)
from_namespace = Catalog.namespace_from(from_table_identifier_nocatalog)
to_namespace = Catalog.namespace_from(to_table_identifier_nocatalog)
catalog.create_namespace(from_namespace)
@@ -800,8 +800,8 @@ def test_rename_table(
def test_rename_table_from_self_identifier(
catalog: SqlCatalog, table_schema_nested: Schema, from_table_identifier:
Identifier, to_table_identifier: Identifier
) -> None:
- from_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(from_table_identifier)
- to_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(to_table_identifier)
+ from_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(from_table_identifier)
+ to_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(to_table_identifier)
from_namespace = Catalog.namespace_from(from_table_identifier_nocatalog)
to_namespace = Catalog.namespace_from(to_table_identifier_nocatalog)
catalog.create_namespace(from_namespace)
@@ -845,8 +845,8 @@ def test_rename_table_from_self_identifier(
def test_rename_table_to_existing_one(
catalog: SqlCatalog, table_schema_nested: Schema, from_table_identifier:
Identifier, to_table_identifier: Identifier
) -> None:
- from_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(from_table_identifier)
- to_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(to_table_identifier)
+ from_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(from_table_identifier)
+ to_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(to_table_identifier)
from_namespace = Catalog.namespace_from(from_table_identifier_nocatalog)
to_namespace = Catalog.namespace_from(to_table_identifier_nocatalog)
catalog.create_namespace(from_namespace)
@@ -884,7 +884,7 @@ def test_rename_table_to_existing_one(
],
)
def test_rename_missing_table(catalog: SqlCatalog, from_table_identifier:
Identifier, to_table_identifier: Identifier) -> None:
- to_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(to_table_identifier)
+ to_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(to_table_identifier)
to_namespace = Catalog.namespace_from(to_table_identifier_nocatalog)
catalog.create_namespace(to_namespace)
with pytest.raises(NoSuchTableError):
@@ -918,7 +918,7 @@ def test_rename_missing_table(catalog: SqlCatalog,
from_table_identifier: Identi
def test_rename_table_to_missing_namespace(
catalog: SqlCatalog, table_schema_nested: Schema, from_table_identifier:
Identifier, to_table_identifier: Identifier
) -> None:
- from_table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(from_table_identifier)
+ from_table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(from_table_identifier)
from_namespace = Catalog.namespace_from(from_table_identifier_nocatalog)
catalog.create_namespace(from_namespace)
table = catalog.create_table(from_table_identifier, table_schema_nested)
@@ -953,8 +953,8 @@ def test_rename_table_to_missing_namespace(
def test_list_tables(
catalog: SqlCatalog, table_schema_nested: Schema, table_identifier_1:
Identifier, table_identifier_2: Identifier
) -> None:
- table_identifier_1_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier_1)
- table_identifier_2_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier_2)
+ table_identifier_1_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier_1)
+ table_identifier_2_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier_2)
namespace_1 = Catalog.namespace_from(table_identifier_1_nocatalog)
namespace_2 = Catalog.namespace_from(table_identifier_2_nocatalog)
catalog.create_namespace(namespace_1)
@@ -1138,7 +1138,7 @@ def test_list_non_existing_namespaces(catalog:
SqlCatalog) -> None:
],
)
def test_drop_namespace(catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
assert namespace in catalog.list_namespaces()
@@ -1266,7 +1266,7 @@ def test_update_namespace_properties(catalog: SqlCatalog,
namespace: str) -> Non
],
)
def test_commit_table(catalog: SqlCatalog, table_schema_nested: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_nested)
@@ -1316,7 +1316,7 @@ def test_commit_table(catalog: SqlCatalog,
table_schema_nested: Schema, table_id
],
)
def test_append_table(catalog: SqlCatalog, table_schema_simple: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table = catalog.create_table(table_identifier, table_schema_simple)
@@ -1366,7 +1366,7 @@ def test_append_table(catalog: SqlCatalog,
table_schema_simple: Schema, table_id
],
)
def test_concurrent_commit_table(catalog: SqlCatalog, table_schema_simple:
Schema, table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
table_a = catalog.create_table(table_identifier, table_schema_simple)
@@ -1510,7 +1510,7 @@ def test_create_table_transaction(catalog: SqlCatalog,
format_version: int) -> N
)
def test_table_properties_int_value(catalog: SqlCatalog, table_schema_simple:
Schema, table_identifier: Identifier) -> None:
# table properties can be set to int, but still serialized to string
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
property_with_int = {"property_name": 42}
@@ -1537,7 +1537,7 @@ def test_table_properties_int_value(catalog: SqlCatalog,
table_schema_simple: Sc
def test_table_properties_raise_for_none_value(
catalog: SqlCatalog, table_schema_simple: Schema, table_identifier:
Identifier
) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
property_with_none = {"property_name": None}
@@ -1562,7 +1562,7 @@ def test_table_properties_raise_for_none_value(
],
)
def test_table_exists(catalog: SqlCatalog, table_schema_simple: Schema,
table_identifier: Identifier) -> None:
- table_identifier_nocatalog =
catalog.identifier_to_tuple_without_catalog(table_identifier)
+ table_identifier_nocatalog =
catalog._identifier_to_tuple_without_catalog(table_identifier)
namespace = Catalog.namespace_from(table_identifier_nocatalog)
catalog.create_namespace(namespace)
catalog.create_table(table_identifier, table_schema_simple,
properties={"format-version": "2"})
diff --git a/tests/integration/test_writes/test_writes.py
b/tests/integration/test_writes/test_writes.py
index 3aaafa85..9b4e2f9f 100644
--- a/tests/integration/test_writes/test_writes.py
+++ b/tests/integration/test_writes/test_writes.py
@@ -1295,3 +1295,17 @@ def
test_rest_catalog_with_empty_catalog_name_append_data(session_catalog: Catal
)
tbl = _create_table(test_catalog, identifier, data=[])
tbl.append(arrow_table_with_null)
+
+
[email protected]
+def test_table_v1_with_null_nested_namespace(session_catalog: Catalog,
arrow_table_with_null: pa.Table) -> None:
+ identifier = "default.lower.table_v1_with_null_nested_namespace"
+ tbl = _create_table(session_catalog, identifier, {"format-version": "1"},
[arrow_table_with_null])
+ assert tbl.format_version == 1, f"Expected v1, got: v{tbl.format_version}"
+ # TODO: Add session_catalog.table_exists check here when we integrate a
REST catalog image
+ # that supports HEAD request on table endpoint
+
+ # assert session_catalog.table_exists(identifier)
+
+ # We expect no error here
+ session_catalog.drop_table(identifier)
