This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 36a505f7 Improve the InMemory Catalog Implementation (#289)
36a505f7 is described below
commit 36a505f7741f814c00b8babf6f26e89efde5b688
Author: Kevin Liu <[email protected]>
AuthorDate: Wed Mar 13 01:20:02 2024 -0700
Improve the InMemory Catalog Implementation (#289)
* extract InMemoryCatalog out of test
* generalize InMemoryCatalog
* make write work
* write to temporary location
* can override table location
* memory.py -> in_memory.py
* fix test_commit_table
* rebase from main
* revert fs changes
* fix tests
* add docs and comments
* comma
* comment
* order
* fix test
* add license
* `create_table` write metadata file
* move InMemoryCatalog back to test_base
* remove unused references
* Update mkdocs/docs/configuration.md
Co-authored-by: Fokko Driesprong <[email protected]>
* Update mkdocs/docs/configuration.md
Co-authored-by: Fokko Driesprong <[email protected]>
* Update tests/catalog/test_base.py
Co-authored-by: Fokko Driesprong <[email protected]>
* remove schema_id
---------
Co-authored-by: Fokko Driesprong <[email protected]>
---
pyiceberg/cli/output.py | 2 +-
tests/catalog/test_base.py | 104 ++++++++++++++++++++++++++++++---------------
tests/cli/test_console.py | 68 +++++++++++++----------------
3 files changed, 101 insertions(+), 73 deletions(-)
diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py
index 67081fbd..56b544c9 100644
--- a/pyiceberg/cli/output.py
+++ b/pyiceberg/cli/output.py
@@ -158,7 +158,7 @@ class ConsoleOutput(Output):
Console().print(output_table)
def text(self, response: str) -> None:
- Console().print(response)
+ Console(soft_wrap=True).print(response)
def schema(self, schema: Schema) -> None:
output_table = self._table
diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py
index 1f006078..44c36a7d 100644
--- a/tests/catalog/test_base.py
+++ b/tests/catalog/test_base.py
@@ -16,6 +16,9 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+import uuid
+from pathlib import PosixPath
from typing import (
Dict,
List,
@@ -40,7 +43,7 @@ from pyiceberg.exceptions import (
NoSuchTableError,
TableAlreadyExistsError,
)
-from pyiceberg.io import load_file_io
+from pyiceberg.io import WAREHOUSE, load_file_io
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC,
PartitionField, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.table import (
@@ -53,15 +56,21 @@ from pyiceberg.table import (
TableIdentifier,
update_table_metadata,
)
-from pyiceberg.table.metadata import TableMetadataV1
+from pyiceberg.table.metadata import new_table_metadata
from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder
from pyiceberg.transforms import IdentityTransform
from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties
from pyiceberg.types import IntegerType, LongType, NestedField
+DEFAULT_WAREHOUSE_LOCATION = "file:///tmp/warehouse"
+
class InMemoryCatalog(Catalog):
- """An in-memory catalog implementation for testing purposes."""
+ """
+ An in-memory catalog implementation that uses in-memory data-structures to
store the namespaces and tables.
+
+ This is useful for test, demo, and playground but not in production as
data is not persisted.
+ """
__tables: Dict[Identifier, Table]
__namespaces: Dict[Identifier, Properties]
@@ -70,6 +79,7 @@ class InMemoryCatalog(Catalog):
super().__init__(name, **properties)
self.__tables = {}
self.__namespaces = {}
+ self._warehouse_location = properties.get(WAREHOUSE,
DEFAULT_WAREHOUSE_LOCATION)
def create_table(
self,
@@ -79,6 +89,7 @@ class InMemoryCatalog(Catalog):
partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC,
sort_order: SortOrder = UNSORTED_SORT_ORDER,
properties: Properties = EMPTY_DICT,
+ table_uuid: Optional[uuid.UUID] = None,
) -> Table:
schema: Schema = self._convert_schema_if_needed(schema) # type: ignore
@@ -91,24 +102,26 @@ class InMemoryCatalog(Catalog):
if namespace not in self.__namespaces:
self.__namespaces[namespace] = {}
- new_location = location or
f's3://warehouse/{"/".join(identifier)}/data'
- metadata = TableMetadataV1(**{
- "format-version": 1,
- "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c",
- "location": new_location,
- "last-updated-ms": 1602638573874,
- "last-column-id": schema.highest_field_id,
- "schema": schema.model_dump(),
- "partition-spec": partition_spec.model_dump()["fields"],
- "properties": properties,
- "current-snapshot-id": -1,
- "snapshots": [{"snapshot-id": 1925, "timestamp-ms":
1602638573822}],
- })
+ if not location:
+ location = f'{self._warehouse_location}/{"/".join(identifier)}'
+
+ metadata_location = self._get_metadata_location(location=location)
+ metadata = new_table_metadata(
+ schema=schema,
+ partition_spec=partition_spec,
+ sort_order=sort_order,
+ location=location,
+ properties=properties,
+ table_uuid=table_uuid,
+ )
+ io = load_file_io({**self.properties, **properties},
location=location)
+ self._write_metadata(metadata, io, metadata_location)
+
table = Table(
identifier=identifier,
metadata=metadata,
-
metadata_location=f's3://warehouse/{"/".join(identifier)}/metadata/metadata.json',
- io=load_file_io(),
+ metadata_location=metadata_location,
+ io=io,
catalog=self,
)
self.__tables[identifier] = table
@@ -118,14 +131,29 @@ class InMemoryCatalog(Catalog):
raise NotImplementedError
def _commit_table(self, table_request: CommitTableRequest) ->
CommitTableResponse:
- identifier = tuple(table_request.identifier.namespace.root) +
(table_request.identifier.name,)
- table = self.__tables[identifier]
- table.metadata = update_table_metadata(base_metadata=table.metadata,
updates=table_request.updates)
-
- return CommitTableResponse(
- metadata=table.metadata.model_dump(),
- metadata_location=table.location(),
+ identifier_tuple = self.identifier_to_tuple_without_catalog(
+ tuple(table_request.identifier.namespace.root +
[table_request.identifier.name])
)
+ current_table = self.load_table(identifier_tuple)
+ base_metadata = current_table.metadata
+
+ for requirement in table_request.requirements:
+ requirement.validate(base_metadata)
+
+ updated_metadata = update_table_metadata(base_metadata,
table_request.updates)
+ if updated_metadata == base_metadata:
+ # no changes, do nothing
+ return CommitTableResponse(metadata=base_metadata,
metadata_location=current_table.metadata_location)
+
+ # write new metadata
+ new_metadata_version =
self._parse_metadata_version(current_table.metadata_location) + 1
+ new_metadata_location =
self._get_metadata_location(current_table.metadata.location,
new_metadata_version)
+ self._write_metadata(updated_metadata, current_table.io,
new_metadata_location)
+
+ # update table state
+ current_table.metadata = updated_metadata
+
+ return CommitTableResponse(metadata=updated_metadata,
metadata_location=new_metadata_location)
def load_table(self, identifier: Union[str, Identifier]) -> Table:
identifier = self.identifier_to_tuple_without_catalog(identifier)
@@ -160,7 +188,7 @@ class InMemoryCatalog(Catalog):
identifier=to_identifier,
metadata=table.metadata,
metadata_location=table.metadata_location,
- io=load_file_io(),
+ io=self._load_file_io(properties=table.metadata.properties,
location=table.metadata_location),
catalog=self,
)
return self.__tables[to_identifier]
@@ -232,8 +260,8 @@ class InMemoryCatalog(Catalog):
@pytest.fixture
-def catalog() -> InMemoryCatalog:
- return InMemoryCatalog("test.in.memory.catalog", **{"test.key":
"test.value"})
+def catalog(tmp_path: PosixPath) -> InMemoryCatalog:
+ return InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE:
tmp_path.absolute().as_posix(), "test.key": "test.value"})
TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table")
@@ -244,7 +272,6 @@ TEST_TABLE_SCHEMA = Schema(
NestedField(2, "y", LongType(), doc="comment"),
NestedField(3, "z", LongType()),
)
-TEST_TABLE_LOCATION = "protocol://some/location"
TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x",
transform=IdentityTransform(), source_id=1, field_id=1000))
TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"}
NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization',
'department', 'my_table'\\)"
@@ -261,7 +288,6 @@ def given_catalog_has_a_table(
return catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=properties or TEST_TABLE_PROPERTIES,
)
@@ -307,13 +333,25 @@ def test_create_table(catalog: InMemoryCatalog) -> None:
table = catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table
+def test_create_table_location_override(catalog: InMemoryCatalog) -> None:
+ new_location = f"{catalog._warehouse_location}/new_location"
+ table = catalog.create_table(
+ identifier=TEST_TABLE_IDENTIFIER,
+ schema=TEST_TABLE_SCHEMA,
+ location=new_location,
+ partition_spec=TEST_TABLE_PARTITION_SPEC,
+ properties=TEST_TABLE_PROPERTIES,
+ )
+ assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table
+ assert table.location() == new_location
+
+
@pytest.mark.parametrize(
"schema,expected",
[
@@ -335,8 +373,6 @@ def test_create_table_pyarrow_schema(catalog:
InMemoryCatalog, pyarrow_schema_si
table = catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=pyarrow_schema_simple_without_ids,
- location=TEST_TABLE_LOCATION,
- partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table
@@ -662,7 +698,7 @@ def test_add_column_with_statement(catalog:
InMemoryCatalog) -> None:
def test_catalog_repr(catalog: InMemoryCatalog) -> None:
s = repr(catalog)
- assert s == "test.in.memory.catalog (<class 'test_base.InMemoryCatalog'>)"
+ assert s == "test.in_memory.catalog (<class 'test_base.InMemoryCatalog'>)"
def test_table_properties_int_value(catalog: InMemoryCatalog) -> None:
diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py
index d77b290e..3c208c0a 100644
--- a/tests/cli/test_console.py
+++ b/tests/cli/test_console.py
@@ -14,13 +14,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import datetime
import os
+import uuid
+from pathlib import PosixPath
+from unittest.mock import MagicMock
import pytest
from click.testing import CliRunner
from pytest_mock import MockFixture
from pyiceberg.cli.console import run
+from pyiceberg.io import WAREHOUSE
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.transforms import IdentityTransform
@@ -48,8 +53,10 @@ def env_vars(mocker: MockFixture) -> None:
@pytest.fixture(name="catalog")
-def fixture_catalog(mocker: MockFixture) -> InMemoryCatalog:
- in_memory_catalog = InMemoryCatalog("test.in.memory.catalog",
**{"test.key": "test.value"})
+def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) ->
InMemoryCatalog:
+ in_memory_catalog = InMemoryCatalog(
+ "test.in_memory.catalog", **{WAREHOUSE:
tmp_path.absolute().as_posix(), "test.key": "test.value"}
+ )
mocker.patch("pyiceberg.cli.console.load_catalog",
return_value=in_memory_catalog)
return in_memory_catalog
@@ -59,6 +66,13 @@ def fixture_namespace_properties() -> Properties:
return TEST_NAMESPACE_PROPERTIES.copy()
[email protected]()
+def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None:
+ datetime_mock = MagicMock(wraps=datetime.datetime)
+ datetime_mock.now.return_value =
datetime.datetime.fromtimestamp(TEST_TIMESTAMP / 1000.0).astimezone()
+ monkeypatch.setattr(datetime, "datetime", datetime_mock)
+
+
TEST_TABLE_IDENTIFIER = ("default", "my_table")
TEST_TABLE_NAMESPACE = "default"
TEST_NAMESPACE_PROPERTIES = {"location": "s3://warehouse/database/location"}
@@ -68,9 +82,10 @@ TEST_TABLE_SCHEMA = Schema(
NestedField(2, "y", LongType(), doc="comment"),
NestedField(3, "z", LongType()),
)
-TEST_TABLE_LOCATION = "s3://bucket/test/location"
TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x",
transform=IdentityTransform(), source_id=1, field_id=1000))
TEST_TABLE_PROPERTIES = {"read.split.target.size": "134217728"}
+TEST_TABLE_UUID = uuid.UUID("d20125c8-7284-442c-9aea-15fee620737c")
+TEST_TIMESTAMP = 1602638573874
MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI":
"test://doesnotexist"}
@@ -88,7 +103,6 @@ def test_list_namespace(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -120,12 +134,13 @@ def test_describe_namespace_does_not_exists(catalog:
InMemoryCatalog) -> None:
assert result.output == "Namespace does not exist: ('doesnotexist',)\n"
-def test_describe_table(catalog: InMemoryCatalog) -> None:
[email protected]()
+def test_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) ->
None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
+ table_uuid=TEST_TABLE_UUID,
)
runner = CliRunner()
@@ -134,7 +149,7 @@ def test_describe_table(catalog: InMemoryCatalog) -> None:
assert (
# Strip the whitespace on the end
"\n".join([line.rstrip() for line in result.output.split("\n")])
- == """Table format version 1
+ == """Table format version 2
Metadata location s3://warehouse/default/my_table/metadata/metadata.json
Table UUID d20125c8-7284-442c-9aea-15fee620737c
Last Updated 1602638573874
@@ -167,7 +182,6 @@ def test_schema(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -196,7 +210,6 @@ def test_spec(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -225,8 +238,8 @@ def test_uuid(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
+ table_uuid=TEST_TABLE_UUID,
)
runner = CliRunner()
@@ -248,14 +261,12 @@ def test_location(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
-
runner = CliRunner()
result = runner.invoke(run, ["location", "default.my_table"])
assert result.exit_code == 0
- assert result.output == """s3://bucket/test/location\n"""
+ assert result.output ==
f"""{catalog._warehouse_location}/default/my_table\n"""
def test_location_does_not_exists(catalog: InMemoryCatalog) -> None:
@@ -271,7 +282,6 @@ def test_drop_table(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -328,7 +338,6 @@ def test_rename_table(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -351,7 +360,6 @@ def test_properties_get_table(catalog: InMemoryCatalog) ->
None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -366,7 +374,6 @@ def test_properties_get_table_specific_property(catalog:
InMemoryCatalog) -> Non
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -381,7 +388,6 @@ def
test_properties_get_table_specific_property_that_doesnt_exist(catalog: InMem
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -450,7 +456,6 @@ def test_properties_set_table(catalog: InMemoryCatalog) ->
None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -491,7 +496,6 @@ def test_properties_remove_table(catalog: InMemoryCatalog)
-> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -506,7 +510,6 @@ def
test_properties_remove_table_property_does_not_exists(catalog: InMemoryCatal
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -538,7 +541,6 @@ def test_json_list_namespace(catalog: InMemoryCatalog) ->
None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -566,12 +568,13 @@ def test_json_describe_namespace_does_not_exists(catalog:
InMemoryCatalog) -> No
assert result.output == """{"type": "NoSuchNamespaceError", "message":
"Namespace does not exist: ('doesnotexist',)"}\n"""
-def test_json_describe_table(catalog: InMemoryCatalog) -> None:
[email protected]()
+def test_json_describe_table(catalog: InMemoryCatalog, mock_datetime_now:
None) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
+ table_uuid=TEST_TABLE_UUID,
)
runner = CliRunner()
@@ -579,7 +582,7 @@ def test_json_describe_table(catalog: InMemoryCatalog) ->
None:
assert result.exit_code == 0
assert (
result.output
- ==
"""{"identifier":["default","my_table"],"metadata_location":"s3://warehouse/default/my_table/metadata/metadata.json","metadata":{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"sche
[...]
+ ==
"""{"identifier":["default","my_table"],"metadata_location":"s3://warehouse/default/my_table/metadata/metadata.json","metadata":{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"sche
[...]
)
@@ -599,7 +602,6 @@ def test_json_schema(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -625,7 +627,6 @@ def test_json_spec(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -648,8 +649,8 @@ def test_json_uuid(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
+ table_uuid=TEST_TABLE_UUID,
)
runner = CliRunner()
@@ -671,14 +672,13 @@ def test_json_location(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
runner = CliRunner()
result = runner.invoke(run, ["--output=json", "location",
"default.my_table"])
assert result.exit_code == 0
- assert result.output == """"s3://bucket/test/location"\n"""
+ assert result.output ==
f'"{catalog._warehouse_location}/default/my_table"\n'
def test_json_location_does_not_exists(catalog: InMemoryCatalog) -> None:
@@ -694,7 +694,6 @@ def test_json_drop_table(catalog: InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -735,7 +734,6 @@ def test_json_rename_table(catalog: InMemoryCatalog) ->
None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
)
@@ -758,7 +756,6 @@ def test_json_properties_get_table(catalog:
InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -773,7 +770,6 @@ def
test_json_properties_get_table_specific_property(catalog: InMemoryCatalog) -
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -788,7 +784,6 @@ def
test_json_properties_get_table_specific_property_that_doesnt_exist(catalog:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -862,7 +857,6 @@ def test_json_properties_set_table(catalog:
InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -908,7 +902,6 @@ def test_json_properties_remove_table(catalog:
InMemoryCatalog) -> None:
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
@@ -923,7 +916,6 @@ def
test_json_properties_remove_table_property_does_not_exists(catalog: InMemory
catalog.create_table(
identifier=TEST_TABLE_IDENTIFIER,
schema=TEST_TABLE_SCHEMA,
- location=TEST_TABLE_LOCATION,
partition_spec=TEST_TABLE_PARTITION_SPEC,
properties=TEST_TABLE_PROPERTIES,
)
