This is an automated email from the ASF dual-hosted git repository.
honahx pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new c69b8d28 Tests: Unify the test catalog setting (#609)
c69b8d28 is described below
commit c69b8d28c89e73e2bf42fe66ea4539b8428c6fd1
Author: frankliee <[email protected]>
AuthorDate: Tue Apr 16 23:37:36 2024 +0800
Tests: Unify the test catalog setting (#609)
---
tests/integration/test_partition_evolution.py | 86 +++++++++----------------
tests/integration/test_reads.py | 90 +++++++++------------------
2 files changed, 60 insertions(+), 116 deletions(-)
diff --git a/tests/integration/test_partition_evolution.py
b/tests/integration/test_partition_evolution.py
index 85ae3237..785b34b8 100644
--- a/tests/integration/test_partition_evolution.py
+++ b/tests/integration/test_partition_evolution.py
@@ -18,7 +18,7 @@
import pytest
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import NoSuchTableError
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema
@@ -41,34 +41,6 @@ from pyiceberg.types import (
)
[email protected]()
-def catalog_rest() -> Catalog:
- return load_catalog(
- "local",
- **{
- "type": "rest",
- "uri": "http://localhost:8181";,
- "s3.endpoint": "http://localhost:9000";,
- "s3.access-key-id": "admin",
- "s3.secret-access-key": "password",
- },
- )
-
-
[email protected]()
-def catalog_hive() -> Catalog:
- return load_catalog(
- "local",
- **{
- "type": "hive",
- "uri": "http://localhost:9083";,
- "s3.endpoint": "http://localhost:9000";,
- "s3.access-key-id": "admin",
- "s3.secret-access-key": "password",
- },
- )
-
-
def _simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
return _create_table_with_schema(catalog, table_schema_simple, "1")
@@ -101,7 +73,7 @@ def _create_table_with_schema(catalog: Catalog, schema:
Schema, format_version:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_identity_partition(catalog: Catalog, table_schema_simple: Schema)
-> None:
simple_table = _simple_table(catalog, table_schema_simple)
simple_table.update_spec().add_identity("foo").commit()
@@ -113,7 +85,7 @@ def test_add_identity_partition(catalog: Catalog,
table_schema_simple: Schema) -
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_year(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", YearTransform(),
"year_transform").commit()
@@ -121,7 +93,7 @@ def test_add_year(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_month(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", MonthTransform(),
"month_transform").commit()
@@ -129,7 +101,7 @@ def test_add_month(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_day(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", DayTransform(),
"day_transform").commit()
@@ -137,7 +109,7 @@ def test_add_day(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_hour(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", HourTransform(),
"hour_transform").commit()
@@ -145,7 +117,7 @@ def test_add_hour(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_bucket(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", BucketTransform(12),
"bucket_transform").commit()
@@ -153,7 +125,7 @@ def test_add_bucket(catalog: Catalog, table_schema_simple:
Schema) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_truncate(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", TruncateTransform(1),
"truncate_transform").commit()
@@ -163,7 +135,7 @@ def test_add_truncate(catalog: Catalog,
table_schema_simple: Schema) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_multiple_adds(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_identity("id").add_field("event_ts",
HourTransform(), "hourly_partitioned").add_field(
@@ -181,7 +153,7 @@ def test_multiple_adds(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_hour_to_day(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", DayTransform(),
"daily_partitioned").commit()
@@ -197,7 +169,7 @@ def test_add_hour_to_day(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_add_multiple_buckets(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("id", BucketTransform(16)).add_field("id",
BucketTransform(4)).commit()
@@ -212,7 +184,7 @@ def test_add_multiple_buckets(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_identity(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_identity("id").commit()
@@ -225,7 +197,7 @@ def test_remove_identity(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_identity_v2(catalog: Catalog) -> None:
table_v2 = _table_v2(catalog)
table_v2.update_spec().add_identity("id").commit()
@@ -236,7 +208,7 @@ def test_remove_identity_v2(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_bucket(catalog: Catalog) -> None:
table = _table(catalog)
with table.update_spec() as update:
@@ -257,7 +229,7 @@ def test_remove_bucket(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_bucket_v2(catalog: Catalog) -> None:
table_v2 = _table_v2(catalog)
with table_v2.update_spec() as update:
@@ -272,7 +244,7 @@ def test_remove_bucket_v2(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_day(catalog: Catalog) -> None:
table = _table(catalog)
with table.update_spec() as update:
@@ -293,7 +265,7 @@ def test_remove_day(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_remove_day_v2(catalog: Catalog) -> None:
table_v2 = _table_v2(catalog)
with table_v2.update_spec() as update:
@@ -308,7 +280,7 @@ def test_remove_day_v2(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_rename(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_identity("id").commit()
@@ -319,7 +291,7 @@ def test_rename(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_add_and_remove(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -328,7 +300,7 @@ def test_cannot_add_and_remove(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_add_redundant_time_partition(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -339,7 +311,7 @@ def test_cannot_add_redundant_time_partition(catalog:
Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_delete_and_rename(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -349,7 +321,7 @@ def test_cannot_delete_and_rename(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_rename_and_delete(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -359,7 +331,7 @@ def test_cannot_rename_and_delete(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_add_same_tranform_for_same_field(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -370,7 +342,7 @@ def test_cannot_add_same_tranform_for_same_field(catalog:
Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_add_same_field_multiple_times(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -381,7 +353,7 @@ def test_cannot_add_same_field_multiple_times(catalog:
Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_cannot_add_multiple_specs_same_name(catalog: Catalog) -> None:
table = _table(catalog)
with pytest.raises(ValueError) as exc_info:
@@ -392,7 +364,7 @@ def test_cannot_add_multiple_specs_same_name(catalog:
Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_change_specs_and_schema_transaction(catalog: Catalog) -> None:
table = _table(catalog)
with table.transaction() as transaction:
@@ -425,7 +397,7 @@ def test_change_specs_and_schema_transaction(catalog:
Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_multiple_adds_and_remove_v1(catalog: Catalog) -> None:
table = _table(catalog)
with table.update_spec() as update:
@@ -447,7 +419,7 @@ def test_multiple_adds_and_remove_v1(catalog: Catalog) ->
None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_multiple_adds_and_remove_v2(catalog: Catalog) -> None:
table_v2 = _table_v2(catalog)
with table_v2.update_spec() as update:
@@ -461,7 +433,7 @@ def test_multiple_adds_and_remove_v2(catalog: Catalog) ->
None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_multiple_remove_and_add_reuses_v2(catalog: Catalog) -> None:
table_v2 = _table_v2(catalog)
with table_v2.update_spec() as update:
diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py
index c670bc48..ee9b17e4 100644
--- a/tests/integration/test_reads.py
+++ b/tests/integration/test_reads.py
@@ -26,7 +26,7 @@ from hive_metastore.ttypes import LockRequest, LockResponse,
LockState, UnlockRe
from pyarrow.fs import S3FileSystem
from pydantic_core import ValidationError
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog
from pyiceberg.catalog.hive import HiveCatalog, _HiveClient
from pyiceberg.exceptions import CommitFailedException, NoSuchTableError
from pyiceberg.expressions import (
@@ -52,34 +52,6 @@ from pyiceberg.types import (
DEFAULT_PROPERTIES = {'write.parquet.compression-codec': 'zstd'}
[email protected]()
-def catalog_rest() -> Catalog:
- return load_catalog(
- "local",
- **{
- "type": "rest",
- "uri": "http://localhost:8181";,
- "s3.endpoint": "http://localhost:9000";,
- "s3.access-key-id": "admin",
- "s3.secret-access-key": "password",
- },
- )
-
-
[email protected]()
-def catalog_hive() -> Catalog:
- return load_catalog(
- "local",
- **{
- "type": "hive",
- "uri": "http://localhost:9083";,
- "s3.endpoint": "http://localhost:9000";,
- "s3.access-key-id": "admin",
- "s3.secret-access-key": "password",
- },
- )
-
-
TABLE_NAME = ("default", "t1")
@@ -100,7 +72,7 @@ def create_table(catalog: Catalog) -> Table:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_table_properties(catalog: Catalog) -> None:
table = create_table(catalog)
@@ -130,7 +102,7 @@ def test_table_properties(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_table_properties_dict(catalog: Catalog) -> None:
table = create_table(catalog)
@@ -160,7 +132,7 @@ def test_table_properties_dict(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_table_properties_error(catalog: Catalog) -> None:
table = create_table(catalog)
properties = {"abc": "def"}
@@ -170,7 +142,7 @@ def test_table_properties_error(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_nan(catalog: Catalog) -> None:
table_test_null_nan = catalog.load_table("default.test_null_nan")
arrow_table = table_test_null_nan.scan(row_filter=IsNaN("col_numeric"),
selected_fields=("idx", "col_numeric")).to_arrow()
@@ -180,7 +152,7 @@ def test_pyarrow_nan(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_nan_rewritten(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
arrow_table = table_test_null_nan_rewritten.scan(
@@ -192,7 +164,7 @@ def test_pyarrow_nan_rewritten(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
@pytest.mark.skip(reason="Fixing issues with NaN's:
https://github.com/apache/arrow/issues/34162";)
def test_pyarrow_not_nan_count(catalog: Catalog) -> None:
table_test_null_nan = catalog.load_table("default.test_null_nan")
@@ -201,7 +173,7 @@ def test_pyarrow_not_nan_count(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_duckdb_nan(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
con = table_test_null_nan_rewritten.scan().to_duckdb("table_test_null_nan")
@@ -211,7 +183,7 @@ def test_duckdb_nan(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_limit(catalog: Catalog) -> None:
table_test_limit = catalog.load_table("default.test_limit")
limited_result = table_test_limit.scan(selected_fields=("idx",),
limit=1).to_arrow()
@@ -226,7 +198,7 @@ def test_pyarrow_limit(catalog: Catalog) -> None:
@pytest.mark.integration
@pytest.mark.filterwarnings("ignore")
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_daft_nan(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
df = table_test_null_nan_rewritten.to_daft()
@@ -235,7 +207,7 @@ def test_daft_nan(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_daft_nan_rewritten(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
df = table_test_null_nan_rewritten.to_daft()
@@ -248,7 +220,7 @@ def test_daft_nan_rewritten(catalog: Catalog) -> None:
@pytest.mark.integration
@pytest.mark.filterwarnings("ignore")
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_ray_nan(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
ray_dataset = table_test_null_nan_rewritten.scan().to_ray()
@@ -257,7 +229,7 @@ def test_ray_nan(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_ray_nan_rewritten(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
ray_dataset = table_test_null_nan_rewritten.scan(
@@ -269,7 +241,7 @@ def test_ray_nan_rewritten(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
@pytest.mark.skip(reason="Fixing issues with NaN's:
https://github.com/apache/arrow/issues/34162";)
def test_ray_not_nan_count(catalog: Catalog) -> None:
table_test_null_nan_rewritten =
catalog.load_table("default.test_null_nan_rewritten")
@@ -278,7 +250,7 @@ def test_ray_not_nan_count(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_ray_all_types(catalog: Catalog) -> None:
table_test_all_types = catalog.load_table("default.test_all_types")
ray_dataset = table_test_all_types.scan().to_ray()
@@ -288,7 +260,7 @@ def test_ray_all_types(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_to_iceberg_all_types(catalog: Catalog) -> None:
table_test_all_types = catalog.load_table("default.test_all_types")
fs = S3FileSystem(
@@ -307,7 +279,7 @@ def test_pyarrow_to_iceberg_all_types(catalog: Catalog) ->
None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_deletes(catalog: Catalog) -> None:
# number, letter
# (1, 'a'),
@@ -344,7 +316,7 @@ def test_pyarrow_deletes(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_pyarrow_deletes_double(catalog: Catalog) -> None:
# number, letter
# (1, 'a'),
@@ -381,7 +353,7 @@ def test_pyarrow_deletes_double(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_partitioned_tables(catalog: Catalog) -> None:
for table_name, predicate in [
("test_partitioned_by_identity", "ts >= '2023-03-05T00:00:00+00:00'"),
@@ -398,7 +370,7 @@ def test_partitioned_tables(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_unpartitioned_uuid_table(catalog: Catalog) -> None:
unpartitioned_uuid =
catalog.load_table("default.test_uuid_and_fixed_unpartitioned")
arrow_table_eq = unpartitioned_uuid.scan(row_filter="uuid_col ==
'102cb62f-e6f8-4eb0-9973-d9b012ff0967'").to_arrow()
@@ -415,7 +387,7 @@ def test_unpartitioned_uuid_table(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_unpartitioned_fixed_table(catalog: Catalog) -> None:
fixed_table =
catalog.load_table("default.test_uuid_and_fixed_unpartitioned")
arrow_table_eq = fixed_table.scan(row_filter=EqualTo("fixed_col",
b"1234567890123456789012345")).to_arrow()
@@ -434,7 +406,7 @@ def test_unpartitioned_fixed_table(catalog: Catalog) ->
None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_scan_tag(catalog: Catalog) -> None:
test_positional_mor_deletes =
catalog.load_table("default.test_positional_mor_deletes")
arrow_table =
test_positional_mor_deletes.scan().use_ref("tag_12").to_arrow()
@@ -442,7 +414,7 @@ def test_scan_tag(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_scan_branch(catalog: Catalog) -> None:
test_positional_mor_deletes =
catalog.load_table("default.test_positional_mor_deletes")
arrow_table =
test_positional_mor_deletes.scan().use_ref("without_5").to_arrow()
@@ -450,7 +422,7 @@ def test_scan_branch(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_filter_on_new_column(catalog: Catalog) -> None:
test_table_add_column = catalog.load_table("default.test_table_add_column")
arrow_table = test_table_add_column.scan(row_filter="b == '2'").to_arrow()
@@ -464,7 +436,7 @@ def test_filter_on_new_column(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_upgrade_table_version(catalog: Catalog) -> None:
table_test_table_version = catalog.load_table("default.test_table_version")
@@ -492,7 +464,7 @@ def test_upgrade_table_version(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_sanitize_character(catalog: Catalog) -> None:
table_test_table_sanitized_character =
catalog.load_table("default.test_table_sanitized_character")
arrow_table = table_test_table_sanitized_character.scan().to_arrow()
@@ -502,7 +474,7 @@ def test_sanitize_character(catalog: Catalog) -> None:
@pytest.mark.integration
[email protected]('catalog', [pytest.lazy_fixture('catalog_hive'),
pytest.lazy_fixture('catalog_rest')])
[email protected]('catalog',
[pytest.lazy_fixture('session_catalog_hive'),
pytest.lazy_fixture('session_catalog')])
def test_null_list_and_map(catalog: Catalog) -> None:
table_test_empty_list_and_map =
catalog.load_table("default.test_table_empty_list_and_map")
arrow_table = table_test_empty_list_and_map.scan().to_arrow()
@@ -515,15 +487,15 @@ def test_null_list_and_map(catalog: Catalog) -> None:
@pytest.mark.integration
-def test_hive_locking(catalog_hive: HiveCatalog) -> None:
- table = create_table(catalog_hive)
+def test_hive_locking(session_catalog_hive: HiveCatalog) -> None:
+ table = create_table(session_catalog_hive)
database_name: str
table_name: str
_, database_name, table_name = table.identifier
- hive_client: _HiveClient = _HiveClient(catalog_hive.properties["uri"])
- blocking_lock_request: LockRequest =
catalog_hive._create_lock_request(database_name, table_name)
+ hive_client: _HiveClient =
_HiveClient(session_catalog_hive.properties["uri"])
+ blocking_lock_request: LockRequest =
session_catalog_hive._create_lock_request(database_name, table_name)
with hive_client as open_client:
# Force a lock on the test table
