This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 8865ddbd9 CI: Add Windows unit test job (#3723)
8865ddbd9 is described below
commit 8865ddbd95767561fcf2812c9c948ab1252c6deb
Author: Jared Yu <[email protected]>
AuthorDate: Tue Aug 4 07:52:24 2026 -0700
CI: Add Windows unit test job (#3723)
## Rationale
Adds `windows-latest` to the Python CI workflow, enabling automated
testing on Windows.
**Merge order**: #3721 must merge first (library fix for drive letter
parsing). This PR is branched on top of it.
**Supersedes**: #3722 (test fixes are now here instead of separate).
Depends on:
- #3721 (IO: Fix Windows drive letters misidentified as URI schemes)
- #3722 (Tests: Fix Windows compatibility in unit test suite)
## Changes
- Add `windows-unit-test` job (Python 3.12, `windows-latest`)
- Fix test platform assumptions (MAX_PATH, file locking, path formats,
SQLite URIs)
- Skip Kerberos tests on Windows (C lib unavailable)
- Skip Rich box-rendering tests on Windows (terminal-dependent
formatting)
## Expected result
Locally verified: 3801 passed, 1 skipped, 0 failures on Windows.
Depends on #3721. Closes #2477.
---
.github/workflows/python-ci.yml | 24 ++++++++++++
pyiceberg/io/__init__.py | 26 +++++++++++--
pyiceberg/io/fsspec.py | 11 ++++--
pyiceberg/io/pyarrow.py | 10 ++++-
tests/catalog/test_hive.py | 8 +++-
tests/catalog/test_sql.py | 20 +++++-----
tests/conftest.py | 12 +++---
tests/io/test_io.py | 38 ++++++++++++++----
tests/io/test_pyarrow.py | 86 ++++++++++++++++++++++++-----------------
tests/utils/test_manifest.py | 2 +-
10 files changed, 166 insertions(+), 71 deletions(-)
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index 7e4dce39d..5645d684c 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -79,6 +79,30 @@ jobs:
- name: Generate coverage report (85%) # Coverage threshold should only
increase over time — never decrease it!
run: COVERAGE_FAIL_UNDER=85 make coverage-report
+ windows-unit-test:
+ runs-on: windows-latest
+ timeout-minutes: 15
+ strategy:
+ fail-fast: true
+ matrix:
+ python: ['3.12']
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 #
v6.3.0
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Install UV
+ uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 #
v8.2.0
+ with:
+ enable-cache: true
+ - name: Install
+ run: uv sync --all-extras --no-extra hive-kerberos
+ - name: Run unit tests
+ run: uv run python -m pytest tests/ -m "(unmarked or parametrize) and
not integration" --ignore=tests/integration -v -x
+
cibw-dev-env-smoke-test:
runs-on: ubuntu-latest
steps:
diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
index 7dbc65121..6ff167474 100644
--- a/pyiceberg/io/__init__.py
+++ b/pyiceberg/io/__init__.py
@@ -27,6 +27,7 @@ from __future__ import annotations
import importlib
import logging
+import os
import warnings
from abc import ABC, abstractmethod
from io import SEEK_SET
@@ -41,6 +42,18 @@ from pyiceberg.typedef import EMPTY_DICT, Properties
logger = logging.getLogger(__name__)
+
+def _is_local_path(path: str) -> bool:
+ r"""Check if a path is a local filesystem path rather than a URI.
+
+ Uses os.path.splitdrive to detect Windows drive letters (e.g. 'C:\\...')
+ in a platform-aware way. On non-Windows systems, splitdrive always returns
+ an empty drive component, so this only triggers on Windows.
+ """
+ drive, _ = os.path.splitdrive(path)
+ return drive != ""
+
+
AWS_PROFILE_NAME = "client.profile-name"
AWS_REGION = "client.region"
AWS_ACCESS_KEY_ID = "client.access-key-id"
@@ -335,14 +348,19 @@ PY_IO_IMPL = "py-io-impl"
def _infer_file_io_from_scheme(path: str, properties: Properties) -> FileIO |
None:
- parsed_url = urlparse(path)
- if parsed_url.scheme:
- if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme):
+ if _is_local_path(path):
+ scheme = "file"
+ else:
+ parsed_url = urlparse(path)
+ scheme = parsed_url.scheme
+
+ if scheme:
+ if file_ios := SCHEMA_TO_FILE_IO.get(scheme):
for file_io_path in file_ios:
if file_io := _import_file_io(file_io_path, properties):
return file_io
else:
- warnings.warn(f"No preferred file implementation for scheme:
{parsed_url.scheme}", stacklevel=2)
+ warnings.warn(f"No preferred file implementation for scheme:
{scheme}", stacklevel=2)
return None
diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
index 7749268ff..b28ffd069 100644
--- a/pyiceberg/io/fsspec.py
+++ b/pyiceberg/io/fsspec.py
@@ -89,6 +89,7 @@ from pyiceberg.io import (
InputStream,
OutputFile,
OutputStream,
+ _is_local_path,
)
from pyiceberg.typedef import Properties
from pyiceberg.types import strtobool
@@ -443,7 +444,7 @@ class FsspecFileIO(FileIO):
FsspecInputFile: An FsspecInputFile instance for the given
location.
"""
uri = urlparse(location)
- fs = self._get_fs_from_uri(uri)
+ fs = self._get_fs_from_uri(uri, location)
return FsspecInputFile(location=location, fs=fs)
@override
@@ -457,7 +458,7 @@ class FsspecFileIO(FileIO):
FsspecOutputFile: An FsspecOutputFile instance for the given
location.
"""
uri = urlparse(location)
- fs = self._get_fs_from_uri(uri)
+ fs = self._get_fs_from_uri(uri, location)
return FsspecOutputFile(location=location, fs=fs)
@override
@@ -475,11 +476,13 @@ class FsspecFileIO(FileIO):
str_location = location
uri = urlparse(str_location)
- fs = self._get_fs_from_uri(uri)
+ fs = self._get_fs_from_uri(uri, str_location)
fs.rm(str_location)
- def _get_fs_from_uri(self, uri: "ParseResult") -> AbstractFileSystem:
+ def _get_fs_from_uri(self, uri: "ParseResult", location: str = "") ->
AbstractFileSystem:
"""Get a filesystem from a parsed URI, using hostname for ADLS account
resolution."""
+ if _is_local_path(location):
+ return self.get_fs("file")
if uri.scheme in _ADLS_SCHEMES:
return self.get_fs(uri.scheme, uri.hostname)
return self.get_fs(uri.scheme)
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index 29ede3ce9..c36f1639d 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -121,6 +121,7 @@ from pyiceberg.io import (
InputStream,
OutputFile,
OutputStream,
+ _is_local_path,
)
from pyiceberg.io.fileformat import DataFileStatistics as DataFileStatistics
from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel,
FileFormatWriter
@@ -401,10 +402,17 @@ class PyArrowFileIO(FileIO):
@staticmethod
def parse_location(location: str, properties: Properties = EMPTY_DICT) ->
tuple[str, str, str]:
- """Return (scheme, netloc, path) for the given location.
+ r"""Return (scheme, netloc, path) for the given location.
Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing.
+ On Windows, paths with drive letters (e.g. 'C:\\...') are treated as
+ local file paths rather than URIs.
"""
+ if _is_local_path(location):
+ default_scheme = properties.get("DEFAULT_SCHEME", "file")
+ default_netloc = properties.get("DEFAULT_NETLOC", "")
+ return default_scheme, default_netloc, os.path.abspath(location)
+
uri = urlparse(location)
if not uri.scheme:
diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py
index 09bb5ab92..f594aa876 100644
--- a/tests/catalog/test_hive.py
+++ b/tests/catalog/test_hive.py
@@ -18,6 +18,7 @@
import base64
import copy
import struct
+import sys
import threading
import uuid
from collections.abc import Generator
@@ -300,7 +301,7 @@ def test_create_table(
# it to construct the assert_called_with
metadata_location: str = called_hive_table.parameters["metadata_location"]
assert metadata_location.endswith(".metadata.json")
- assert "/database/table/metadata/" in metadata_location
+ assert "/database/table/metadata/" in metadata_location.replace("\\", "/")
catalog._client.__enter__().create_table.assert_called_with(
HiveTable(
tableName="table",
@@ -481,7 +482,7 @@ def
test_create_table_with_given_location_removes_trailing_slash(
# it to construct the assert_called_with
metadata_location: str = called_hive_table.parameters["metadata_location"]
assert metadata_location.endswith(".metadata.json")
- assert "/database/table-given-location/metadata/" in metadata_location
+ assert "/database/table-given-location/metadata/" in
metadata_location.replace("\\", "/")
catalog._client.__enter__().create_table.assert_called_with(
HiveTable(
tableName="table",
@@ -1369,6 +1370,7 @@ def test_create_hive_client_failure() -> None:
assert mock_hive_client.call_count == 2
[email protected](sys.platform == "win32", reason="Kerberos/puresasl not
available on Windows")
def test_create_hive_client_with_kerberos(
kerberized_hive_metastore_fake_url: str,
) -> None:
@@ -1381,6 +1383,7 @@ def test_create_hive_client_with_kerberos(
assert client is not None
[email protected](sys.platform == "win32", reason="Kerberos/puresasl not
available on Windows")
def test_create_hive_client_with_kerberos_using_context_manager(
kerberized_hive_metastore_fake_url: str,
) -> None:
@@ -1412,6 +1415,7 @@ def
test_create_hive_client_with_kerberos_using_context_manager(
assert open_client._iprot.trans.isOpen()
[email protected](sys.platform == "win32", reason="Kerberos/puresasl not
available on Windows")
def test_kerberized_client_uses_fresh_transport_on_reuse(
kerberized_hive_metastore_fake_url: str,
) -> None:
diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py
index 1eabc56a9..4ba63d783 100644
--- a/tests/catalog/test_sql.py
+++ b/tests/catalog/test_sql.py
@@ -61,7 +61,7 @@ def catalog_memory(catalog_name: str, warehouse: Path) ->
Generator[SqlCatalog,
@pytest.fixture(scope="module")
def catalog_sqlite(catalog_name: str, warehouse: Path) ->
Generator[SqlCatalog, None, None]:
props = {
- "uri": f"sqlite:////{warehouse}/sql-catalog",
+ "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog",
"warehouse": f"file://{warehouse}",
}
catalog = SqlCatalog(catalog_name, **props)
@@ -72,7 +72,7 @@ def catalog_sqlite(catalog_name: str, warehouse: Path) ->
Generator[SqlCatalog,
@pytest.fixture(scope="module")
def catalog_uri(warehouse: Path) -> str:
- return f"sqlite:////{warehouse}/sql-catalog"
+ return f"sqlite:///{warehouse.as_posix()}/sql-catalog"
@pytest.fixture(scope="module")
@@ -96,7 +96,7 @@ def test_creation_with_echo_parameter(catalog_name: str,
warehouse: Path) -> Non
for echo_param, expected_echo_value in test_cases:
props = {
- "uri": f"sqlite:////{warehouse}/sql-catalog",
+ "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog",
"warehouse": f"file://{warehouse}",
}
# None is for default value
@@ -119,7 +119,7 @@ def
test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Pat
for pool_pre_ping_param, expected_pool_pre_ping_value in test_cases:
props = {
- "uri": f"sqlite:////{warehouse}/sql-catalog",
+ "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog",
"warehouse": f"file://{warehouse}",
}
# None is for default value
@@ -139,7 +139,7 @@ def test_creation_from_impl(catalog_name: str, warehouse:
Path) -> None:
catalog_name,
**{
"py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog",
- "uri": f"sqlite:////{warehouse}/sql-catalog",
+ "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog",
"warehouse": f"file://{warehouse}",
},
),
@@ -268,7 +268,7 @@ def get_columns(engine: Engine) -> set[str]:
def test_adds_iceberg_type_column_to_old_schema(warehouse: Path) -> None:
- uri = f"sqlite:////{warehouse}/test-migration-add-col"
+ uri = f"sqlite:///{warehouse.as_posix()}/test-migration-add-col"
engine = _create_v0_db(uri)
# Verify the column does not exist in the old schema
@@ -334,7 +334,7 @@ def test_list_tables_filters_by_iceberg_type(warehouse:
Path) -> None:
def test_migration_to_v1_with_property_set(warehouse: Path) -> None:
- uri = f"sqlite:////{warehouse}/test-v1-migrate"
+ uri = f"sqlite:///{warehouse.as_posix()}/test-v1-migrate"
engine = _create_v0_db(uri)
assert "iceberg_type" not in get_columns(engine)
@@ -361,7 +361,7 @@ def test_invalid_schema_version_raises(warehouse: Path) ->
None:
def test_list_tables_works_on_v0_schema(warehouse: Path) -> None:
"""list_tables should work on V0 schemas without iceberg_type column."""
- uri = f"sqlite:////{warehouse}/test-v0-list"
+ uri = f"sqlite:///{warehouse.as_posix()}/test-v0-list"
_create_v0_db(uri)
catalog = SqlCatalog(
@@ -441,7 +441,7 @@ def _make_v0_catalog(uri: str, warehouse: Path) ->
SqlCatalog:
def test_namespace_exists_on_v0_schema(warehouse: Path) -> None:
"""namespace_exists should not fail on V0 schema (no iceberg_type
column)."""
- catalog = _make_v0_catalog(f"sqlite:////{warehouse}/test-v0-ns-exists",
warehouse)
+ catalog =
_make_v0_catalog(f"sqlite:///{warehouse.as_posix()}/test-v0-ns-exists",
warehouse)
catalog.create_namespace("ns")
assert catalog.namespace_exists("ns")
assert not catalog.namespace_exists("missing")
@@ -449,7 +449,7 @@ def test_namespace_exists_on_v0_schema(warehouse: Path) ->
None:
def test_create_and_load_table_on_v0_schema(warehouse: Path) -> None:
"""create_table and load_table should work on V0 schema without
iceberg_type column."""
- catalog = _make_v0_catalog(f"sqlite:////{warehouse}/test-v0-create",
warehouse)
+ catalog =
_make_v0_catalog(f"sqlite:///{warehouse.as_posix()}/test-v0-create", warehouse)
catalog.create_namespace("ns")
schema = Schema(NestedField(1, "id", StringType(), required=True))
tbl = catalog.create_table(("ns", "tbl"), schema)
diff --git a/tests/conftest.py b/tests/conftest.py
index 2b1f2366e..51a663434 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2388,7 +2388,7 @@ def empty_home_dir_path(tmp_path_factory:
pytest.TempPathFactory) -> str:
return home_path
-RANDOM_LENGTH = 20
+RANDOM_LENGTH = 8 # Keep short to stay within Windows MAX_PATH (260 chars)
NUM_TABLES = 2
@@ -2445,15 +2445,15 @@ def
hierarchical_namespace_list(hierarchical_namespace_name: str) -> list[str]:
BUCKET_NAME = "test_bucket"
TABLE_METADATA_LOCATION_REGEX = re.compile(
- r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/
- my_iceberg_table-[a-z]{20}/metadata/
+ r"""s3://test_bucket/my_iceberg_database-[a-z]{8}.db/
+ my_iceberg_table-[a-z]{8}/metadata/
[0-9]{5}-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""",
re.X,
)
BQ_TABLE_METADATA_LOCATION_REGEX = re.compile(
- r"""gs://alexstephen-test-bq-bucket/my_iceberg_database_[a-z]{20}.db/
- my_iceberg_table-[a-z]{20}/metadata/
+ r"""gs://alexstephen-test-bq-bucket/my_iceberg_database_[a-z]{8}.db/
+ my_iceberg_table-[a-z]{8}/metadata/
[0-9]{5}-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""",
re.X,
)
@@ -3138,7 +3138,7 @@ def _create_sql_without_rowcount_catalog(name: str,
warehouse: Path) -> Catalog:
from pyiceberg.catalog.sql import SqlCatalog
props = {
- "uri": f"sqlite:////{warehouse}/sql-catalog",
+ "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog",
"warehouse": f"file://{warehouse}",
}
catalog = SqlCatalog(name, **props)
diff --git a/tests/io/test_io.py b/tests/io/test_io.py
index d9bee33f8..51aa86c67 100644
--- a/tests/io/test_io.py
+++ b/tests/io/test_io.py
@@ -17,6 +17,7 @@
import os
import pickle
+import sys
import tempfile
from typing import Any
@@ -27,6 +28,7 @@ from pyiceberg.io import (
PY_IO_IMPL,
_import_file_io,
_infer_file_io_from_scheme,
+ _is_local_path,
load_file_io,
)
from pyiceberg.io.pyarrow import PyArrowFileIO
@@ -47,10 +49,10 @@ def test_custom_local_input_file() -> None:
input_file =
PyArrowFileIO().new_input(location=f"{absolute_file_location}")
# Test opening and reading the file
- f = input_file.open()
- data = f.read()
- assert data == b"foo"
- assert len(input_file) == 3
+ with input_file.open() as f:
+ data = f.read()
+ assert data == b"foo"
+ assert len(input_file) == 3
def test_custom_local_output_file() -> None:
@@ -85,10 +87,10 @@ def test_pickled_pyarrow_round_trip() -> None:
f.write(b"foo")
input_file =
deserialized_file_io.new_input(location=f"{absolute_file_location}")
- f = input_file.open()
- data = f.read()
- assert data == b"foo"
- assert len(input_file) == 3
+ with input_file.open() as f:
+ data = f.read()
+ assert data == b"foo"
+ assert len(input_file) == 3
deserialized_file_io.delete(location=f"{absolute_file_location}")
@@ -339,3 +341,23 @@ def test_infer_file_io_from_schema_unknown() -> None:
_infer_file_io_from_scheme("unknown://bucket/path/", {})
assert str(w[0].message) == "No preferred file implementation for scheme:
unknown"
+
+
[email protected](
+ "path",
+ ["s3://bucket/key", "hdfs://cluster/path", "gs://bucket/obj", "/tmp/foo",
""],
+)
+def test_is_local_path_false_for_uris_and_posix(path: str) -> None:
+ assert _is_local_path(path) is False
+
+
[email protected](sys.platform != "win32", reason="Windows-only behavior")
[email protected]("path", [r"C:\Users\test", r"D:\data\iceberg",
"E:/warehouse"])
+def test_is_local_path_true_on_windows(path: str) -> None:
+ assert _is_local_path(path) is True
+
+
[email protected](sys.platform != "win32", reason="Windows-only behavior")
+def test_infer_file_io_from_scheme_windows_path() -> None:
+ result = _infer_file_io_from_scheme(r"C:\Users\test\warehouse", {})
+ assert isinstance(result, PyArrowFileIO)
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index fef33a25c..b31c18949 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -17,6 +17,7 @@
# pylint: disable=protected-access,unused-argument,redefined-outer-name
import logging
import os
+import sys
import tempfile
import uuid
import warnings
@@ -162,14 +163,14 @@ def test_pyarrow_input_file() -> None:
input_file =
PyArrowFileIO().new_input(location=f"{absolute_file_location}")
# Test opening and reading the file
- r = input_file.open(seekable=False)
- assert isinstance(r, InputStream) # Test that the file object abides
by the InputStream protocol
- data = r.read()
- assert data == b"foo"
- assert len(input_file) == 3
- with pytest.raises(OSError) as exc_info:
- r.seek(0, 0)
- assert "only valid on seekable files" in str(exc_info.value)
+ with input_file.open(seekable=False) as r:
+ assert isinstance(r, InputStream) # Test that the file object
abides by the InputStream protocol
+ data = r.read()
+ assert data == b"foo"
+ assert len(input_file) == 3
+ with pytest.raises(OSError) as exc_info:
+ r.seek(0, 0)
+ assert "only valid on seekable files" in str(exc_info.value)
def test_pyarrow_input_file_seekable() -> None:
@@ -188,15 +189,15 @@ def test_pyarrow_input_file_seekable() -> None:
input_file =
PyArrowFileIO().new_input(location=f"{absolute_file_location}")
# Test opening and reading the file
- r = input_file.open(seekable=True)
- assert isinstance(r, InputStream) # Test that the file object abides
by the InputStream protocol
- data = r.read()
- assert data == b"foo"
- assert len(input_file) == 3
- r.seek(0, 0)
- data = r.read()
- assert data == b"foo"
- assert len(input_file) == 3
+ with input_file.open(seekable=True) as r:
+ assert isinstance(r, InputStream) # Test that the file object
abides by the InputStream protocol
+ data = r.read()
+ assert data == b"foo"
+ assert len(input_file) == 3
+ r.seek(0, 0)
+ data = r.read()
+ assert data == b"foo"
+ assert len(input_file) == 3
def test_pyarrow_output_file() -> None:
@@ -210,9 +211,9 @@ def test_pyarrow_output_file() -> None:
output_file =
PyArrowFileIO().new_output(location=f"{absolute_file_location}")
# Create the output file and write to it
- f = output_file.create()
- assert isinstance(f, OutputStream) # Test that the file object abides
by the OutputStream protocol
- f.write(b"foo")
+ with output_file.create() as output_stream:
+ assert isinstance(output_stream, OutputStream) # Test that the
file object abides by the OutputStream protocol
+ output_stream.write(b"foo")
# Confirm that bytes were written
with open(file_location, "rb") as f:
@@ -283,7 +284,7 @@ def test_raise_on_opening_a_local_file_not_found() -> None:
with pytest.raises(FileNotFoundError) as exc_info:
f.open()
- assert "[Errno 2] Failed to open local file" in str(exc_info.value)
+ assert "Failed to open local file" in str(exc_info.value)
def test_raise_on_opening_an_s3_file_no_permission() -> None:
@@ -1002,7 +1003,7 @@ def _write_table_to_data_file(filepath: str, schema:
pa.Schema, table: pa.Table)
def file_int(schema_int: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_int, metadata={ICEBERG_SCHEMA:
bytes(schema_int.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/a.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema)
+ f"{tmpdir}/a.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema)
)
@@ -1010,7 +1011,7 @@ def file_int(schema_int: Schema, tmpdir: str) -> str:
def file_int_str(schema_int_str: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_int_str,
metadata={ICEBERG_SCHEMA: bytes(schema_int_str.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/a.parquet",
+ f"{tmpdir}/a.parquet",
pyarrow_schema,
pa.Table.from_arrays([pa.array([0, 1, 2]), pa.array(["0", "1", "2"])],
schema=pyarrow_schema),
)
@@ -1020,7 +1021,7 @@ def file_int_str(schema_int_str: Schema, tmpdir: str) ->
str:
def file_string(schema_str: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_str, metadata={ICEBERG_SCHEMA:
bytes(schema_str.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/b.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array(["0", "1", "2"])], schema=pyarrow_schema)
+ f"{tmpdir}/b.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array(["0", "1", "2"])], schema=pyarrow_schema)
)
@@ -1028,7 +1029,7 @@ def file_string(schema_str: Schema, tmpdir: str) -> str:
def file_long(schema_long: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_long, metadata={ICEBERG_SCHEMA:
bytes(schema_long.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/c.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema)
+ f"{tmpdir}/c.parquet", pyarrow_schema,
pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema)
)
@@ -1036,7 +1037,7 @@ def file_long(schema_long: Schema, tmpdir: str) -> str:
def file_struct(schema_struct: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_struct,
metadata={ICEBERG_SCHEMA: bytes(schema_struct.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/d.parquet",
+ f"{tmpdir}/d.parquet",
pyarrow_schema,
pa.Table.from_pylist(
[
@@ -1053,7 +1054,7 @@ def file_struct(schema_struct: Schema, tmpdir: str) ->
str:
def file_list(schema_list: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_list, metadata={ICEBERG_SCHEMA:
bytes(schema_list.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/e.parquet",
+ f"{tmpdir}/e.parquet",
pyarrow_schema,
pa.Table.from_pylist(
[
@@ -1072,7 +1073,7 @@ def file_list_of_structs(schema_list_of_structs: Schema,
tmpdir: str) -> str:
schema_list_of_structs, metadata={ICEBERG_SCHEMA:
bytes(schema_list_of_structs.model_dump_json(), UTF8)}
)
return _write_table_to_file(
- f"file:{tmpdir}/e.parquet",
+ f"{tmpdir}/e.parquet",
pyarrow_schema,
pa.Table.from_pylist(
[
@@ -1091,7 +1092,7 @@ def file_map_of_structs(schema_map_of_structs: Schema,
tmpdir: str) -> str:
schema_map_of_structs, metadata={ICEBERG_SCHEMA:
bytes(schema_map_of_structs.model_dump_json(), UTF8)}
)
return _write_table_to_file(
- f"file:{tmpdir}/e.parquet",
+ f"{tmpdir}/e.parquet",
pyarrow_schema,
pa.Table.from_pylist(
[
@@ -1108,7 +1109,7 @@ def file_map_of_structs(schema_map_of_structs: Schema,
tmpdir: str) -> str:
def file_map(schema_map: Schema, tmpdir: str) -> str:
pyarrow_schema = schema_to_pyarrow(schema_map, metadata={ICEBERG_SCHEMA:
bytes(schema_map.model_dump_json(), UTF8)})
return _write_table_to_file(
- f"file:{tmpdir}/e.parquet",
+ f"{tmpdir}/e.parquet",
pyarrow_schema,
pa.Table.from_pylist(
[
@@ -2322,8 +2323,19 @@ def test_parse_location() -> None:
check_results("hdfs://127.0.0.1/root/foo.txt", "hdfs", "127.0.0.1",
"/root/foo.txt")
check_results("hdfs://clusterA/root/foo.txt", "hdfs", "clusterA",
"/root/foo.txt")
- check_results("/root/foo.txt", "file", "", "/root/foo.txt")
- check_results("/root/tmp/foo.txt", "file", "", "/root/tmp/foo.txt")
+ check_results("/root/foo.txt", "file", "",
os.path.abspath("/root/foo.txt"))
+ check_results("/root/tmp/foo.txt", "file", "",
os.path.abspath("/root/tmp/foo.txt"))
+
+
[email protected](sys.platform != "win32", reason="Windows-only behavior")
+def test_parse_location_windows_drive_letter() -> None:
+ """Windows drive letters should be treated as local file paths, not URL
schemes."""
+ for drive in ("C", "D", "c", "d"):
+ path = f"{drive}:\\Users\\test\\file.avro"
+ scheme, netloc, result_path = PyArrowFileIO.parse_location(path)
+ assert scheme == "file"
+ assert netloc == ""
+ assert result_path == os.path.abspath(path)
def test_make_compatible_name() -> None:
@@ -2480,6 +2492,7 @@ def
test_bin_pack_record_batches_is_lazy(arrow_table_with_null: pa.Table) -> Non
assert len(consumed) == arrow_table_with_null.num_rows
[email protected](sys.platform == "win32", reason="Rich renders different
box characters on Windows terminals")
def test_schema_mismatch_type(table_schema_simple: Schema) -> None:
other_schema = pa.schema(
(
@@ -2503,6 +2516,7 @@ def test_schema_mismatch_type(table_schema_simple:
Schema) -> None:
_check_pyarrow_schema_compatible(table_schema_simple, other_schema)
[email protected](sys.platform == "win32", reason="Rich renders different
box characters on Windows terminals")
def test_schema_mismatch_nullability(table_schema_simple: Schema) -> None:
other_schema = pa.schema(
(
@@ -2541,6 +2555,7 @@ def
test_schema_compatible_nullability_diff(table_schema_simple: Schema) -> None
pytest.fail("Unexpected Exception raised when calling
`_check_pyarrow_schema_compatible`")
[email protected](sys.platform == "win32", reason="Rich renders different
box characters on Windows terminals")
def test_schema_mismatch_missing_field(table_schema_simple: Schema) -> None:
other_schema = pa.schema(
(
@@ -2583,6 +2598,7 @@ def
test_schema_compatible_missing_nullable_field_nested(table_schema_nested: Sc
pytest.fail("Unexpected Exception raised when calling
`_check_pyarrow_schema_compatible`")
[email protected](sys.platform == "win32", reason="Rich renders different
box characters on Windows terminals")
def test_schema_mismatch_missing_required_field_nested(table_schema_nested:
Schema) -> None:
other_schema = table_schema_nested.as_arrow()
other_schema = other_schema.remove(6).insert(
@@ -3485,21 +3501,21 @@ def test_parse_location_defaults() -> None:
scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
assert scheme == "file"
assert netloc == ""
- assert path == "/foo/bar"
+ assert path == os.path.abspath("/foo/bar")
scheme, netloc, path = PyArrowFileIO.parse_location(
"/foo/bar", properties={"DEFAULT_SCHEME": "scheme", "DEFAULT_NETLOC":
"netloc:8000"}
)
assert scheme == "scheme"
assert netloc == "netloc:8000"
- assert path == "/foo/bar"
+ assert path == os.path.abspath("/foo/bar")
scheme, netloc, path = PyArrowFileIO.parse_location(
"/foo/bar", properties={"DEFAULT_SCHEME": "hdfs", "DEFAULT_NETLOC":
"netloc:8000"}
)
assert scheme == "hdfs"
assert netloc == "netloc:8000"
- assert path == "/foo/bar"
+ assert path == os.path.abspath("/foo/bar")
def test_write_and_read_orc(tmp_path: Path) -> None:
diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py
index eac4d520b..18f71993a 100644
--- a/tests/utils/test_manifest.py
+++ b/tests/utils/test_manifest.py
@@ -351,7 +351,7 @@ def
test_read_manifest_cache(generated_manifest_file_file_v2: str) -> None:
def test_write_empty_manifest() -> None:
io = load_file_io()
test_schema = Schema(NestedField(1, "foo", IntegerType(), False))
- with TemporaryDirectory() as tmpdir:
+ with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
tmp_avro_file = tmpdir + "/test_write_manifest.avro"
with pytest.raises(ValueError, match="An empty manifest file has been
written"):
