This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 9fd440ed Deprecate for 0.8.0 release (#1269)
9fd440ed is described below
commit 9fd440edba6d6f45821fea244fa00e2ed1f0f363
Author: Kevin Liu <[email protected]>
AuthorDate: Fri Nov 1 14:47:44 2024 -0400
Deprecate for 0.8.0 release (#1269)
* remove deprecated functions
* remove deprecated properties
* fix tests based on deprecated properties
* update readme
* get_first_property_value
* deprecate properties
---
mkdocs/docs/configuration.md | 8 +++----
pyiceberg/catalog/__init__.py | 23 -------------------
pyiceberg/catalog/dynamodb.py | 23 +++++--------------
pyiceberg/catalog/glue.py | 23 +++++--------------
pyiceberg/io/pyarrow.py | 17 --------------
pyiceberg/table/__init__.py | 50 ------------------------------------------
tests/catalog/test_dynamodb.py | 27 -----------------------
tests/catalog/test_glue.py | 27 +----------------------
tests/conftest.py | 8 -------
9 files changed, 15 insertions(+), 191 deletions(-)
diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md
index 1269e34a..2ed58091 100644
--- a/mkdocs/docs/configuration.md
+++ b/mkdocs/docs/configuration.md
@@ -340,8 +340,8 @@ catalog:
<!-- prettier-ignore-start -->
-!!! warning "Deprecated Properties"
- `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`,
`aws_secret_access_key`, `aws_session_token` are deprecated and will be removed
in 0.8.0:
+!!! warning "Removed Properties"
+ The properties `profile_name`, `region_name`, `botocore_session`,
`aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were
deprecated and removed in 0.8.0
<!-- prettier-ignore-end -->
@@ -396,8 +396,8 @@ catalog:
<!-- prettier-ignore-start -->
-!!! warning "Deprecated Properties"
- `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`,
`aws_secret_access_key`, `aws_session_token` are deprecated and will be removed
in 0.8.0:
+!!! warning "Removed Properties"
+ The properties `profile_name`, `region_name`, `botocore_session`,
`aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were
deprecated and removed in 0.8.0
<!-- prettier-ignore-end -->
diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py
index cc953a88..2104fcab 100644
--- a/pyiceberg/catalog/__init__.py
+++ b/pyiceberg/catalog/__init__.py
@@ -106,21 +106,6 @@ TABLE_METADATA_FILE_NAME_REGEX = re.compile(
re.X,
)
-DEPRECATED_PROFILE_NAME = "profile_name"
-DEPRECATED_REGION = "region_name"
-DEPRECATED_BOTOCORE_SESSION = "botocore_session"
-DEPRECATED_ACCESS_KEY_ID = "aws_access_key_id"
-DEPRECATED_SECRET_ACCESS_KEY = "aws_secret_access_key"
-DEPRECATED_SESSION_TOKEN = "aws_session_token"
-DEPRECATED_PROPERTY_NAMES = {
- DEPRECATED_PROFILE_NAME,
- DEPRECATED_REGION,
- DEPRECATED_BOTOCORE_SESSION,
- DEPRECATED_ACCESS_KEY_ID,
- DEPRECATED_SECRET_ACCESS_KEY,
- DEPRECATED_SESSION_TOKEN,
-}
-
class CatalogType(Enum):
REST = "rest"
@@ -794,14 +779,6 @@ class MetastoreCatalog(Catalog, ABC):
def __init__(self, name: str, **properties: str):
super().__init__(name, **properties)
- for property_name in DEPRECATED_PROPERTY_NAMES:
- if self.properties.get(property_name):
- deprecation_message(
- deprecated_in="0.7.0",
- removed_in="0.8.0",
- help_message=f"The property {property_name} is deprecated.
Please use properties that start with client., glue., and dynamo. instead",
- )
-
def create_table_transaction(
self,
identifier: Union[str, Identifier],
diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py
index 59cbe8ce..b999df21 100644
--- a/pyiceberg/catalog/dynamodb.py
+++ b/pyiceberg/catalog/dynamodb.py
@@ -30,12 +30,6 @@ from typing import (
import boto3
from pyiceberg.catalog import (
- DEPRECATED_ACCESS_KEY_ID,
- DEPRECATED_BOTOCORE_SESSION,
- DEPRECATED_PROFILE_NAME,
- DEPRECATED_REGION,
- DEPRECATED_SECRET_ACCESS_KEY,
- DEPRECATED_SESSION_TOKEN,
ICEBERG,
METADATA_LOCATION,
PREVIOUS_METADATA_LOCATION,
@@ -102,18 +96,11 @@ class DynamoDbCatalog(MetastoreCatalog):
super().__init__(name, **properties)
session = boto3.Session(
- profile_name=get_first_property_value(properties,
DYNAMODB_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
- region_name=get_first_property_value(properties, DYNAMODB_REGION,
AWS_REGION, DEPRECATED_REGION),
- botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
- aws_access_key_id=get_first_property_value(
- properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID,
DEPRECATED_ACCESS_KEY_ID
- ),
- aws_secret_access_key=get_first_property_value(
- properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY,
DEPRECATED_SECRET_ACCESS_KEY
- ),
- aws_session_token=get_first_property_value(
- properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN,
DEPRECATED_SESSION_TOKEN
- ),
+ profile_name=properties.get(DYNAMODB_PROFILE_NAME),
+ region_name=get_first_property_value(properties, DYNAMODB_REGION,
AWS_REGION),
+ aws_access_key_id=get_first_property_value(properties,
DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+ aws_secret_access_key=get_first_property_value(properties,
DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+ aws_session_token=get_first_property_value(properties,
DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN),
)
self.dynamodb = session.client(DYNAMODB_CLIENT)
self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME,
DYNAMODB_TABLE_NAME_DEFAULT)
diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py
index 79850dbb..385873c4 100644
--- a/pyiceberg/catalog/glue.py
+++ b/pyiceberg/catalog/glue.py
@@ -40,12 +40,6 @@ from mypy_boto3_glue.type_defs import (
)
from pyiceberg.catalog import (
- DEPRECATED_ACCESS_KEY_ID,
- DEPRECATED_BOTOCORE_SESSION,
- DEPRECATED_PROFILE_NAME,
- DEPRECATED_REGION,
- DEPRECATED_SECRET_ACCESS_KEY,
- DEPRECATED_SESSION_TOKEN,
EXTERNAL_TABLE,
ICEBERG,
LOCATION,
@@ -303,18 +297,11 @@ class GlueCatalog(MetastoreCatalog):
super().__init__(name, **properties)
session = boto3.Session(
- profile_name=get_first_property_value(properties,
GLUE_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
- region_name=get_first_property_value(properties, GLUE_REGION,
AWS_REGION, DEPRECATED_REGION),
- botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
- aws_access_key_id=get_first_property_value(
- properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID,
DEPRECATED_ACCESS_KEY_ID
- ),
- aws_secret_access_key=get_first_property_value(
- properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY,
DEPRECATED_SECRET_ACCESS_KEY
- ),
- aws_session_token=get_first_property_value(
- properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN,
DEPRECATED_SESSION_TOKEN
- ),
+ profile_name=properties.get(GLUE_PROFILE_NAME),
+ region_name=get_first_property_value(properties, GLUE_REGION,
AWS_REGION),
+ aws_access_key_id=get_first_property_value(properties,
GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+ aws_secret_access_key=get_first_property_value(properties,
GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+ aws_session_token=get_first_property_value(properties,
GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN),
)
self.glue: GlueClient = session.client("glue",
endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT))
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index c6ecd4a6..ab4de518 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -1688,23 +1688,6 @@ def project_batches(
total_row_count += len(batch)
-@deprecated(
- deprecated_in="0.7.0",
- removed_in="0.8.0",
- help_message="The public API for 'to_requested_schema' is deprecated and
is replaced by '_to_requested_schema'",
-)
-def to_requested_schema(requested_schema: Schema, file_schema: Schema, table:
pa.Table) -> pa.Table:
- struct_array = visit_with_partner(requested_schema, table,
ArrowProjectionVisitor(file_schema), ArrowAccessor(file_schema))
-
- arrays = []
- fields = []
- for pos, field in enumerate(requested_schema.fields):
- array = struct_array.field(pos)
- arrays.append(array)
- fields.append(pa.field(field.name, array.type, field.optional))
- return pa.Table.from_arrays(arrays, schema=pa.schema(fields))
-
-
def _to_requested_schema(
requested_schema: Schema,
file_schema: Schema,
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index 0163c425..e431101b 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -89,7 +89,6 @@ from pyiceberg.table.sorting import UNSORTED_SORT_ORDER,
SortOrder
from pyiceberg.table.update import (
AddPartitionSpecUpdate,
AddSchemaUpdate,
- AddSnapshotUpdate,
AddSortOrderUpdate,
AssertCreate,
AssertRefSnapshotId,
@@ -314,55 +313,6 @@ class Transaction:
updates = properties or kwargs
return self._apply((SetPropertiesUpdate(updates=updates),))
- @deprecated(
- deprecated_in="0.7.0",
- removed_in="0.8.0",
- help_message="Please use one of the functions in ManageSnapshots
instead",
- )
- def add_snapshot(self, snapshot: Snapshot) -> Transaction:
- """Add a new snapshot to the table.
-
- Returns:
- The transaction with the add-snapshot staged.
- """
- updates = (AddSnapshotUpdate(snapshot=snapshot),)
-
- return self._apply(updates, ())
-
- @deprecated(
- deprecated_in="0.7.0",
- removed_in="0.8.0",
- help_message="Please use one of the functions in ManageSnapshots
instead",
- )
- def set_ref_snapshot(
- self,
- snapshot_id: int,
- parent_snapshot_id: Optional[int],
- ref_name: str,
- type: str,
- max_ref_age_ms: Optional[int] = None,
- max_snapshot_age_ms: Optional[int] = None,
- min_snapshots_to_keep: Optional[int] = None,
- ) -> Transaction:
- """Update a ref to a snapshot.
-
- Returns:
- The transaction with the set-snapshot-ref staged
- """
- updates = (
- SetSnapshotRefUpdate(
- snapshot_id=snapshot_id,
- ref_name=ref_name,
- type=type,
- max_ref_age_ms=max_ref_age_ms,
- max_snapshot_age_ms=max_snapshot_age_ms,
- min_snapshots_to_keep=min_snapshots_to_keep,
- ),
- )
-
- requirements = (AssertRefSnapshotId(snapshot_id=parent_snapshot_id,
ref="main"),)
- return self._apply(updates, requirements)
-
def _set_ref_snapshot(
self,
snapshot_id: int,
diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py
index 0289c5d0..194caf46 100644
--- a/tests/catalog/test_dynamodb.py
+++ b/tests/catalog/test_dynamodb.py
@@ -45,7 +45,6 @@ from pyiceberg.schema import Schema
from pyiceberg.typedef import Properties
from tests.conftest import (
BUCKET_NAME,
- DEPRECATED_AWS_SESSION_PROPERTIES,
TABLE_METADATA_LOCATION_REGEX,
UNIFIED_AWS_SESSION_PROPERTIES,
)
@@ -563,28 +562,6 @@ def
test_update_namespace_properties_overlap_update_removal(_bucket_initialize:
assert test_catalog.load_namespace_properties(database_name) ==
test_properties
-def test_passing_provided_profile() -> None:
- catalog_name = "test_ddb_catalog"
- session_props = {
- "aws_access_key_id": "abc",
- "aws_secret_access_key": "def",
- "aws_session_token": "ghi",
- "region_name": "eu-central-1",
- "botocore_session": None,
- "profile_name": None,
- }
- props = {"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}
- props.update(session_props) # type: ignore
- with mock.patch("boto3.Session", return_value=mock.Mock()) as mock_session:
- mock_client = mock.Mock()
- mock_session.return_value.client.return_value = mock_client
- mock_client.describe_table.return_value = {"Table": {"TableStatus":
"ACTIVE"}}
- test_catalog = DynamoDbCatalog(catalog_name, **props)
- assert test_catalog.dynamodb is mock_client
- mock_session.assert_called_with(**session_props)
- assert test_catalog.dynamodb is mock_session().client()
-
-
@mock_aws
def test_passing_glue_session_properties() -> None:
session_properties: Properties = {
@@ -594,7 +571,6 @@ def test_passing_glue_session_properties() -> None:
"dynamodb.region": "dynamodb.region",
"dynamodb.session-token": "dynamodb.session-token",
**UNIFIED_AWS_SESSION_PROPERTIES,
- **DEPRECATED_AWS_SESSION_PROPERTIES,
}
with mock.patch("boto3.Session") as mock_session:
@@ -609,7 +585,6 @@ def test_passing_glue_session_properties() -> None:
aws_session_token="dynamodb.session-token",
region_name="dynamodb.region",
profile_name="dynamodb.profile-name",
- botocore_session=None,
)
assert test_catalog.dynamodb is mock_session().client()
@@ -619,7 +594,6 @@ def test_passing_unified_session_properties_to_dynamodb()
-> None:
session_properties: Properties = {
"dynamodb.profile-name": "dynamodb.profile-name",
**UNIFIED_AWS_SESSION_PROPERTIES,
- **DEPRECATED_AWS_SESSION_PROPERTIES,
}
with mock.patch("boto3.Session") as mock_session:
@@ -634,7 +608,6 @@ def test_passing_unified_session_properties_to_dynamodb()
-> None:
aws_session_token="client.session-token",
region_name="client.region",
profile_name="dynamodb.profile-name",
- botocore_session=None,
)
assert test_catalog.dynamodb is mock_session().client()
diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py
index 53af7504..41b196fa 100644
--- a/tests/catalog/test_glue.py
+++ b/tests/catalog/test_glue.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Dict, List
+from typing import List
from unittest import mock
import boto3
@@ -40,7 +40,6 @@ from pyiceberg.typedef import Properties
from pyiceberg.types import IntegerType
from tests.conftest import (
BUCKET_NAME,
- DEPRECATED_AWS_SESSION_PROPERTIES,
TABLE_METADATA_LOCATION_REGEX,
UNIFIED_AWS_SESSION_PROPERTIES,
)
@@ -638,26 +637,6 @@ def
test_update_namespace_properties_overlap_update_removal(
assert test_catalog.load_namespace_properties(database_name) ==
test_properties
-@mock_aws
-def test_passing_profile_name() -> None:
- session_properties: Dict[str, Any] = {
- "aws_access_key_id": "abc",
- "aws_secret_access_key": "def",
- "aws_session_token": "ghi",
- "region_name": "eu-central-1",
- "profile_name": "sandbox",
- "botocore_session": None,
- }
- test_properties = {"type": "glue"}
- test_properties.update(session_properties)
-
- with mock.patch("boto3.Session") as mock_session:
- test_catalog = GlueCatalog("glue", **test_properties)
-
- mock_session.assert_called_with(**session_properties)
- assert test_catalog.glue is mock_session().client()
-
-
@mock_aws
def test_passing_glue_session_properties() -> None:
session_properties: Properties = {
@@ -667,7 +646,6 @@ def test_passing_glue_session_properties() -> None:
"glue.region": "glue.region",
"glue.session-token": "glue.session-token",
**UNIFIED_AWS_SESSION_PROPERTIES,
- **DEPRECATED_AWS_SESSION_PROPERTIES,
}
with mock.patch("boto3.Session") as mock_session:
@@ -679,7 +657,6 @@ def test_passing_glue_session_properties() -> None:
aws_session_token="glue.session-token",
region_name="glue.region",
profile_name="glue.profile-name",
- botocore_session=None,
)
assert test_catalog.glue is mock_session().client()
@@ -689,7 +666,6 @@ def test_passing_unified_session_properties_to_glue() ->
None:
session_properties: Properties = {
"glue.profile-name": "glue.profile-name",
**UNIFIED_AWS_SESSION_PROPERTIES,
- **DEPRECATED_AWS_SESSION_PROPERTIES,
}
with mock.patch("boto3.Session") as mock_session:
@@ -701,7 +677,6 @@ def test_passing_unified_session_properties_to_glue() ->
None:
aws_session_token="client.session-token",
region_name="client.region",
profile_name="glue.profile-name",
- botocore_session=None,
)
assert test_catalog.glue is mock_session().client()
diff --git a/tests/conftest.py b/tests/conftest.py
index b05947eb..e7e73375 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2029,14 +2029,6 @@ TABLE_METADATA_LOCATION_REGEX = re.compile(
re.X,
)
-DEPRECATED_AWS_SESSION_PROPERTIES = {
- "aws_access_key_id": "aws_access_key_id",
- "aws_secret_access_key": "aws_secret_access_key",
- "aws_session_token": "aws_session_token",
- "region_name": "region_name",
- "profile_name": "profile_name",
-}
-
UNIFIED_AWS_SESSION_PROPERTIES = {
"client.access-key-id": "client.access-key-id",
"client.secret-access-key": "client.secret-access-key",
