This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new bcb8eb126e [#9386] feat(python-client): Add minimum viable integration
tests for relational table (#9396)
bcb8eb126e is described below
commit bcb8eb126ec77d829416272b66d29f74423a3a64
Author: George T. C. Lai <[email protected]>
AuthorDate: Thu Dec 25 10:55:21 2025 +0800
[#9386] feat(python-client): Add minimum viable integration tests for
relational table (#9396)
### What changes were proposed in this pull request?
We shall add integration tests for the least implementation of
relational table.
### Why are the changes needed?
This PR added the following classes.
- `TableCreateRequest`
- `TableResponse`
- integration tests of partition operations for relational table.
Fix: #9386
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit tests and integration tests
---------
Signed-off-by: George T. C. Lai <[email protected]>
---
.../client-python/gravitino/dto/rel/column_dto.py | 2 +-
.../dto/requests/add_partitions_request.py | 15 ++-
clients/client-python/gravitino/rest/rest_utils.py | 3 +-
.../tests/integration/test_relational_table.py | 150 +++++++++++++++++++++
.../client-python/tests/unittests/test_requests.py | 59 +++++++-
5 files changed, 222 insertions(+), 7 deletions(-)
diff --git a/clients/client-python/gravitino/dto/rel/column_dto.py
b/clients/client-python/gravitino/dto/rel/column_dto.py
index 521348c7c0..87e30b14a4 100644
--- a/clients/client-python/gravitino/dto/rel/column_dto.py
+++ b/clients/client-python/gravitino/dto/rel/column_dto.py
@@ -50,7 +50,7 @@ class ColumnDTO(Column, DataClassJsonMixin):
)
"""The data type of the column."""
- _comment: str = field(metadata=config(field_name="comment"))
+ _comment: Optional[str] = field(default=None,
metadata=config(field_name="comment"))
"""The comment associated with the column."""
_default_value: Optional[Union[Expression, List[Expression]]] = field(
diff --git
a/clients/client-python/gravitino/dto/requests/add_partitions_request.py
b/clients/client-python/gravitino/dto/requests/add_partitions_request.py
index e54b9f9200..bbf450d9f6 100644
--- a/clients/client-python/gravitino/dto/requests/add_partitions_request.py
+++ b/clients/client-python/gravitino/dto/requests/add_partitions_request.py
@@ -19,6 +19,9 @@ from dataclasses import dataclass, field
from dataclasses_json import config
+from gravitino.dto.rel.partitions.json_serdes.partition_dto_serdes import (
+ PartitionDTOSerdes,
+)
from gravitino.dto.rel.partitions.partition_dto import PartitionDTO
from gravitino.rest.rest_message import RESTRequest
from gravitino.utils.precondition import Precondition
@@ -28,7 +31,17 @@ from gravitino.utils.precondition import Precondition
class AddPartitionsRequest(RESTRequest):
"""Request to add partitions to a table."""
- _partitions: list[PartitionDTO] =
field(metadata=config(field_name="partitions"))
+ _partitions: list[PartitionDTO] = field(
+ metadata=config(
+ field_name="partitions",
+ encoder=lambda items: [
+ PartitionDTOSerdes.serialize(item) for item in items
+ ],
+ decoder=lambda values: [
+ PartitionDTOSerdes.deserialize(value) for value in values
+ ],
+ )
+ )
def validate(self):
Precondition.check_argument(
diff --git a/clients/client-python/gravitino/rest/rest_utils.py
b/clients/client-python/gravitino/rest/rest_utils.py
index 58c4d55b7a..a760f36756 100644
--- a/clients/client-python/gravitino/rest/rest_utils.py
+++ b/clients/client-python/gravitino/rest/rest_utils.py
@@ -16,6 +16,7 @@
# under the License.
import urllib.parse
+
from gravitino.exceptions.base import IllegalArgumentException
@@ -23,4 +24,4 @@ def encode_string(to_encode: str):
if to_encode is None:
raise IllegalArgumentException("Invalid string to encode: None")
- return urllib.parse.quote(to_encode, encoding="utf-8")
+ return urllib.parse.quote(to_encode, safe="", encoding="utf-8")
diff --git a/clients/client-python/tests/integration/test_relational_table.py
b/clients/client-python/tests/integration/test_relational_table.py
new file mode 100644
index 0000000000..c696cdd374
--- /dev/null
+++ b/clients/client-python/tests/integration/test_relational_table.py
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+from datetime import date
+from random import randint
+
+from gravitino import (
+ Catalog,
+ GravitinoAdminClient,
+ GravitinoClient,
+ NameIdentifier,
+)
+from gravitino.api.rel.expressions.literals.literals import Literals
+from gravitino.api.rel.partitions.partitions import Partitions
+from gravitino.api.rel.types.types import Types
+from gravitino.dto.rel.column_dto import ColumnDTO
+from gravitino.dto.rel.partitioning.identity_partitioning_dto import (
+ IdentityPartitioningDTO,
+)
+from tests.integration.containers.hdfs_container import HDFSContainer
+from tests.integration.integration_test_env import IntegrationTestEnv
+
+logger = logging.getLogger(__name__)
+
+
+class TestRelationalTable(IntegrationTestEnv):
+ METALAKE_NAME: str = "TestRelationalTable_metalake" + str(randint(1,
10000))
+ CATALOG_NAME: str = "relational_catalog"
+ CATALOG_PROVIDER: str = "hive"
+ SCHEMA_NAME: str = "test_schema"
+ TABLE_NAME: str = "test_table"
+ TABLE_IDENT: NameIdentifier = NameIdentifier.of(SCHEMA_NAME, TABLE_NAME)
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.hdfs_container: HDFSContainer = HDFSContainer()
+ hive_metastore_uri = f"thrift://{cls.hdfs_container.get_ip()}:9083"
+ logger.info("Started Hive container with metastore URI: %s",
hive_metastore_uri)
+ cls.gravitino_admin_client =
GravitinoAdminClient(uri="http://localhost:8090";)
+ cls.gravitino_admin_client.create_metalake(
+ cls.METALAKE_NAME,
+ comment="Test metalake for relational catalog",
+ properties={},
+ )
+ cls.gravitino_client: GravitinoClient = GravitinoClient(
+ uri="http://localhost:8090";, metalake_name=cls.METALAKE_NAME
+ )
+ cls.catalog = cls.gravitino_client.create_catalog(
+ name=cls.CATALOG_NAME,
+ catalog_type=Catalog.Type.RELATIONAL,
+ provider=cls.CATALOG_PROVIDER,
+ comment="Test relational catalog",
+ properties={"metastore.uris": hive_metastore_uri},
+ )
+ cls.schema = cls.catalog.as_schemas().create_schema(
+ schema_name=cls.SCHEMA_NAME,
+ comment="Test schema",
+ properties={},
+ )
+ cls.relational_catalog = cls.catalog.as_table_catalog()
+ cls.relational_table = cls.relational_catalog.create_table(
+ identifier=cls.TABLE_IDENT,
+ columns=[
+ ColumnDTO.builder()
+ .with_name("dt")
+ .with_data_type(Types.DateType.get())
+ .build(),
+ ColumnDTO.builder()
+ .with_name("country")
+ .with_data_type(Types.StringType.get())
+ .build(),
+ ],
+ partitioning=[
+ IdentityPartitioningDTO("dt"),
+ IdentityPartitioningDTO("country"),
+ ],
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cls.catalog.as_schemas().drop_schema(
+ schema_name=cls.SCHEMA_NAME, cascade=True
+ )
+ cls.gravitino_client.drop_catalog(name=cls.CATALOG_NAME,
force=True)
+ cls.gravitino_admin_client.drop_metalake(name=cls.METALAKE_NAME,
force=True)
+ except Exception as e: # pylint: disable=broad-exception-caught
+ logger.warning("Failed to clean up class-level resources: %s", e)
+
+ # Clean up the HDFS/Hive container
+ if cls.hdfs_container:
+ try:
+ cls.hdfs_container.close()
+ except Exception as e: # pylint: disable=broad-exception-caught
+ logger.warning("Failed to clean up HDFS container: %s", e)
+
+ super().tearDownClass()
+
+ def test_relational_table_partition_ops(self):
+ """Tests add/get/list/drop partition and list partition names of a
relational table."""
+ relational_table = self.relational_catalog.load_table(self.TABLE_IDENT)
+
+ # Tests list partition names
+ partition_names = relational_table.list_partition_names()
+ self.assertEqual(len(partition_names), 0)
+
+ # Tests add partition
+ new_partition = relational_table.add_partition(
+ Partitions.identity(
+ name="dt=2025-12-03/country=us",
+ field_names=[["dt"], ["country"]],
+ values=[
+ Literals.date_literal(date.fromisoformat("2025-12-03")),
+ Literals.string_literal("us"),
+ ],
+ )
+ )
+ partition_names = relational_table.list_partition_names()
+ self.assertEqual(len(partition_names), 1)
+
+ # Tests list partitions
+ partitions = relational_table.list_partitions()
+ self.assertEqual(len(partitions), 1)
+ self.assertEqual(partitions[0], new_partition)
+
+ # Tests get partition
+ partition = relational_table.get_partition(new_partition.name())
+ self.assertEqual(new_partition, partition)
+
+ # Tests drop partition
+ result = relational_table.drop_partition(new_partition.name())
+ self.assertTrue(result)
+ partition_names = relational_table.list_partition_names()
+ self.assertEqual(len(partition_names), 0)
diff --git a/clients/client-python/tests/unittests/test_requests.py
b/clients/client-python/tests/unittests/test_requests.py
index 66d799a71b..f3c2b19d1f 100644
--- a/clients/client-python/tests/unittests/test_requests.py
+++ b/clients/client-python/tests/unittests/test_requests.py
@@ -26,15 +26,66 @@ from gravitino.exceptions.base import
IllegalArgumentException
class TestRequests(unittest.TestCase):
def test_add_partitions_request(self):
- partitions = ["p202508_California"]
- json_str = json.dumps({"partitions": partitions})
+ json_str = """
+ {
+ "partitions": [
+ {
+ "type": "identity",
+ "name": "partition_1",
+ "fieldNames": [["id"]],
+ "values": [
+ {
+ "type": "literal",
+ "dataType": "integer",
+ "value": "0"
+ }
+ ],
+ "properties": {
+ "key1": "value1",
+ "key2": "value2"
+ }
+ }
+ ]
+ }
+ """
+ partitions = json.loads(json_str)
req = AddPartitionsRequest.from_json(json_str)
req_dict = cast(dict, req.to_dict())
- self.assertListEqual(req_dict["partitions"], partitions)
+ self.assertListEqual(req_dict["partitions"], partitions["partitions"])
+ multiple_partitions_json = """
+ {
+ "partitions": [
+ {
+ "type": "identity",
+ "name": "partition_1",
+ "fieldNames": [["id"]],
+ "values": [
+ {
+ "type": "literal",
+ "dataType": "integer",
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "type": "identity",
+ "name": "partition_2",
+ "fieldNames": [["id"]],
+ "values": [
+ {
+ "type": "literal",
+ "dataType": "integer",
+ "value": "1"
+ }
+ ]
+ }
+ ]
+ }
+ """
exceptions = {
"partitions must not be null": '{"partitions": null}',
- "Haven't yet implemented multiple partitions": '{"partitions":
["p1", "p2"]}',
+ "Haven't yet implemented multiple partitions":
multiple_partitions_json,
}
for exception_str, json_str in exceptions.items():
with self.assertRaisesRegex(IllegalArgumentException,
exception_str):
