This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new b97bd9412b [#8797] feat(client-python): add class AddPartitionsRequest
(#9045)
b97bd9412b is described below
commit b97bd9412b2fef17cc36e4df14fa4cbea38b0620
Author: George T. C. Lai <[email protected]>
AuthorDate: Fri Nov 14 01:47:37 2025 +0800
[#8797] feat(client-python): add class AddPartitionsRequest (#9045)
### What changes were proposed in this pull request?
The following methods and classes were included in this PR:
- method `toDTO(Partition partition)` in `DTOConverters`,
- method `toFunctionArg(Expression expression)` in `DTOConverters`, and
- class `AddPartitionsRequest`.
### Why are the changes needed?
We need to enable table operations in Python client that requires
implementation of the aforementioned methods and classes used in
`RelationalTable`.
Fix: #8797
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit tests
---------
Signed-off-by: George T. C. Lai <[email protected]>
---
.../dto/requests/add_partitions_request.py | 39 +++++
.../gravitino/dto/util/dto_converters.py | 107 +++++++++++-
.../unittests/dto/util/test_dto_converters.py | 182 ++++++++++++++++++++-
.../client-python/tests/unittests/test_requests.py | 41 +++++
4 files changed, 366 insertions(+), 3 deletions(-)
diff --git
a/clients/client-python/gravitino/dto/requests/add_partitions_request.py
b/clients/client-python/gravitino/dto/requests/add_partitions_request.py
new file mode 100644
index 0000000000..e54b9f9200
--- /dev/null
+++ b/clients/client-python/gravitino/dto/requests/add_partitions_request.py
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+
+from dataclasses_json import config
+
+from gravitino.dto.rel.partitions.partition_dto import PartitionDTO
+from gravitino.rest.rest_message import RESTRequest
+from gravitino.utils.precondition import Precondition
+
+
+@dataclass
+class AddPartitionsRequest(RESTRequest):
+ """Request to add partitions to a table."""
+
+ _partitions: list[PartitionDTO] =
field(metadata=config(field_name="partitions"))
+
+ def validate(self):
+ Precondition.check_argument(
+ self._partitions is not None, "partitions must not be null"
+ )
+ Precondition.check_argument(
+ len(self._partitions) == 1, "Haven't yet implemented multiple
partitions"
+ )
diff --git a/clients/client-python/gravitino/dto/util/dto_converters.py
b/clients/client-python/gravitino/dto/util/dto_converters.py
index e2a781e19a..94d7224379 100644
--- a/clients/client-python/gravitino/dto/util/dto_converters.py
+++ b/clients/client-python/gravitino/dto/util/dto_converters.py
@@ -23,8 +23,9 @@ from gravitino.api.rel.expressions.distributions.distribution
import Distributio
from gravitino.api.rel.expressions.distributions.distributions import
Distributions
from gravitino.api.rel.expressions.expression import Expression
from gravitino.api.rel.expressions.function_expression import
FunctionExpression
+from gravitino.api.rel.expressions.literals.literal import Literal
from gravitino.api.rel.expressions.literals.literals import Literals
-from gravitino.api.rel.expressions.named_reference import NamedReference
+from gravitino.api.rel.expressions.named_reference import FieldReference,
NamedReference
from gravitino.api.rel.expressions.sorts.sort_order import SortOrder
from gravitino.api.rel.expressions.sorts.sort_orders import SortOrders
from gravitino.api.rel.expressions.transforms.transform import Transform
@@ -32,7 +33,9 @@ from gravitino.api.rel.expressions.transforms.transforms
import Transforms
from gravitino.api.rel.expressions.unparsed_expression import
UnparsedExpression
from gravitino.api.rel.indexes.index import Index
from gravitino.api.rel.indexes.indexes import Indexes
+from gravitino.api.rel.partitions.identity_partition import IdentityPartition
from gravitino.api.rel.partitions.list_partition import ListPartition
+from gravitino.api.rel.partitions.partition import Partition
from gravitino.api.rel.partitions.range_partition import RangePartition
from gravitino.api.rel.table import Table
from gravitino.api.rel.types.types import Types
@@ -58,6 +61,10 @@ from gravitino.dto.rel.partitioning.range_partitioning_dto
import RangePartition
from gravitino.dto.rel.partitioning.truncate_partitioning_dto import (
TruncatePartitioningDTO,
)
+from gravitino.dto.rel.partitions.identity_partition_dto import
IdentityPartitionDTO
+from gravitino.dto.rel.partitions.list_partition_dto import ListPartitionDTO
+from gravitino.dto.rel.partitions.partition_dto import PartitionDTO
+from gravitino.dto.rel.partitions.range_partition_dto import RangePartitionDTO
from gravitino.dto.rel.sort_order_dto import SortOrderDTO
from gravitino.dto.rel.table_dto import TableDTO
from gravitino.exceptions.base import IllegalArgumentException
@@ -302,3 +309,101 @@ class DTOConverters:
if not dtos:
return []
return [DTOConverters.from_dto(dto) for dto in dtos]
+
+ @staticmethod
+ def to_function_arg(expression: Expression) -> FunctionArg:
+ """Converts an Expression to an FunctionArg DTO.
+
+ Args:
+ expression (Expression): The expression to be converted.
+
+ Returns:
+ FunctionArg: The expression DTO.
+ """
+
+ if isinstance(expression, FunctionArg):
+ return cast(FunctionArg, expression)
+ if isinstance(expression, Literal):
+ if expression is Literals.NULL:
+ return LiteralDTO.NULL
+ return (
+ LiteralDTO.builder()
+ .with_value(str(expression.value()))
+ .with_data_type(expression.data_type())
+ .build()
+ )
+ if isinstance(expression, FieldReference):
+ return (
+ FieldReferenceDTO.builder()
+ .with_field_name(expression.field_name())
+ .build()
+ )
+ if isinstance(expression, FunctionExpression):
+ return (
+ FuncExpressionDTO.builder()
+ .with_function_name(expression.function_name())
+ .with_function_args(
+ [
+ DTOConverters.to_function_arg(arg)
+ for arg in expression.arguments()
+ ]
+ )
+ .build()
+ )
+ if isinstance(expression, UnparsedExpression):
+ return (
+ UnparsedExpressionDTO.builder()
+ .with_unparsed_expression(expression.unparsed_expression())
+ .build()
+ )
+ raise IllegalArgumentException(
+ f"Unsupported expression type: {expression.__class__.__name__}"
+ )
+
+ @singledispatchmethod
+ @staticmethod
+ def to_dto(obj) -> object:
+ raise IllegalArgumentException(f"Unsupported type: {type(obj)}")
+
+ @to_dto.register
+ @staticmethod
+ def _(obj: Partition) -> PartitionDTO:
+ if isinstance(obj, RangePartition):
+ range_partition = cast(RangePartition, obj)
+ return RangePartitionDTO(
+ name=range_partition.name(),
+ upper=cast(
+ LiteralDTO,
DTOConverters.to_function_arg(range_partition.upper())
+ ),
+ lower=cast(
+ LiteralDTO,
DTOConverters.to_function_arg(range_partition.lower())
+ ),
+ properties=range_partition.properties(),
+ )
+ if isinstance(obj, IdentityPartition):
+ identity_partition = cast(IdentityPartition, obj)
+ return IdentityPartitionDTO(
+ name=identity_partition.name(),
+ values=[
+ cast(LiteralDTO, DTOConverters.to_function_arg(v))
+ for v in identity_partition.values()
+ ],
+ field_names=identity_partition.field_names(),
+ properties=identity_partition.properties(),
+ )
+ if isinstance(obj, ListPartition):
+ list_partition = cast(ListPartition, obj)
+ return ListPartitionDTO(
+ name=list_partition.name(),
+ lists=[
+ [
+ cast(LiteralDTO, DTOConverters.to_function_arg(v))
+ for v in list_item
+ ]
+ for list_item in list_partition.lists()
+ ],
+ properties=list_partition.properties(),
+ )
+ raise IllegalArgumentException(
+ f"Unsupported partition type: {obj.__class__.__name__}"
+ )
diff --git
a/clients/client-python/tests/unittests/dto/util/test_dto_converters.py
b/clients/client-python/tests/unittests/dto/util/test_dto_converters.py
index 2fd7a1ec55..171f7ee864 100644
--- a/clients/client-python/tests/unittests/dto/util/test_dto_converters.py
+++ b/clients/client-python/tests/unittests/dto/util/test_dto_converters.py
@@ -16,10 +16,10 @@
# under the License.
import unittest
-from datetime import datetime
+from datetime import date, datetime
from itertools import product
from random import randint, random
-from typing import cast
+from typing import Dict, cast
from unittest.mock import MagicMock, patch
from gravitino.api.rel.column import Column
@@ -36,6 +36,8 @@ from gravitino.api.rel.expressions.transforms.transforms
import Transforms
from gravitino.api.rel.expressions.unparsed_expression import
UnparsedExpression
from gravitino.api.rel.indexes.index import Index
from gravitino.api.rel.indexes.indexes import Indexes
+from gravitino.api.rel.partitions.partition import Partition
+from gravitino.api.rel.partitions.partitions import Partitions
from gravitino.api.rel.table import Table
from gravitino.api.rel.types.types import Types
from gravitino.dto.rel.column_dto import ColumnDTO
@@ -63,6 +65,9 @@ from gravitino.dto.rel.partitioning.truncate_partitioning_dto
import (
TruncatePartitioningDTO,
)
from gravitino.dto.rel.partitioning.year_partitioning_dto import
YearPartitioningDTO
+from gravitino.dto.rel.partitions.identity_partition_dto import
IdentityPartitionDTO
+from gravitino.dto.rel.partitions.list_partition_dto import ListPartitionDTO
+from gravitino.dto.rel.partitions.range_partition_dto import RangePartitionDTO
from gravitino.dto.rel.sort_order_dto import SortOrderDTO
from gravitino.dto.rel.table_dto import TableDTO
from gravitino.dto.util.dto_converters import DTOConverters
@@ -93,6 +98,16 @@ class TestDTOConverters(unittest.TestCase):
Types.VarCharType.of(10): "test",
Types.FixedCharType.of(10): "test",
}
+ cls.function_args = {
+ FieldReference(field_names=["score"]): FieldReferenceDTO.builder()
+ .with_field_name(field_name=["score"])
+ .build(),
+ UnparsedExpression.of(
+ unparsed_expression="unparsed"
+ ): UnparsedExpressionDTO.builder()
+ .with_unparsed_expression("unparsed")
+ .build(),
+ }
cls.table_dto_json = """
{
"name": "example_table",
@@ -590,3 +605,166 @@ class TestDTOConverters(unittest.TestCase):
)
self.assertEqual(table.audit_info(), dto.audit_info())
self.assertEqual(table.properties(), dto.properties())
+
+ def test_to_function_arg_function_arg(self):
+ for expression, function_arg in
TestDTOConverters.function_args.items():
+ converted = DTOConverters.to_function_arg(function_arg)
+ self.assertTrue(converted == function_arg)
+ converted = DTOConverters.to_function_arg(expression)
+ self.assertTrue(converted == function_arg)
+
+ for data_type, value in TestDTOConverters.literals.items():
+ literal = Literals.of(value=value, data_type=data_type)
+ expected = (
+ LiteralDTO.builder()
+ .with_data_type(data_type)
+ .with_value(str(value))
+ .build()
+ )
+ self.assertTrue(DTOConverters.to_function_arg(literal) == expected)
+ self.assertTrue(DTOConverters.to_function_arg(Literals.NULL) ==
LiteralDTO.NULL)
+
+ function_name = "test_function"
+ args: list[FunctionArg] = [
+ LiteralDTO.builder()
+ .with_data_type(Types.IntegerType.get())
+ .with_value("-1")
+ .build(),
+ LiteralDTO.builder()
+ .with_data_type(Types.BooleanType.get())
+ .with_value("True")
+ .build(),
+ ]
+ expected = (
+ FuncExpressionDTO.builder()
+ .with_function_name(function_name)
+ .with_function_args(args)
+ .build()
+ )
+ func_expression = FunctionExpression.of(
+ function_name,
+ Literals.of(value="-1", data_type=Types.IntegerType.get()),
+ Literals.of(value="True", data_type=Types.BooleanType.get()),
+ )
+ converted = DTOConverters.to_function_arg(func_expression)
+ self.assertTrue(converted == expected)
+
+ with self.assertRaisesRegex(
+ IllegalArgumentException, "Unsupported expression type"
+ ):
+ DTOConverters.to_function_arg(DistributionDTO.NONE)
+
+ def test_to_dto_raise_exception(self):
+ with self.assertRaisesRegex(IllegalArgumentException, "Unsupported
type"):
+ DTOConverters.to_dto("test")
+
+ def test_to_dto_range_partition(self):
+ converted = DTOConverters.to_dto(
+ Partitions.range(
+ name="p0",
+ upper=Literals.NULL,
+ lower=Literals.integer_literal(6),
+ properties={},
+ )
+ )
+ expected = RangePartitionDTO(
+ name="p0",
+ upper=LiteralDTO.NULL,
+ lower=LiteralDTO.builder()
+ .with_value("6")
+ .with_data_type(Types.IntegerType.get())
+ .build(),
+ properties={},
+ )
+ self.assertTrue(converted == expected)
+
+ def test_to_dto_partition_raise_exception(self):
+ class InvalidPartition(Partition):
+ def name(self) -> str:
+ return "invalid_partition"
+
+ def properties(self) -> Dict[str, str]:
+ return {}
+
+ with self.assertRaisesRegex(
+ IllegalArgumentException, "Unsupported partition type"
+ ):
+ DTOConverters.to_dto(InvalidPartition())
+
+ def test_to_dto_identity_partition(self):
+ partition_name = "dt=2025-08-08/country=us"
+ field_names = [["dt"], ["country"]]
+ properties = {"location":
"/user/hive/warehouse/tpch_flat_orc_2.db/orders"}
+ values = [
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.DateType.get())
+ .with_value(value="2025-08-08")
+ .build(),
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.StringType.get())
+ .with_value(value="us")
+ .build(),
+ ]
+ expected = IdentityPartitionDTO(
+ name=partition_name,
+ field_names=field_names,
+ values=values,
+ properties=properties,
+ )
+ partition = Partitions.identity(
+ name=partition_name,
+ field_names=field_names,
+ values=[
+ Literals.date_literal(date(2025, 8, 8)),
+ Literals.string_literal("us"),
+ ],
+ properties=properties,
+ )
+ converted = DTOConverters.to_dto(partition)
+ self.assertTrue(converted == expected)
+
+ def test_to_dto_list_partition(self):
+ partition_name = "p202508_California"
+ properties = {"key": "value"}
+ partition = Partitions.list(
+ partition_name,
+ [
+ [
+ Literals.date_literal(date(2025, 8, 8)),
+ Literals.string_literal("Los Angeles"),
+ ],
+ [
+ Literals.date_literal(date(2025, 8, 8)),
+ Literals.string_literal("San Francisco"),
+ ],
+ ],
+ properties,
+ )
+ expected = ListPartitionDTO(
+ name=partition_name,
+ lists=[
+ [
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.DateType.get())
+ .with_value(value="2025-08-08")
+ .build(),
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.StringType.get())
+ .with_value(value="Los Angeles")
+ .build(),
+ ],
+ [
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.DateType.get())
+ .with_value(value="2025-08-08")
+ .build(),
+ LiteralDTO.builder()
+ .with_data_type(data_type=Types.StringType.get())
+ .with_value(value="San Francisco")
+ .build(),
+ ],
+ ],
+ properties=properties,
+ )
+ converted = DTOConverters.to_dto(partition)
+ self.assertTrue(converted == expected)
diff --git a/clients/client-python/tests/unittests/test_requests.py
b/clients/client-python/tests/unittests/test_requests.py
new file mode 100644
index 0000000000..260855965b
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_requests.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import unittest
+from typing import cast
+
+from gravitino.dto.requests.add_partitions_request import AddPartitionsRequest
+from gravitino.exceptions.base import IllegalArgumentException
+
+
+class TestRequests(unittest.TestCase):
+ def test_add_partitions_request(self):
+ partitions = ["p202508_California"]
+ json_str = json.dumps({"partitions": partitions})
+ req = AddPartitionsRequest.from_json(json_str)
+ req_dict = cast(dict, req.to_dict())
+ self.assertListEqual(req_dict["partitions"], partitions)
+
+ exceptions = {
+ "partitions must not be null": '{"partitions": null}',
+ "Haven't yet implemented multiple partitions": '{"partitions":
["p1", "p2"]}',
+ }
+ for exception_str, json_str in exceptions.items():
+ with self.assertRaisesRegex(IllegalArgumentException,
exception_str):
+ req = AddPartitionsRequest.from_json(json_str)
+ req.validate()
