Copilot commented on code in PR #9478: URL: https://github.com/apache/gravitino/pull/9478#discussion_r2622029235
########## clients/client-python/tests/unittests/test_relational_catalog.py: ########## @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from http.client import HTTPResponse +from unittest.mock import Mock, patch + +from gravitino.client.relational_catalog import RelationalCatalog +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.rel.distribution_dto import DistributionDTO +from gravitino.dto.rel.table_dto import TableDTO +from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.dto.responses.table_response import TableResponse +from gravitino.dto.util.dto_converters import DTOConverters +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace +from gravitino.utils import HTTPClient, Response + + +class TestRelationalCatalog(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.metalake_name = "test_metalake" + cls.catalog_name = "test_catalog" + cls.schema_name = "test_schema" + cls.table_name = "test_table" + cls.catalog_namespace = Namespace.of(cls.metalake_name) + cls.table_identifier = NameIdentifier.of(cls.schema_name, cls.table_name) + cls.rest_client = HTTPClient("http://localhost:8090";) + cls.catalog = RelationalCatalog( + catalog_namespace=cls.catalog_namespace, + name=cls.catalog_name, + catalog_type=RelationalCatalog.Type.RELATIONAL, + provider="test_provider", + audit=AuditDTO("anonymous"), + rest_client=cls.rest_client, + ) + cls.TABLE_DTO_JSON_STRING = """ + { + "name": "example_table", + "comment": "This is an example table", + "audit": { + "creator": "Apache Gravitino", + "createTime":"2025-10-10T00:00:00" + }, + "columns": [ + { + "name": "id", + "type": "integer", + "comment": "id column comment", + "nullable": false, + "autoIncrement": true, + "defaultValue": { + "type": "literal", + "dataType": "integer", + "value": "-1" + } + }, + { + "name": "name", + "type": "varchar(500)", + "comment": "name column comment", + "nullable": true, + "autoIncrement": false, + "defaultValue": { + "type": "literal", + "dataType": "null", + "value": "null" + } + }, + { + "name": "StartingDate", + "type": "timestamp", + "comment": "StartingDate column comment", + "nullable": false, + "autoIncrement": false, + "defaultValue": { + "type": "function", + "funcName": "current_timestamp", + "funcArgs": [] + } + }, + { + "name": "info", + "type": { + "type": "struct", + "fields": [ + { + "name": "position", + "type": "string", + "nullable": true, + "comment": "position field comment" + }, + { + "name": "contact", + "type": { + "type": "list", + "elementType": "integer", + "containsNull": false + }, + "nullable": true, + "comment": "contact field comment" + }, + { + "name": "rating", + "type": { + "type": "map", + "keyType": "string", + "valueType": "integer", + "valueContainsNull": false + }, + "nullable": true, + "comment": "rating field comment" + } + ] + }, + "comment": "info column comment", + "nullable": true + }, + { + "name": "dt", + "type": "date", + "comment": "dt column comment", + "nullable": true + } + ], + "partitioning": [ + { + "strategy": "identity", + "fieldName": [ "dt" ] + } + ], + "distribution": { + "strategy": "hash", + "number": 32, + "funcArgs": [ + { + "type": "field", + "fieldName": [ "id" ] + } + ] + }, + "sortOrders": [ + { + "sortTerm": { + "type": "field", + "fieldName": [ "StartingDate" ] + }, + "direction": "asc", + "nullOrdering": "nulls_first" + } + ], + "indexes": [ + { + "indexType": "primary_key", + "name": "PRIMARY", + "fieldNames": [["id"]] + } + ], + "properties": { + "format": "ORC" + } + } + """ + cls.table_dto = TableDTO.from_json(cls.TABLE_DTO_JSON_STRING) + + def _get_mock_http_resp(self, json_str: str): + mock_http_resp = Mock(HTTPResponse) + mock_http_resp.getcode.return_value = 200 + mock_http_resp.read.return_value = json_str + mock_http_resp.info.return_value = None + mock_http_resp.url = None + mock_resp = Response(mock_http_resp) + return mock_resp + + def test_as_table_catalog(self): + table_catalog = self.catalog.as_table_catalog() + self.assertIs(table_catalog, self.catalog) + + def test_create_table(self): + resp_body = TableResponse(0, self.table_dto) + mock_resp = self._get_mock_http_resp(resp_body.to_json()) + + with patch( + "gravitino.utils.http_client.HTTPClient.post", + return_value=mock_resp, + ): + table = self.catalog.create_table( + identifier=self.table_identifier, + columns=DTOConverters.from_dtos(self.table_dto.columns()), + partitioning=DTOConverters.from_dtos(self.table_dto.partitioning()), + distribution=DTOConverters.from_dto( + self.table_dto.distribution() or DistributionDTO.NONE + ), + sort_orders=DTOConverters.from_dtos(self.table_dto.sort_order()), + indexes=DTOConverters.from_dtos(self.table_dto.index()), + properties=self.table_dto.properties(), + ) + self.assertEqual(table.name(), self.table_dto.name()) + + def test_load_table(self): + resp_body = TableResponse(0, self.table_dto) + mock_resp = self._get_mock_http_resp(resp_body.to_json()) + + with patch( + "gravitino.utils.http_client.HTTPClient.get", + return_value=mock_resp, + ): + table = self.catalog.load_table(self.table_identifier) + self.assertEqual(table.name(), self.table_dto.name()) + + def test_list_tables(self): + resp_body = EntityListResponse( + 0, + [ + NameIdentifier.of( + self.metalake_name, + self.catalog_name, + self.schema_name, + self.table_name, + ) + ], + ) + mock_resp = self._get_mock_http_resp(resp_body.to_json()) + + with patch( + "gravitino.utils.http_client.HTTPClient.get", + return_value=mock_resp, + ): + tables = self.catalog.list_tables(namespace=Namespace.of(self.schema_name)) + self.assertEqual(len(tables), 1) + self.assertEqual(tables[0], self.table_identifier) Review Comment: The test coverage for `test_relational_catalog.py` lacks error handling scenarios. Consider adding test cases for invalid identifiers, namespaces, or error responses from the server to ensure the error handler is working correctly. For example, test cases for `NoSuchSchemaException`, `TableAlreadyExistsException`, and invalid namespace scenarios would improve robustness. ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.name(), expected.name()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) + self.assertListEqual(converted.field_names(), expected.field_names()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_single_field_transforms(self): + converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) + for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): + transform_class = DTOConverters._SINGLE_FIELD_TRANSFORM_TYPES[ # pylint: disable=protected-access + key + ] + expected = transform_class(*converted.field_name()) + self.assertTrue(converted.field_name() == expected.field_name()) + + def test_to_dtos_bucket_truncate_transforms(self): + num_buckets, width = 10, 5 + field_name = ["score"] + bucket_transform = Transforms.bucket(num_buckets, field_name) + trunc_transform = Transforms.truncate(width, field_name) + transforms = [bucket_transform, trunc_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_bucket_dto = BucketPartitioningDTO(num_buckets, field_name) + expected_trunc_dto = TruncatePartitioningDTO(width=width, field_name=field_name) + expected_dtos = [expected_bucket_dto, expected_trunc_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, BucketPartitioningDTO): + self.assertEqual(converted.num_buckets(), expected.num_buckets()) + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertEqual(converted.width(), expected.width()) + self.assertListEqual(converted.field_name(), expected.field_name()) + + def test_to_dtos_list_range_transforms(self): + field_names = [["createTime"], ["city"]] + list_transform = Transforms.list( + field_names=field_names, + assignments=[ + Partitions.list( + name="p0", + lists=[ + [Literals.date_literal(date(2025, 8, 8))], + [Literals.string_literal("Los Angeles")], + ], + properties={}, + ), + ], + ) + range_transform = Transforms.range( + field_name=["score"], + assignments=[ + Partitions.range( + name="p1", + lower=Literals.integer_literal(0), + upper=Literals.integer_literal(100), + properties={}, + ) + ], + ) + transforms = [list_transform, range_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_list_dto = ListPartitioningDTO( + field_names=field_names, + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in list_transform.assignments() + ], + ) + expected_range_dto = RangePartitioningDTO( + field_name=["score"], + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in range_transform.assignments() + ], + ) + expected_dtos = [expected_list_dto, expected_range_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, ListPartitioningDTO): + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertListEqual(converted.field_name(), expected.field_name()) + self.assertListEqual(converted.assignments(), expected.assignments()) + + def test_to_dtos_apply_transform(self): + function_name = "test_function" + args: list[FunctionArg] = [ + LiteralDTO.builder() + .with_data_type(Types.IntegerType.get()) + .with_value("-1") + .build(), + LiteralDTO.builder() + .with_data_type(Types.BooleanType.get()) + .with_value("True") + .build(), + ] + apply_transform = Transforms.apply( + name=function_name, + arguments=[ + Literals.of(value="-1", data_type=Types.IntegerType.get()), + Literals.of(value="True", data_type=Types.BooleanType.get()), + ], + ) + expected = FunctionPartitioningDTO(function_name, *args) + converted = DTOConverters.to_dto(apply_transform) + self.assertTrue(converted.function_name() == expected.function_name()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.function_name(), expected.function_name()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.sort_term(), expected.sort_term()) self.assertEqual(converted.direction(), expected.direction()) self.assertEqual(converted.null_ordering(), expected.null_ordering()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) + self.assertListEqual(converted.field_names(), expected.field_names()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_single_field_transforms(self): + converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) + for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): + transform_class = DTOConverters._SINGLE_FIELD_TRANSFORM_TYPES[ # pylint: disable=protected-access + key + ] + expected = transform_class(*converted.field_name()) + self.assertTrue(converted.field_name() == expected.field_name()) + + def test_to_dtos_bucket_truncate_transforms(self): + num_buckets, width = 10, 5 + field_name = ["score"] + bucket_transform = Transforms.bucket(num_buckets, field_name) + trunc_transform = Transforms.truncate(width, field_name) + transforms = [bucket_transform, trunc_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_bucket_dto = BucketPartitioningDTO(num_buckets, field_name) + expected_trunc_dto = TruncatePartitioningDTO(width=width, field_name=field_name) + expected_dtos = [expected_bucket_dto, expected_trunc_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, BucketPartitioningDTO): + self.assertEqual(converted.num_buckets(), expected.num_buckets()) + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertEqual(converted.width(), expected.width()) + self.assertListEqual(converted.field_name(), expected.field_name()) + + def test_to_dtos_list_range_transforms(self): + field_names = [["createTime"], ["city"]] + list_transform = Transforms.list( + field_names=field_names, + assignments=[ + Partitions.list( + name="p0", + lists=[ + [Literals.date_literal(date(2025, 8, 8))], + [Literals.string_literal("Los Angeles")], + ], + properties={}, + ), + ], + ) + range_transform = Transforms.range( + field_name=["score"], + assignments=[ + Partitions.range( + name="p1", + lower=Literals.integer_literal(0), + upper=Literals.integer_literal(100), + properties={}, + ) + ], + ) + transforms = [list_transform, range_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_list_dto = ListPartitioningDTO( + field_names=field_names, + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in list_transform.assignments() + ], + ) + expected_range_dto = RangePartitioningDTO( + field_name=["score"], + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in range_transform.assignments() + ], + ) + expected_dtos = [expected_list_dto, expected_range_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, ListPartitioningDTO): + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertListEqual(converted.field_name(), expected.field_name()) + self.assertListEqual(converted.assignments(), expected.assignments()) + + def test_to_dtos_apply_transform(self): + function_name = "test_function" + args: list[FunctionArg] = [ + LiteralDTO.builder() + .with_data_type(Types.IntegerType.get()) + .with_value("-1") + .build(), + LiteralDTO.builder() + .with_data_type(Types.BooleanType.get()) + .with_value("True") + .build(), + ] + apply_transform = Transforms.apply( + name=function_name, + arguments=[ + Literals.of(value="-1", data_type=Types.IntegerType.get()), + Literals.of(value="True", data_type=Types.BooleanType.get()), + ], + ) + expected = FunctionPartitioningDTO(function_name, *args) + converted = DTOConverters.to_dto(apply_transform) + self.assertTrue(converted.function_name() == expected.function_name()) + self.assertListEqual(converted.args(), expected.args()) + + def test_to_dtos_raise_exception(self): + with self.assertRaisesRegex(IllegalArgumentException, "Unsupported transform"): + DTOConverters.to_dto( + cast(Transform, MagicMock(name="UnsupportedTransform", spec=Transform)) + ) + + def test_to_dto_distribution(self): + field_names = [f"field_{i}" for i in range(2)] + field_ref_dtos = [ + FieldReferenceDTO.builder().with_field_name(field_name=[field_name]).build() + for field_name in field_names + ] + distribution = Distributions.of( + Strategy.HASH, + 4, + *[FieldReference(field_names=[field_name]) for field_name in field_names], + ) + distribution_dto = DistributionDTO( + strategy=Strategy.HASH, number=4, args=field_ref_dtos + ) + converted = DTOConverters.to_dto(distribution) + self.assertTrue(converted == distribution_dto) + + self.assertTrue( + DTOConverters.to_dto(Distributions.NONE) == DistributionDTO.NONE + ) + self.assertTrue(DTOConverters.to_dto(distribution_dto) == distribution_dto) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(DTOConverters.to_dto(distribution_dto), distribution_dto) ``` ########## clients/client-python/gravitino/client/relational_catalog.py: ########## @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Optional + +from gravitino.api.catalog import Catalog +from gravitino.api.rel.column import Column +from gravitino.api.rel.expressions.distributions.distribution import Distribution +from gravitino.api.rel.expressions.sorts.sort_order import SortOrder +from gravitino.api.rel.expressions.transforms.transform import Transform +from gravitino.api.rel.indexes.index import Index +from gravitino.api.rel.table import Table +from gravitino.api.rel.table_catalog import TableCatalog +from gravitino.client.base_schema_catalog import BaseSchemaCatalog +from gravitino.client.relational_table import RelationalTable +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.rel.distribution_dto import DistributionDTO +from gravitino.dto.requests.table_create_request import TableCreateRequest +from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.dto.responses.table_response import TableResponse +from gravitino.dto.util.dto_converters import DTOConverters +from gravitino.exceptions.handlers.table_error_handler import TABLE_ERROR_HANDLER +from gravitino.name_identifier import NameIdentifier +from gravitino.namespace import Namespace +from gravitino.rest.rest_utils import encode_string +from gravitino.utils import HTTPClient + + +class RelationalCatalog(BaseSchemaCatalog, TableCatalog): + """Relational catalog is a catalog implementation that supports relational database. + + It's like metadata operations, for example, schemas and tables list, creation, update + and deletion. A Relational catalog is under the metalake. + """ + + def __init__( + self, + catalog_namespace: Namespace, + name: str, + catalog_type: Catalog.Type, + provider: str, + audit: AuditDTO, + rest_client: HTTPClient, + comment: Optional[str] = None, + properties: Optional[dict[str, str]] = None, + ): + super().__init__( + catalog_namespace, + name, + catalog_type, + provider, + comment, + properties, + audit, + rest_client, + ) + + def as_table_catalog(self) -> TableCatalog: + return self + + @staticmethod + def check_table_name_identifier(identifier: NameIdentifier) -> None: + """Check whether the `NameIdentifier` of a table is valid. + Args: + identifier (NameIdentifier): + The NameIdentifier to check, which should be "schema.table" format. + + Raises: + IllegalNameIdentifierException: If the Namespace is not valid. + """ Review Comment: The docstring formatting is inconsistent with the project style. There should be a blank line between the description and the "Args:" section. Compare with other methods in the codebase like `load_schema` in `base_schema_catalog.py`. ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) + self.assertListEqual(converted.field_names(), expected.field_names()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_single_field_transforms(self): + converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) + for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): + transform_class = DTOConverters._SINGLE_FIELD_TRANSFORM_TYPES[ # pylint: disable=protected-access + key + ] + expected = transform_class(*converted.field_name()) + self.assertTrue(converted.field_name() == expected.field_name()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.field_name(), expected.field_name()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.null_ordering(), expected.null_ordering()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.type(), expected.type()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) + self.assertListEqual(converted.field_names(), expected.field_names()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_single_field_transforms(self): + converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) + for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): + transform_class = DTOConverters._SINGLE_FIELD_TRANSFORM_TYPES[ # pylint: disable=protected-access + key + ] + expected = transform_class(*converted.field_name()) + self.assertTrue(converted.field_name() == expected.field_name()) + + def test_to_dtos_bucket_truncate_transforms(self): + num_buckets, width = 10, 5 + field_name = ["score"] + bucket_transform = Transforms.bucket(num_buckets, field_name) + trunc_transform = Transforms.truncate(width, field_name) + transforms = [bucket_transform, trunc_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_bucket_dto = BucketPartitioningDTO(num_buckets, field_name) + expected_trunc_dto = TruncatePartitioningDTO(width=width, field_name=field_name) + expected_dtos = [expected_bucket_dto, expected_trunc_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, BucketPartitioningDTO): + self.assertEqual(converted.num_buckets(), expected.num_buckets()) + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertEqual(converted.width(), expected.width()) + self.assertListEqual(converted.field_name(), expected.field_name()) + + def test_to_dtos_list_range_transforms(self): + field_names = [["createTime"], ["city"]] + list_transform = Transforms.list( + field_names=field_names, + assignments=[ + Partitions.list( + name="p0", + lists=[ + [Literals.date_literal(date(2025, 8, 8))], + [Literals.string_literal("Los Angeles")], + ], + properties={}, + ), + ], + ) + range_transform = Transforms.range( + field_name=["score"], + assignments=[ + Partitions.range( + name="p1", + lower=Literals.integer_literal(0), + upper=Literals.integer_literal(100), + properties={}, + ) + ], + ) + transforms = [list_transform, range_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_list_dto = ListPartitioningDTO( + field_names=field_names, + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in list_transform.assignments() + ], + ) + expected_range_dto = RangePartitioningDTO( + field_name=["score"], + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in range_transform.assignments() + ], + ) + expected_dtos = [expected_list_dto, expected_range_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, ListPartitioningDTO): + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertListEqual(converted.field_name(), expected.field_name()) + self.assertListEqual(converted.assignments(), expected.assignments()) + + def test_to_dtos_apply_transform(self): + function_name = "test_function" + args: list[FunctionArg] = [ + LiteralDTO.builder() + .with_data_type(Types.IntegerType.get()) + .with_value("-1") + .build(), + LiteralDTO.builder() + .with_data_type(Types.BooleanType.get()) + .with_value("True") + .build(), + ] + apply_transform = Transforms.apply( + name=function_name, + arguments=[ + Literals.of(value="-1", data_type=Types.IntegerType.get()), + Literals.of(value="True", data_type=Types.BooleanType.get()), + ], + ) + expected = FunctionPartitioningDTO(function_name, *args) + converted = DTOConverters.to_dto(apply_transform) + self.assertTrue(converted.function_name() == expected.function_name()) + self.assertListEqual(converted.args(), expected.args()) + + def test_to_dtos_raise_exception(self): + with self.assertRaisesRegex(IllegalArgumentException, "Unsupported transform"): + DTOConverters.to_dto( + cast(Transform, MagicMock(name="UnsupportedTransform", spec=Transform)) + ) + + def test_to_dto_distribution(self): + field_names = [f"field_{i}" for i in range(2)] + field_ref_dtos = [ + FieldReferenceDTO.builder().with_field_name(field_name=[field_name]).build() + for field_name in field_names + ] + distribution = Distributions.of( + Strategy.HASH, + 4, + *[FieldReference(field_names=[field_name]) for field_name in field_names], + ) + distribution_dto = DistributionDTO( + strategy=Strategy.HASH, number=4, args=field_ref_dtos + ) + converted = DTOConverters.to_dto(distribution) + self.assertTrue(converted == distribution_dto) + + self.assertTrue( + DTOConverters.to_dto(Distributions.NONE) == DistributionDTO.NONE + ) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted.sort_term(), expected.sort_term()) ``` ########## clients/client-python/tests/unittests/dto/util/test_dto_converters.py: ########## @@ -768,3 +778,211 @@ def test_to_dto_list_partition(self): ) converted = DTOConverters.to_dto(partition) self.assertTrue(converted == expected) + + def test_to_dtos_columns(self): + column_names = {f"column_{i}" for i in range(2)} + column_data_types = {Types.IntegerType.get(), Types.BooleanType.get()} + columns: list[Column] = [ + Column.of(name=column_name, data_type=column_data_type) + for column_name, column_data_type in zip(column_names, column_data_types) + ] + expected = [ + ColumnDTO.builder() + .with_name(column.name()) + .with_data_type(column.data_type()) + .with_default_value(Column.DEFAULT_VALUE_NOT_SET) + .build() + for column in columns + ] + self.assertListEqual(DTOConverters.to_dtos(columns), expected) + + def test_to_dtos_sort_orders(self): + directions = {SortDirection.ASCENDING, SortDirection.DESCENDING} + null_orderings = {NullOrdering.NULLS_LAST, NullOrdering.NULLS_FIRST} + field_names = [ + [f"score_{i}"] for i in range(len(directions) * len(null_orderings)) + ] + sort_orders: list[SortOrder] = [] + expected_dtos: list[SortOrderDTO] = [] + for field_name, (direction, null_ordering) in zip( + field_names, product(directions, null_orderings) + ): + field_ref = FieldReference(field_names=field_name) + sort_orders.append( + SortOrders.of( + expression=field_ref, + direction=direction, + null_ordering=null_ordering, + ) + ) + field_ref_dto = ( + FieldReferenceDTO.builder() + .with_field_name(field_name=field_name) + .build() + ) + expected_dtos.append( + SortOrderDTO( + sort_term=field_ref_dto, + direction=direction, + null_ordering=null_ordering, + ) + ) + converted_dtos = DTOConverters.to_dtos(sort_orders) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.sort_term() == expected.sort_term()) + self.assertTrue(converted.direction() == expected.direction()) + self.assertTrue(converted.null_ordering() == expected.null_ordering()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_indexes(self): + field_names = [[f"field_{i}"] for i in range(2)] + + indexes: list[Index] = [ + Indexes.of(index_type, index_type.value, field_names) + for index_type in Index.IndexType + ] + expected_dtos: list[IndexDTO] = [ + IndexDTO( + index_type=index_type, + name=index_type.value, + field_names=field_names, + ) + for index_type in Index.IndexType + ] + converted_dtos = DTOConverters.to_dtos(indexes) + for converted, expected in zip(converted_dtos, expected_dtos): + self.assertTrue(converted.type() == expected.type()) + self.assertTrue(converted.name() == expected.name()) + self.assertListEqual(converted.field_names(), expected.field_names()) + + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + + def test_to_dtos_single_field_transforms(self): + converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) + for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): + transform_class = DTOConverters._SINGLE_FIELD_TRANSFORM_TYPES[ # pylint: disable=protected-access + key + ] + expected = transform_class(*converted.field_name()) + self.assertTrue(converted.field_name() == expected.field_name()) + + def test_to_dtos_bucket_truncate_transforms(self): + num_buckets, width = 10, 5 + field_name = ["score"] + bucket_transform = Transforms.bucket(num_buckets, field_name) + trunc_transform = Transforms.truncate(width, field_name) + transforms = [bucket_transform, trunc_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_bucket_dto = BucketPartitioningDTO(num_buckets, field_name) + expected_trunc_dto = TruncatePartitioningDTO(width=width, field_name=field_name) + expected_dtos = [expected_bucket_dto, expected_trunc_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, BucketPartitioningDTO): + self.assertEqual(converted.num_buckets(), expected.num_buckets()) + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertEqual(converted.width(), expected.width()) + self.assertListEqual(converted.field_name(), expected.field_name()) + + def test_to_dtos_list_range_transforms(self): + field_names = [["createTime"], ["city"]] + list_transform = Transforms.list( + field_names=field_names, + assignments=[ + Partitions.list( + name="p0", + lists=[ + [Literals.date_literal(date(2025, 8, 8))], + [Literals.string_literal("Los Angeles")], + ], + properties={}, + ), + ], + ) + range_transform = Transforms.range( + field_name=["score"], + assignments=[ + Partitions.range( + name="p1", + lower=Literals.integer_literal(0), + upper=Literals.integer_literal(100), + properties={}, + ) + ], + ) + transforms = [list_transform, range_transform] + converted_dtos = DTOConverters.to_dtos(transforms) + expected_list_dto = ListPartitioningDTO( + field_names=field_names, + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in list_transform.assignments() + ], + ) + expected_range_dto = RangePartitioningDTO( + field_name=["score"], + assignments=[ + DTOConverters.to_dto(assignment) + for assignment in range_transform.assignments() + ], + ) + expected_dtos = [expected_list_dto, expected_range_dto] + for converted, expected in zip(converted_dtos, expected_dtos): + if isinstance(expected, ListPartitioningDTO): + self.assertListEqual(converted.field_names(), expected.field_names()) + else: + self.assertListEqual(converted.field_name(), expected.field_name()) + self.assertListEqual(converted.assignments(), expected.assignments()) + + def test_to_dtos_apply_transform(self): + function_name = "test_function" + args: list[FunctionArg] = [ + LiteralDTO.builder() + .with_data_type(Types.IntegerType.get()) + .with_value("-1") + .build(), + LiteralDTO.builder() + .with_data_type(Types.BooleanType.get()) + .with_value("True") + .build(), + ] + apply_transform = Transforms.apply( + name=function_name, + arguments=[ + Literals.of(value="-1", data_type=Types.IntegerType.get()), + Literals.of(value="True", data_type=Types.BooleanType.get()), + ], + ) + expected = FunctionPartitioningDTO(function_name, *args) + converted = DTOConverters.to_dto(apply_transform) + self.assertTrue(converted.function_name() == expected.function_name()) + self.assertListEqual(converted.args(), expected.args()) + + def test_to_dtos_raise_exception(self): + with self.assertRaisesRegex(IllegalArgumentException, "Unsupported transform"): + DTOConverters.to_dto( + cast(Transform, MagicMock(name="UnsupportedTransform", spec=Transform)) + ) + + def test_to_dto_distribution(self): + field_names = [f"field_{i}" for i in range(2)] + field_ref_dtos = [ + FieldReferenceDTO.builder().with_field_name(field_name=[field_name]).build() + for field_name in field_names + ] + distribution = Distributions.of( + Strategy.HASH, + 4, + *[FieldReference(field_names=[field_name]) for field_name in field_names], + ) + distribution_dto = DistributionDTO( + strategy=Strategy.HASH, number=4, args=field_ref_dtos + ) + converted = DTOConverters.to_dto(distribution) + self.assertTrue(converted == distribution_dto) Review Comment: assertTrue(a == b) cannot provide an informative message. Using assertEqual(a, b) instead will give more informative messages. ```suggestion self.assertEqual(converted, distribution_dto) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
