jun-he commented on a change in pull request #3407: URL: https://github.com/apache/iceberg/pull/3407#discussion_r791198968
########## File path: python/src/iceberg/partitioning.py ########## @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations +from typing import List, Tuple, Dict + +from iceberg.types import NestedField, Schema, index_by_id +from iceberg.exceptions import ValidationException + + +class PartitionField: + def __init__(self, source_id: int, field_id: int, name: str): + self._source_id = source_id + self._field_id = field_id + self._name = name + # TODO: add transform field + + @property + def source_id(self) -> int: + return self._source_id + + @property + def field_id(self) -> int: + return self._field_id + + @property + def name(self) -> str: + return self._name + + # TODO: add property for transform + + def __eq__(self, other): + if isinstance(other, PartitionField): + return ( + self.source_id == other.source_id + and self.field_id == other.field_id + and self.name == other.name + ) + return False + + def __str__(self): + # TODO: add transform to str representation + return f"{self.field_id}: {self.name} ({self._source_id})" + + def __repr__(self): + return f"PartitionField(field_id={self.field_id}, name={self.name}, source_id={self._source_id})" + + +def check_compatibility(spec, schema: Schema): + for field in spec.fields: + source_type = schema.find_type(field.source_id) + if source_type is None: + raise ValidationException( + f"Cannot find source column for partition field: {field}" + ) + if not source_type.is_primitive: + raise ValidationException( + f"Cannot partition by non-primitive source field: {source_type}" + ) + # TODO: Add transform check + + +PARTITION_DATA_ID_START = 1000 + + +class PartitionSpec: + def __init__( + self, + schema: Schema, + spec_id: int, + part_fields: List[PartitionField], + last_assigned_field_id: int, + ): + self._schema = schema + self._spec_id = spec_id + self._fields = tuple(part_fields) + self._last_assigned_field_id = last_assigned_field_id + self.fields_by_source_id: Dict[int, List[PartitionField]] = {} + + @property + def schema(self) -> Schema: + return self._schema + + @property + def spec_id(self) -> int: + return self._spec_id + + @property + def fields(self) -> Tuple[PartitionField]: + return self._fields + + @property + def last_assigned_field_id(self) -> int: + return self._last_assigned_field_id + + def is_partitioned(self): + return len(self.fields) < 1 + + def get_fields_by_source_id(self, field_id: int) -> List[PartitionField]: + if not self.fields_by_source_id: + for field in self.fields: + self.fields_by_source_id[field.source_id] = [field] + return self.fields_by_source_id.get(field_id) + + def __eq__(self, other): + if isinstance(other, PartitionSpec): + if self.spec_id != other.spec_id: + return False + return self.fields == other.fields + return False + + def __str__(self): + partition_spec_str = "[" + for field in self.fields: + partition_spec_str += "\n" + partition_spec_str += " " + str(field) + partition_spec_str += "]" if len(self.fields) > 0 else "\n" + return partition_spec_str + + def __repr__(self): + return "PartitionSpec" + self.__str__() + + def compatible_with(self, other): + if self.__eq__(other): + return True + + if len(self.fields) != len(other.fields): + return False + + index = 0 + # TODO: Need to fix with transforms + for field in self.fields: + other_field = other.fields[index] + if ( + field.source_id != other_field.source_id + or field.name != other_field.name + ): + # TODO: Add transform check + return False + index += 1 + return True + + def partition_type(self): + struct_fields = [] + # TODO: Needs transform + pass + + def partition_to_path(self): + # TODO: Needs transform + pass + + def unpartitioned(self): + return PartitionSpec( + schema=Schema, + spec_id=0, + part_fields=[], + last_assigned_field_id=PARTITION_DATA_ID_START - 1, + ) + + def has_sequential_ids(self, spec): + index = 0 + for field in spec.fields: + if field.field_id != PARTITION_DATA_ID_START + index: + return False + index += 1 + return True + + +class Builder: + def __init__(self, schema: Schema): + self._schema = schema + self.fields: List[PartitionField] = [] + self.partition_names = set() + self.dedup_fields = dict() + self.spec_id = 0 + self.last_assigned_field_id = PARTITION_DATA_ID_START - 1 + + def builder_for(self, schema): + return Builder(schema=schema) + + def next_field_id(self): Review comment: add type info for the return result. ########## File path: python/src/iceberg/partitioning.py ########## @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations +from typing import List, Tuple, Dict + +from iceberg.types import NestedField, Schema, index_by_id +from iceberg.exceptions import ValidationException + + +class PartitionField: + def __init__(self, source_id: int, field_id: int, name: str): + self._source_id = source_id + self._field_id = field_id + self._name = name + # TODO: add transform field + + @property + def source_id(self) -> int: + return self._source_id + + @property + def field_id(self) -> int: + return self._field_id + + @property + def name(self) -> str: + return self._name + + # TODO: add property for transform + + def __eq__(self, other): + if isinstance(other, PartitionField): + return ( + self.source_id == other.source_id + and self.field_id == other.field_id + and self.name == other.name + ) + return False + + def __str__(self): + # TODO: add transform to str representation + return f"{self.field_id}: {self.name} ({self._source_id})" + + def __repr__(self): + return f"PartitionField(field_id={self.field_id}, name={self.name}, source_id={self._source_id})" + + +def check_compatibility(spec, schema: Schema): + for field in spec.fields: + source_type = schema.find_type(field.source_id) + if source_type is None: + raise ValidationException( + f"Cannot find source column for partition field: {field}" + ) + if not source_type.is_primitive: + raise ValidationException( + f"Cannot partition by non-primitive source field: {source_type}" + ) + # TODO: Add transform check + + +PARTITION_DATA_ID_START = 1000 + + +class PartitionSpec: + def __init__( + self, + schema: Schema, + spec_id: int, + part_fields: List[PartitionField], + last_assigned_field_id: int, + ): + self._schema = schema + self._spec_id = spec_id + self._fields = tuple(part_fields) + self._last_assigned_field_id = last_assigned_field_id + self.fields_by_source_id: Dict[int, List[PartitionField]] = {} + + @property + def schema(self) -> Schema: + return self._schema + + @property + def spec_id(self) -> int: + return self._spec_id + + @property + def fields(self) -> Tuple[PartitionField]: + return self._fields + + @property + def last_assigned_field_id(self) -> int: + return self._last_assigned_field_id + + def is_partitioned(self): + return len(self.fields) < 1 + + def get_fields_by_source_id(self, field_id: int) -> List[PartitionField]: + if not self.fields_by_source_id: + for field in self.fields: + self.fields_by_source_id[field.source_id] = [field] + return self.fields_by_source_id.get(field_id) + + def __eq__(self, other): + if isinstance(other, PartitionSpec): + if self.spec_id != other.spec_id: + return False + return self.fields == other.fields + return False + + def __str__(self): + partition_spec_str = "[" + for field in self.fields: + partition_spec_str += "\n" + partition_spec_str += " " + str(field) + partition_spec_str += "]" if len(self.fields) > 0 else "\n" + return partition_spec_str + + def __repr__(self): + return "PartitionSpec" + self.__str__() + + def compatible_with(self, other): + if self.__eq__(other): + return True + + if len(self.fields) != len(other.fields): + return False + + index = 0 + # TODO: Need to fix with transforms + for field in self.fields: + other_field = other.fields[index] + if ( + field.source_id != other_field.source_id + or field.name != other_field.name + ): + # TODO: Add transform check + return False + index += 1 + return True + + def partition_type(self): + struct_fields = [] + # TODO: Needs transform + pass + + def partition_to_path(self): + # TODO: Needs transform + pass + + def unpartitioned(self): + return PartitionSpec( + schema=Schema, + spec_id=0, + part_fields=[], + last_assigned_field_id=PARTITION_DATA_ID_START - 1, + ) + + def has_sequential_ids(self, spec): + index = 0 + for field in spec.fields: + if field.field_id != PARTITION_DATA_ID_START + index: + return False + index += 1 + return True + + +class Builder: + def __init__(self, schema: Schema): + self._schema = schema + self.fields: List[PartitionField] = [] + self.partition_names = set() + self.dedup_fields = dict() + self.spec_id = 0 + self.last_assigned_field_id = PARTITION_DATA_ID_START - 1 + + def builder_for(self, schema): Review comment: nit: add type for schema ########## File path: python/src/iceberg/partitioning.py ########## @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations +from typing import List, Tuple, Dict + +from iceberg.types import NestedField, Schema, index_by_id +from iceberg.exceptions import ValidationException + + +class PartitionField: + def __init__(self, source_id: int, field_id: int, name: str): + self._source_id = source_id + self._field_id = field_id + self._name = name + # TODO: add transform field + + @property + def source_id(self) -> int: + return self._source_id + + @property + def field_id(self) -> int: + return self._field_id + + @property + def name(self) -> str: + return self._name + + # TODO: add property for transform + + def __eq__(self, other): + if isinstance(other, PartitionField): + return ( + self.source_id == other.source_id + and self.field_id == other.field_id + and self.name == other.name + ) + return False + + def __str__(self): + # TODO: add transform to str representation + return f"{self.field_id}: {self.name} ({self._source_id})" + + def __repr__(self): + return f"PartitionField(field_id={self.field_id}, name={self.name}, source_id={self._source_id})" + + +def check_compatibility(spec, schema: Schema): + for field in spec.fields: + source_type = schema.find_type(field.source_id) + if source_type is None: + raise ValidationException( + f"Cannot find source column for partition field: {field}" + ) + if not source_type.is_primitive: + raise ValidationException( + f"Cannot partition by non-primitive source field: {source_type}" + ) + # TODO: Add transform check + + +PARTITION_DATA_ID_START = 1000 + + +class PartitionSpec: + def __init__( + self, + schema: Schema, + spec_id: int, + part_fields: List[PartitionField], + last_assigned_field_id: int, + ): + self._schema = schema + self._spec_id = spec_id + self._fields = tuple(part_fields) + self._last_assigned_field_id = last_assigned_field_id + self.fields_by_source_id: Dict[int, List[PartitionField]] = {} + + @property + def schema(self) -> Schema: + return self._schema + + @property + def spec_id(self) -> int: + return self._spec_id + + @property + def fields(self) -> Tuple[PartitionField]: + return self._fields + + @property + def last_assigned_field_id(self) -> int: + return self._last_assigned_field_id + + def is_partitioned(self): + return len(self.fields) < 1 + + def get_fields_by_source_id(self, field_id: int) -> List[PartitionField]: + if not self.fields_by_source_id: + for field in self.fields: + self.fields_by_source_id[field.source_id] = [field] + return self.fields_by_source_id.get(field_id) + + def __eq__(self, other): + if isinstance(other, PartitionSpec): + if self.spec_id != other.spec_id: + return False + return self.fields == other.fields + return False + + def __str__(self): + partition_spec_str = "[" + for field in self.fields: + partition_spec_str += "\n" + partition_spec_str += " " + str(field) + partition_spec_str += "]" if len(self.fields) > 0 else "\n" + return partition_spec_str + + def __repr__(self): + return "PartitionSpec" + self.__str__() + + def compatible_with(self, other): + if self.__eq__(other): + return True + + if len(self.fields) != len(other.fields): + return False + + index = 0 + # TODO: Need to fix with transforms + for field in self.fields: + other_field = other.fields[index] + if ( + field.source_id != other_field.source_id + or field.name != other_field.name + ): + # TODO: Add transform check + return False + index += 1 + return True + + def partition_type(self): + struct_fields = [] + # TODO: Needs transform + pass + + def partition_to_path(self): + # TODO: Needs transform + pass + + def unpartitioned(self): + return PartitionSpec( + schema=Schema, + spec_id=0, + part_fields=[], + last_assigned_field_id=PARTITION_DATA_ID_START - 1, + ) + + def has_sequential_ids(self, spec): + index = 0 + for field in spec.fields: + if field.field_id != PARTITION_DATA_ID_START + index: + return False + index += 1 + return True + + +class Builder: Review comment: Should we make the Builder to be the inner class of PartitionSpec class? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
