amogh-jahagirdar commented on code in PR #15049: URL: https://github.com/apache/iceberg/pull/15049#discussion_r2921660522
########## core/src/main/java/org/apache/iceberg/ManifestInfo.java: ########## @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import org.apache.iceberg.types.Types; + +/** Summary information about a manifest referenced by a v4 root manifest entry. */ +interface ManifestInfo { + Types.NestedField ADDED_FILES_COUNT = + Types.NestedField.required( + 504, "added_files_count", Types.IntegerType.get(), "Number of files added"); + Types.NestedField EXISTING_FILES_COUNT = + Types.NestedField.required( + 505, "existing_files_count", Types.IntegerType.get(), "Number of existing files"); + Types.NestedField DELETED_FILES_COUNT = + Types.NestedField.required( + 506, "deleted_files_count", Types.IntegerType.get(), "Number of deleted files"); + Types.NestedField REPLACED_FILES_COUNT = + Types.NestedField.required( + 160, "replaced_files_count", Types.IntegerType.get(), "Number of replaced files"); + Types.NestedField ADDED_ROWS_COUNT = + Types.NestedField.required( + 512, "added_rows_count", Types.LongType.get(), "Number of rows in added files"); + Types.NestedField EXISTING_ROWS_COUNT = + Types.NestedField.required( + 513, "existing_rows_count", Types.LongType.get(), "Number of rows in existing files"); + Types.NestedField DELETED_ROWS_COUNT = + Types.NestedField.required( + 514, "deleted_rows_count", Types.LongType.get(), "Number of rows in deleted files"); + Types.NestedField REPLACED_ROWS_COUNT = + Types.NestedField.required( + 161, "replaced_rows_count", Types.LongType.get(), "Number of rows in replaced files"); + Types.NestedField MIN_SEQUENCE_NUMBER = + Types.NestedField.required( + 516, + "min_sequence_number", + Types.LongType.get(), + "Minimum sequence number of files in this manifest"); + Types.NestedField INLINE_DV = + Types.NestedField.optional( + 162, + "inline_dv", + Types.BinaryType.get(), + "Inline deletion vector for marking manifest entries as deleted without rewriting"); + Types.NestedField INLINE_DV_CARDINALITY = + Types.NestedField.optional( + 163, + "inline_dv_cardinality", + Types.LongType.get(), + "Number of entries marked as deleted in the inline DV"); + + static Types.StructType schema() { + return Types.StructType.of( + ADDED_FILES_COUNT, + EXISTING_FILES_COUNT, + DELETED_FILES_COUNT, + REPLACED_FILES_COUNT, + ADDED_ROWS_COUNT, + EXISTING_ROWS_COUNT, + DELETED_ROWS_COUNT, + REPLACED_ROWS_COUNT, + MIN_SEQUENCE_NUMBER, + INLINE_DV, + INLINE_DV_CARDINALITY); + } + + /** Returns the number of files added by this manifest. */ + int addedFilesCount(); + + /** Returns the number of existing files referenced by this manifest. */ + int existingFilesCount(); + + /** Returns the number of deleted files in this manifest. */ + int deletedFilesCount(); + + /** Returns the number of replaced files in this manifest. */ + int replacedFilesCount(); + + /** Returns the number of rows in added files. */ + long addedRowsCount(); + + /** Returns the number of rows in existing files. */ + long existingRowsCount(); + + /** Returns the number of rows in deleted files. */ + long deletedRowsCount(); + + /** Returns the number of rows in replaced files. */ + long replacedRowsCount(); + + /** Returns the minimum sequence number of files in this manifest. */ + long minSequenceNumber(); + + /** Returns the inline deletion vector bitmap, or null if not present. */ + ByteBuffer inlineDV(); + + /** Returns the number of entries marked as deleted in the inline DV, or null if not present. */ + Long inlineDVCardinality(); Review Comment: Minor since we can always change it but have we already considered just calling this dv() and dvCardinality()? We're already in ManifestInfo and we know it's inline? ########## api/src/main/java/org/apache/iceberg/FileContent.java: ########## @@ -18,11 +18,13 @@ */ package org.apache.iceberg; -/** Content type stored in a file, one of DATA, POSITION_DELETES, or EQUALITY_DELETES. */ +/** Content type stored in a file. */ public enum FileContent { DATA(0), POSITION_DELETES(1), - EQUALITY_DELETES(2); + EQUALITY_DELETES(2), + DATA_MANIFEST(3), + DELETE_MANIFEST(4); Review Comment: Though I agree with the changes, this is the one set of changes that's in the public API. One option is we hold off on these changes until 1.11, and then that gives more time. The other option is we go ahead but there's a risk (although super unlikely imo) of having to go through deprecation cycle in case we want to remove something. Personally I'm good with moving forward as it is cc @rdblue @nastra in case they have any thoughts here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
