the-other-tim-brown commented on code in PR #772: URL: https://github.com/apache/incubator-xtable/pull/772#discussion_r3368141963
########## xtable-core/src/main/java/org/apache/hudi/stats/XTableValueMetadata.java: ########## @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.stats; + +import static org.apache.xtable.model.schema.InternalSchema.MetadataKey.TIMESTAMP_PRECISION; +import static org.apache.xtable.model.schema.InternalSchema.MetadataValue.MICROS; + +import java.lang.reflect.Constructor; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; + +import org.apache.hudi.metadata.HoodieIndexVersion; + +import org.apache.xtable.model.schema.InternalSchema; +import org.apache.xtable.model.stat.ColumnStat; + +/** + * Utility class for creating and converting Hudi {@link ValueMetadata} instances from XTable's + * internal schema representation. + * + * <p>This class bridges XTable's {@link InternalSchema} types to Hudi's {@link ValueType} and + * {@link ValueMetadata} used for column statistics. It handles the conversion of various data types + * including timestamps, decimals, and dates. + * + * <p>Note: This class uses reflection to create {@link ValueMetadata} instances because XTable + * classes may be loaded by a different classloader than Hudi classes in Spark environments, making + * direct constructor access illegal. + */ +public class XTableValueMetadata { + + /** + * Creates a {@link ValueMetadata} instance from a {@link ColumnStat} for the specified Hudi index + * version. + * + * @param columnStat the column statistics containing schema information + * @param indexVersion the Hudi index version to use for metadata creation + * @return the appropriate {@link ValueMetadata} for the column's data type + * @throws IllegalArgumentException if columnStat is null (for V2+ index), or if decimal metadata + * is missing required precision/scale + * @throws IllegalStateException if an unsupported internal type is encountered + */ + public static ValueMetadata getValueMetadata( + ColumnStat columnStat, HoodieIndexVersion indexVersion) { + if (indexVersion.lowerThan(HoodieIndexVersion.V2)) { + return ValueMetadata.V1EmptyMetadata.get(); + } + if (columnStat == null) { + throw new IllegalArgumentException("ColumnStat cannot be null"); + } + InternalSchema internalSchema = columnStat.getField().getSchema(); + ValueType valueType = fromInternalSchema(internalSchema); + if (valueType == ValueType.V1) { + throw new IllegalStateException( + "InternalType V1 should not be returned from fromInternalSchema"); + } else if (valueType == ValueType.DECIMAL) { + if (internalSchema.getMetadata() == null) { + throw new IllegalArgumentException("Decimal metadata is null"); + } else if (!internalSchema + .getMetadata() + .containsKey(InternalSchema.MetadataKey.DECIMAL_SCALE)) { + throw new IllegalArgumentException("Decimal scale is null"); + } else if (!internalSchema + .getMetadata() + .containsKey(InternalSchema.MetadataKey.DECIMAL_PRECISION)) { + throw new IllegalArgumentException("Decimal precision is null"); + } + int scale = (int) internalSchema.getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE); + int precision = + (int) internalSchema.getMetadata().get(InternalSchema.MetadataKey.DECIMAL_PRECISION); + return ValueMetadata.DecimalMetadata.create(precision, scale); + } else { + return createValueMetadata(valueType); + } + } + + /** + * Maps an XTable {@link InternalSchema} to the corresponding Hudi {@link ValueType}. + * + * @param internalSchema the internal schema to convert + * @return the corresponding Hudi value type + * @throws UnsupportedOperationException if the internal data type is not supported + */ + static ValueType fromInternalSchema(InternalSchema internalSchema) { + switch (internalSchema.getDataType()) { + case NULL: + return ValueType.NULL; + case BOOLEAN: + return ValueType.BOOLEAN; + case INT: + return ValueType.INT; + case LONG: + return ValueType.LONG; + case FLOAT: + return ValueType.FLOAT; + case DOUBLE: + return ValueType.DOUBLE; + case STRING: Review Comment: Should we add `ENUM` here? ########## xtable-hudi-support/xtable-hudi-support-utils/src/main/java/org/apache/xtable/hudi/idtracking/IdTracker.java: ########## @@ -122,12 +124,63 @@ public IdTracking getIdTracking( AtomicInteger currentId = new AtomicInteger(existingState.getLastIdUsed()); // add meta fields to the schema in order to ensure they will be assigned IDs Schema schemaForIdMapping = - includeMetaFields ? HoodieAvroUtils.addMetadataFields(schema) : schema; - List<IdMapping> newMappings = - generateIdMappings(schemaForIdMapping, currentId, existingState.getIdMappings()); - return new IdTracking(newMappings, currentId.get()); + includeMetaFields + ? HoodieSchemaUtils.addMetadataFields(HoodieSchema.fromAvroSchema(schema)) + .toAvroSchema() + : schema; + if (currentId.intValue() != 0) { + // if the schema has ID tracking, update the existing mappings. + List<IdMapping> updatedMappings = + updateIdMappings(schemaForIdMapping, currentId, existingState.getIdMappings()); + return new IdTracking(updatedMappings, currentId.get()); + } else { + // if the schema does not have ID tracking, generate new mappings. + List<IdMapping> newMappings = + generateIdMappings(schemaForIdMapping, currentId, existingState.getIdMappings()); + return new IdTracking(newMappings, currentId.get()); + } } + /** + * Updates the IdMappings in the provided schema. For all newly added columns, we process column + * by column for new id assignment. + * + * <p>Different from generateIdMappings which traverse the entire schema tree, this method + * traverse individual columns and update the id mappings. Review Comment: This method will call generateIdMappings which will traverse so I am not sure this comment is accurate ########## xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSource.java: ########## @@ -237,10 +239,21 @@ private HoodieTimeline getCompletedCommits() { } private CommitsPair getCompletedAndPendingCommitsAfterInstant(HoodieInstant commitInstant) { + // Get all instants after the provided commitInstant and instants that are yet to be completed. + // We use completionTime (not requestedTime) for filtering completed commits because we need + // commits that actually finished after the last sync, not just ones that were initiated after. + // This handles out-of-order completion scenarios where a later-requested commit may complete + // before an earlier-requested one. List<HoodieInstant> allInstants = metaClient .getActiveTimeline() - .findInstantsAfter(commitInstant.getTimestamp()) + .filter( + hoodieInstant -> + !hoodieInstant.isCompleted() + || InstantComparison.compareTimestamps( Review Comment: Lets make sure the tests are updated so we can properly test this switch to completion time -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
