voonhous commented on code in PR #19295: URL: https://github.com/apache/hudi/pull/19295#discussion_r3674414489
########## hudi-trino/src/test/java/io/trino/plugin/hudi/testing/MergeModeSemanticsHudiTablesInitializer.java: ########## @@ -0,0 +1,337 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Column; +import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.StorageFormat; +import io.trino.metastore.Table; +import io.trino.plugin.hudi.HudiConnector; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.QueryRunner; +import org.apache.avro.JsonProperties; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.client.HoodieJavaWriteClient; +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.client.common.HoodieJavaEngineContext; +import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.marker.MarkerType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.HUDI_PARQUET_REALTIME_INPUT_FORMAT; +import static io.trino.hive.formats.HiveClassNames.MAPRED_PARQUET_OUTPUT_FORMAT_CLASS; +import static io.trino.hive.formats.HiveClassNames.PARQUET_HIVE_SERDE_CLASS; +import static io.trino.metastore.HiveType.HIVE_BOOLEAN; +import static io.trino.metastore.HiveType.HIVE_LONG; +import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static java.nio.file.Files.createTempDirectory; +import static org.apache.hudi.common.model.HoodieRecord.HOODIE_IS_DELETED_FIELD; + +/** + * Creates two non-partitioned Merge-On-Read tables at test runtime that exercise the read-side + * MERGE-MODE dispatch with deletes (issue apache/hudi#18898). Both tables set ONLY a record merge mode + * (no payload class) so table creation persists the mode as-is: + * <ul> + * <li>{@code mor_deletes} ({@link RecordMergeMode#EVENT_TIME_ORDERING}): a base commit followed by a + * log commit with an update, a soft delete ({@code _hoodie_is_deleted=true}), an OBSOLETE soft + * delete and an OBSOLETE update (both with an ordering value LOWER than the base row's, so + * event-time merging must keep the base row), plus a hard-delete commit + * ({@code writeClient.delete}) that produces a native delete log file read back through the + * connector's {@code getFileRecordIterator}.</li> + * <li>{@code mor_commit_time} ({@link RecordMergeMode#COMMIT_TIME_ORDERING}): the same + * lower-ordering update shape, where latest-write-wins must KEEP the update -- the exact mirror + * of the event-time case, discriminating the two merger dispatches.</li> + * </ul> + * Records are wrapped in {@link HoodieAvroPayload}, which implements {@code HoodieRecordPayload} + * directly (NOT {@code BaseAvroPayload}), so rows with {@code _hoodie_is_deleted=true} are written as + * DATA records and delete semantics are evaluated at READ time. Inline compaction is disabled so log + * files survive. Each table registers a read-optimized and a {@code _rt} metastore table. + */ +public class MergeModeSemanticsHudiTablesInitializer + implements HudiTablesInitializer Review Comment: Done -- one initializer per table extending `AbstractMergerHudiTablesInitializer` (five subclasses), composed with `CompositeHudiTablesInitializer` in both suites. The hooks covered everything, including the hard-delete commits. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
