zhedoubushishi commented on a change in pull request #2833: URL: https://github.com/apache/hudi/pull/2833#discussion_r641742577
########## File path: hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigOption.java ########## @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.config; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class TestConfigOption { Review comment: Good point. Done ########## File path: hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java ########## @@ -50,52 +52,95 @@ * @see HoodieTableMetaClient * @since 0.3.0 */ -public class HoodieTableConfig implements Serializable { +public class HoodieTableConfig extends HoodieConfig implements Serializable { private static final Logger LOG = LogManager.getLogger(HoodieTableConfig.class); public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; - public static final String HOODIE_TABLE_NAME_PROP_NAME = "hoodie.table.name"; - public static final String HOODIE_TABLE_TYPE_PROP_NAME = "hoodie.table.type"; - public static final String HOODIE_TABLE_VERSION_PROP_NAME = "hoodie.table.version"; - public static final String HOODIE_TABLE_PRECOMBINE_FIELD = "hoodie.table.precombine.field"; - public static final String HOODIE_TABLE_PARTITION_COLUMNS = "hoodie.table.partition.columns"; - - @Deprecated - public static final String HOODIE_RO_FILE_FORMAT_PROP_NAME = "hoodie.table.ro.file.format"; - @Deprecated - public static final String HOODIE_RT_FILE_FORMAT_PROP_NAME = "hoodie.table.rt.file.format"; - public static final String HOODIE_BASE_FILE_FORMAT_PROP_NAME = "hoodie.table.base.file.format"; - public static final String HOODIE_LOG_FILE_FORMAT_PROP_NAME = "hoodie.table.log.file.format"; - public static final String HOODIE_TIMELINE_LAYOUT_VERSION = "hoodie.timeline.layout.version"; - public static final String HOODIE_PAYLOAD_CLASS_PROP_NAME = "hoodie.compaction.payload.class"; - public static final String HOODIE_ARCHIVELOG_FOLDER_PROP_NAME = "hoodie.archivelog.folder"; - public static final String HOODIE_BOOTSTRAP_INDEX_ENABLE = "hoodie.bootstrap.index.enable"; - public static final String HOODIE_BOOTSTRAP_INDEX_CLASS_PROP_NAME = "hoodie.bootstrap.index.class"; - public static final String HOODIE_BOOTSTRAP_BASE_PATH = "hoodie.bootstrap.base.path"; - - public static final HoodieTableType DEFAULT_TABLE_TYPE = HoodieTableType.COPY_ON_WRITE; - public static final HoodieTableVersion DEFAULT_TABLE_VERSION = HoodieTableVersion.ZERO; - public static final HoodieFileFormat DEFAULT_BASE_FILE_FORMAT = HoodieFileFormat.PARQUET; - public static final HoodieFileFormat DEFAULT_LOG_FILE_FORMAT = HoodieFileFormat.HOODIE_LOG; - public static final String DEFAULT_PAYLOAD_CLASS = OverwriteWithLatestAvroPayload.class.getName(); - public static final String NO_OP_BOOTSTRAP_INDEX_CLASS = NoOpBootstrapIndex.class.getName(); - public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = HFileBootstrapIndex.class.getName(); - public static final String DEFAULT_ARCHIVELOG_FOLDER = ""; - private Properties props; + public static final ConfigOption<String> HOODIE_TABLE_NAME_PROP_NAME = ConfigOption + .key("hoodie.table.name") + .noDefaultValue() + .withDescription("Table name that will be used for registering with Hive. Needs to be same across runs."); + + public static final ConfigOption<HoodieTableType> HOODIE_TABLE_TYPE_PROP_NAME = ConfigOption + .key("hoodie.table.type") + .defaultValue(HoodieTableType.COPY_ON_WRITE) + .withDescription("The table type for the underlying data, for this write. This can’t change between writes."); + + public static final ConfigOption<HoodieTableVersion> HOODIE_TABLE_VERSION_PROP_NAME = ConfigOption + .key("hoodie.table.version") + .defaultValue(HoodieTableVersion.ZERO) + .withDescription(""); + + public static final ConfigOption<String> HOODIE_TABLE_PRECOMBINE_FIELD = ConfigOption + .key("hoodie.table.precombine.field") + .noDefaultValue() + .withDescription("Field used in preCombining before actual write. When two records have the same key value, " + + "we will pick the one with the largest value for the precombine field, determined by Object.compareTo(..)"); + + public static final ConfigOption<String> HOODIE_TABLE_PARTITION_COLUMNS = ConfigOption + .key("hoodie.table.partition.columns") + .noDefaultValue() + .withDescription("Partition path field. Value to be used at the partitionPath component of HoodieKey. " + + "Actual value ontained by invoking .toString()"); + + public static final ConfigOption<HoodieFileFormat> HOODIE_BASE_FILE_FORMAT_PROP_NAME = ConfigOption + .key("hoodie.table.base.file.format") + .defaultValue(HoodieFileFormat.PARQUET) + .withAlternatives("hoodie.table.ro.file.format") + .withDescription(""); + + public static final ConfigOption<HoodieFileFormat> HOODIE_LOG_FILE_FORMAT_PROP_NAME = ConfigOption + .key("hoodie.table.log.file.format") + .defaultValue(HoodieFileFormat.HOODIE_LOG) + .withAlternatives("hoodie.table.rt.file.format") + .withDescription(""); + + public static final ConfigOption<String> HOODIE_TIMELINE_LAYOUT_VERSION = ConfigOption + .key("hoodie.timeline.layout.version") + .noDefaultValue() + .withDescription(""); Review comment: Great! lots of configurations missing description, would be great to have those. ########## File path: hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java ########## @@ -50,52 +52,95 @@ * @see HoodieTableMetaClient * @since 0.3.0 */ -public class HoodieTableConfig implements Serializable { +public class HoodieTableConfig extends HoodieConfig implements Serializable { private static final Logger LOG = LogManager.getLogger(HoodieTableConfig.class); public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; - public static final String HOODIE_TABLE_NAME_PROP_NAME = "hoodie.table.name"; - public static final String HOODIE_TABLE_TYPE_PROP_NAME = "hoodie.table.type"; - public static final String HOODIE_TABLE_VERSION_PROP_NAME = "hoodie.table.version"; - public static final String HOODIE_TABLE_PRECOMBINE_FIELD = "hoodie.table.precombine.field"; - public static final String HOODIE_TABLE_PARTITION_COLUMNS = "hoodie.table.partition.columns"; - - @Deprecated - public static final String HOODIE_RO_FILE_FORMAT_PROP_NAME = "hoodie.table.ro.file.format"; - @Deprecated - public static final String HOODIE_RT_FILE_FORMAT_PROP_NAME = "hoodie.table.rt.file.format"; - public static final String HOODIE_BASE_FILE_FORMAT_PROP_NAME = "hoodie.table.base.file.format"; - public static final String HOODIE_LOG_FILE_FORMAT_PROP_NAME = "hoodie.table.log.file.format"; - public static final String HOODIE_TIMELINE_LAYOUT_VERSION = "hoodie.timeline.layout.version"; - public static final String HOODIE_PAYLOAD_CLASS_PROP_NAME = "hoodie.compaction.payload.class"; - public static final String HOODIE_ARCHIVELOG_FOLDER_PROP_NAME = "hoodie.archivelog.folder"; - public static final String HOODIE_BOOTSTRAP_INDEX_ENABLE = "hoodie.bootstrap.index.enable"; - public static final String HOODIE_BOOTSTRAP_INDEX_CLASS_PROP_NAME = "hoodie.bootstrap.index.class"; - public static final String HOODIE_BOOTSTRAP_BASE_PATH = "hoodie.bootstrap.base.path"; - - public static final HoodieTableType DEFAULT_TABLE_TYPE = HoodieTableType.COPY_ON_WRITE; - public static final HoodieTableVersion DEFAULT_TABLE_VERSION = HoodieTableVersion.ZERO; - public static final HoodieFileFormat DEFAULT_BASE_FILE_FORMAT = HoodieFileFormat.PARQUET; - public static final HoodieFileFormat DEFAULT_LOG_FILE_FORMAT = HoodieFileFormat.HOODIE_LOG; - public static final String DEFAULT_PAYLOAD_CLASS = OverwriteWithLatestAvroPayload.class.getName(); - public static final String NO_OP_BOOTSTRAP_INDEX_CLASS = NoOpBootstrapIndex.class.getName(); - public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = HFileBootstrapIndex.class.getName(); - public static final String DEFAULT_ARCHIVELOG_FOLDER = ""; - private Properties props; + public static final ConfigOption<String> HOODIE_TABLE_NAME_PROP_NAME = ConfigOption + .key("hoodie.table.name") + .noDefaultValue() + .withDescription("Table name that will be used for registering with Hive. Needs to be same across runs."); + + public static final ConfigOption<HoodieTableType> HOODIE_TABLE_TYPE_PROP_NAME = ConfigOption + .key("hoodie.table.type") + .defaultValue(HoodieTableType.COPY_ON_WRITE) + .withDescription("The table type for the underlying data, for this write. This can’t change between writes."); + + public static final ConfigOption<HoodieTableVersion> HOODIE_TABLE_VERSION_PROP_NAME = ConfigOption + .key("hoodie.table.version") + .defaultValue(HoodieTableVersion.ZERO) + .withDescription(""); + + public static final ConfigOption<String> HOODIE_TABLE_PRECOMBINE_FIELD = ConfigOption + .key("hoodie.table.precombine.field") + .noDefaultValue() + .withDescription("Field used in preCombining before actual write. When two records have the same key value, " + + "we will pick the one with the largest value for the precombine field, determined by Object.compareTo(..)"); + + public static final ConfigOption<String> HOODIE_TABLE_PARTITION_COLUMNS = ConfigOption + .key("hoodie.table.partition.columns") + .noDefaultValue() + .withDescription("Partition path field. Value to be used at the partitionPath component of HoodieKey. " + + "Actual value ontained by invoking .toString()"); + + public static final ConfigOption<HoodieFileFormat> HOODIE_BASE_FILE_FORMAT_PROP_NAME = ConfigOption + .key("hoodie.table.base.file.format") + .defaultValue(HoodieFileFormat.PARQUET) + .withAlternatives("hoodie.table.ro.file.format") + .withDescription(""); + + public static final ConfigOption<HoodieFileFormat> HOODIE_LOG_FILE_FORMAT_PROP_NAME = ConfigOption Review comment: Done. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
