This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 46c2fdbdb88 [HUDI-6062] Fix irregular enum config (#8564)
46c2fdbdb88 is described below
commit 46c2fdbdb88ff2669c7924cde30c6afdfe7b2330
Author: Jon Vexler <[email protected]>
AuthorDate: Sun Sep 24 14:46:46 2023 -0400
[HUDI-6062] Fix irregular enum config (#8564)
Co-authored-by: Jonathan Vexler <=>
Co-authored-by: Y Ethan Guo <[email protected]>
---
.../apache/hudi/config/HoodieClusteringConfig.java | 46 +++++++---------------
.../org/apache/hudi/config/HoodieWriteConfig.java | 3 +-
.../procedures/RunClusteringProcedure.scala | 6 +--
3 files changed, 18 insertions(+), 37 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
index 4d1756f4986..e8eea235168 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
@@ -26,19 +26,14 @@ import org.apache.hudi.common.config.EnumFieldDescription;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.EngineType;
-import org.apache.hudi.common.util.TypeUtils;
import org.apache.hudi.common.util.ValidationUtils;
-import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
-import javax.annotation.Nonnull;
-
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.util.Map;
import java.util.Properties;
/**
@@ -269,11 +264,10 @@ public class HoodieClusteringConfig extends HoodieConfig {
*/
public static final ConfigProperty<String> LAYOUT_OPTIMIZE_STRATEGY =
ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "strategy")
- .defaultValue("linear")
+ .defaultValue(LayoutOptimizationStrategy.LINEAR.name())
.markAdvanced()
.sinceVersion("0.10.0")
- .withDocumentation("Determines ordering strategy used in records layout
optimization. "
- + "Currently supported strategies are \"linear\", \"z-order\" and
\"hilbert\" values are supported.");
+ .withDocumentation(LayoutOptimizationStrategy.class);
/**
* NOTE: This setting only has effect if {@link #LAYOUT_OPTIMIZE_STRATEGY}
value is set to
@@ -693,7 +687,7 @@ public class HoodieClusteringConfig extends HoodieConfig {
}
/**
- * Type of a strategy for building Z-order/Hilbert space-filling curves.
+ * Type of strategy for building Z-order/Hilbert space-filling curves.
*/
@EnumDescription("This configuration only has effect if
`hoodie.layout.optimize.strategy` is "
+ "set to either \"z-order\" or \"hilbert\" (i.e. leveraging
space-filling curves). This "
@@ -723,32 +717,22 @@ public class HoodieClusteringConfig extends HoodieConfig {
/**
* Layout optimization strategies such as Z-order/Hilbert space-curves, etc
*/
+ @EnumDescription("Determines ordering strategy for records layout
optimization.")
public enum LayoutOptimizationStrategy {
- LINEAR("linear"),
- ZORDER("z-order"),
- HILBERT("hilbert");
-
- private static final Map<String, LayoutOptimizationStrategy>
VALUE_TO_ENUM_MAP =
- TypeUtils.getValueToEnumMap(LayoutOptimizationStrategy.class, e ->
e.value);
-
- private final String value;
-
- LayoutOptimizationStrategy(String value) {
- this.value = value;
- }
+ @EnumFieldDescription("Orders records lexicographically")
+ LINEAR,
- @Nonnull
- public static LayoutOptimizationStrategy fromValue(String value) {
- LayoutOptimizationStrategy enumValue = VALUE_TO_ENUM_MAP.get(value);
- if (enumValue == null) {
- throw new HoodieException(String.format("Invalid value (%s)", value));
- }
+ @EnumFieldDescription("Orders records along Z-order spatial-curve.")
+ ZORDER,
- return enumValue;
- }
+ @EnumFieldDescription("Orders records along Hilbert's spatial-curve.")
+ HILBERT
+ }
- public String getValue() {
- return value;
+ public static LayoutOptimizationStrategy
resolveLayoutOptimizationStrategy(String cfgVal) {
+ if (cfgVal.equalsIgnoreCase("z-order")) {
+ return LayoutOptimizationStrategy.ZORDER;
}
+ return LayoutOptimizationStrategy.valueOf(cfgVal.toUpperCase());
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 705fcd06daa..18473e36272 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -1736,8 +1736,7 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public HoodieClusteringConfig.LayoutOptimizationStrategy
getLayoutOptimizationStrategy() {
- return HoodieClusteringConfig.LayoutOptimizationStrategy.fromValue(
- getStringOrDefault(HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY));
+ return
HoodieClusteringConfig.resolveLayoutOptimizationStrategy(getStringOrDefault(HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY));
}
public HoodieClusteringConfig.SpatialCurveCompositionStrategyType
getLayoutOptimizationCurveBuildMethod() {
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
index 4394095d9a7..27f92027a02 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
@@ -23,11 +23,9 @@ import
org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieTimeli
import org.apache.hudi.common.table.{HoodieTableMetaClient,
TableSchemaResolver}
import org.apache.hudi.common.util.ValidationUtils.checkArgument
import org.apache.hudi.common.util.{ClusteringUtils, HoodieTimer, Option =>
HOption}
-import org.apache.hudi.config.HoodieClusteringConfig.LayoutOptimizationStrategy
import org.apache.hudi.config.{HoodieClusteringConfig, HoodieLockConfig}
import org.apache.hudi.exception.HoodieClusteringException
import org.apache.hudi.{AvroConversionUtils, HoodieCLIUtils, HoodieFileIndex}
-
import org.apache.spark.internal.Logging
import org.apache.spark.sql.HoodieCatalystExpressionUtils.{resolveExpr,
splitPartitionAndDataPredicates}
import org.apache.spark.sql.Row
@@ -125,9 +123,9 @@ class RunClusteringProcedure extends BaseProcedure
orderStrategy match {
case Some(o) =>
- val strategy =
LayoutOptimizationStrategy.fromValue(o.asInstanceOf[String])
+ val strategy =
HoodieClusteringConfig.resolveLayoutOptimizationStrategy(o.asInstanceOf[String])
confs = confs ++ Map(
- HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY.key() ->
strategy.getValue
+ HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY.key() ->
strategy.name()
)
case _ =>
logInfo("No order strategy")
