This is an automated email from the ASF dual-hosted git repository.
leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 062d5ba [HUDI-2013] Removed option to fallback to file listing when
Metadata Table is enabled. (#3079)
062d5ba is described below
commit 062d5baf842c22a6cf3b981ad605cafb6be3f130
Author: Prashant Wason <[email protected]>
AuthorDate: Tue Jun 22 08:41:52 2021 -0700
[HUDI-2013] Removed option to fallback to file listing when Metadata Table
is enabled. (#3079)
---
.../apache/hudi/metadata/TestHoodieBackedMetadata.java | 3 +--
.../apache/hudi/common/config/HoodieMetadataConfig.java | 15 ---------------
.../java/org/apache/hudi/metadata/BaseTableMetadata.java | 13 +++----------
3 files changed, 4 insertions(+), 27 deletions(-)
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
index f34f567..ab731f5 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
@@ -969,8 +969,7 @@ public class TestHoodieBackedMetadata extends
HoodieClientTestHarness {
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(useFileListingMetadata)
- .enableMetrics(enableMetrics)
- .enableFallback(false).build())
+ .enableMetrics(enableMetrics).build())
.withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics)
.withExecutorMetrics(true).usePrefix("unit-test").build());
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index 1eff82f..3a6b4f3 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -67,10 +67,6 @@ public final class HoodieMetadataConfig extends
DefaultHoodieConfig {
public static final String CLEANER_COMMITS_RETAINED_PROP = METADATA_PREFIX +
".cleaner.commits.retained";
public static final int DEFAULT_CLEANER_COMMITS_RETAINED = 3;
- // Controls whether or not, upon failure to fetch from metadata table,
should fallback to listing.
- public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX +
".fallback.enable";
- public static final String DEFAULT_ENABLE_FALLBACK = "true";
-
// Regex to filter out matching directories during bootstrap
public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX +
".dir.filter.regex";
public static final String DEFAULT_DIRECTORY_FILTER_REGEX = "";
@@ -101,10 +97,6 @@ public final class HoodieMetadataConfig extends
DefaultHoodieConfig {
return Boolean.parseBoolean(props.getProperty(METADATA_ENABLE_PROP));
}
- public boolean enableFallback() {
- return Boolean.parseBoolean(props.getProperty(ENABLE_FALLBACK_PROP));
- }
-
public boolean validateFileListingMetadata() {
return Boolean.parseBoolean(props.getProperty(METADATA_VALIDATE_PROP));
}
@@ -143,11 +135,6 @@ public final class HoodieMetadataConfig extends
DefaultHoodieConfig {
return this;
}
- public Builder enableFallback(boolean fallback) {
- props.setProperty(ENABLE_FALLBACK_PROP, String.valueOf(fallback));
- return this;
- }
-
public Builder validate(boolean validate) {
props.setProperty(METADATA_VALIDATE_PROP, String.valueOf(validate));
return this;
@@ -218,8 +205,6 @@ public final class HoodieMetadataConfig extends
DefaultHoodieConfig {
String.valueOf(DEFAULT_FILE_LISTING_PARALLELISM));
setDefaultOnCondition(props,
!props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP),
HOODIE_ASSUME_DATE_PARTITIONING_PROP,
DEFAULT_ASSUME_DATE_PARTITIONING);
- setDefaultOnCondition(props, !props.containsKey(ENABLE_FALLBACK_PROP),
ENABLE_FALLBACK_PROP,
- DEFAULT_ENABLE_FALLBACK);
setDefaultOnCondition(props, !props.containsKey(DIRECTORY_FILTER_REGEX),
DIRECTORY_FILTER_REGEX,
DEFAULT_DIRECTORY_FILTER_REGEX);
return config;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index b4143f3..0092853 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -101,11 +101,7 @@ public abstract class BaseTableMetadata implements
HoodieTableMetadata {
try {
return fetchAllPartitionPaths();
} catch (Exception e) {
- if (metadataConfig.enableFallback()) {
- LOG.error("Failed to retrieve list of partition from metadata", e);
- } else {
- throw new HoodieMetadataException("Failed to retrieve list of
partition from metadata", e);
- }
+ throw new HoodieMetadataException("Failed to retrieve list of
partition from metadata", e);
}
}
return new FileSystemBackedTableMetadata(getEngineContext(), hadoopConf,
datasetBasePath,
@@ -129,11 +125,7 @@ public abstract class BaseTableMetadata implements
HoodieTableMetadata {
try {
return fetchAllFilesInPartition(partitionPath);
} catch (Exception e) {
- if (metadataConfig.enableFallback()) {
- LOG.error("Failed to retrieve files in partition " + partitionPath +
" from metadata", e);
- } else {
- throw new HoodieMetadataException("Failed to retrieve files in
partition " + partitionPath + " from metadata", e);
- }
+ throw new HoodieMetadataException("Failed to retrieve files in
partition " + partitionPath + " from metadata", e);
}
}
@@ -293,6 +285,7 @@ public abstract class BaseTableMetadata implements
HoodieTableMetadata {
protected abstract List<HoodieInstant> findInstantsToSync();
+ @Override
public boolean isInSync() {
return enabled && findInstantsToSync().isEmpty();
}
