This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch release-0.10.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9d7f681ca6d3d8c244854040298569f54ba701b8 Author: Sivabalan Narayanan <[email protected]> AuthorDate: Wed Dec 1 03:02:30 2021 -0500 [HUDI-2880] Fixing loading of props from default dir (#4167) * Fixing loading of props from default dir * addressing comments (cherry picked from commit f4c25ba3fd11e2da1adcf2e089969a0b7f3dc62e) --- .../common/config/DFSPropertiesConfiguration.java | 31 ++++++++++++---------- .../util/TestDFSPropertiesConfiguration.java | 5 ++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java index 01585f9..afce8f6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java @@ -18,18 +18,21 @@ package org.apache.hudi.common.config; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -53,10 +56,9 @@ public class DFSPropertiesConfiguration { private static final Logger LOG = LogManager.getLogger(DFSPropertiesConfiguration.class); public static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf"; - public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR"; - public static final String DEFAULT_CONF_FILE_DIR = "file:/etc/hudi/conf"; + public static final Path DEFAULT_PATH = new Path(DEFAULT_CONF_FILE_DIR + "/" + DEFAULT_PROPERTIES_FILE); // props read from hudi-defaults.conf private static TypedProperties GLOBAL_PROPS = loadGlobalProps(); @@ -97,11 +99,7 @@ public class DFSPropertiesConfiguration { if (defaultConfPath.isPresent()) { conf.addPropsFromFile(defaultConfPath.get()); } else { - try { - conf.addPropsFromFile(new Path(DEFAULT_CONF_FILE_DIR)); - } catch (Exception ignored) { - LOG.warn("Didn't find config file under default conf file dir: " + DEFAULT_CONF_FILE_DIR); - } + conf.addPropsFromFile(DEFAULT_PATH); } return conf.getProps(); } @@ -128,14 +126,19 @@ public class DFSPropertiesConfiguration { filePath.toString(), Option.ofNullable(hadoopConfig).orElseGet(Configuration::new) ); + try { + if (filePath.equals(DEFAULT_PATH) && !fs.exists(filePath)) { + LOG.warn("Properties file " + filePath + " not found. Ignoring to load props file"); + return; + } - try (BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath)))) { + BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath))); visitedFilePaths.add(filePath.toString()); currentFilePath = filePath; addPropsFromStream(reader); } catch (IOException ioe) { - LOG.error("Error reading in properties from dfs"); - throw new IllegalArgumentException("Cannot read properties from dfs", ioe); + LOG.error("Error reading in properties from dfs from file " + filePath); + throw new HoodieIOException("Cannot read properties from dfs from file " + filePath, ioe); } } @@ -192,7 +195,7 @@ public class DFSPropertiesConfiguration { } private String[] splitProperty(String line) { - line = line.replaceAll("\\s+"," "); + line = line.replaceAll("\\s+", " "); String delimiter = line.contains("=") ? "=" : " "; int ind = line.indexOf(delimiter); String k = line.substring(0, ind).trim(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 6a7075f..4657393 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -145,7 +145,7 @@ public class TestDFSPropertiesConfiguration { } @Test - public void testLocalFileSystemLoading() { + public void testLocalFileSystemLoading() throws IOException { DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t1.props")); cfg.addPropsFromFile( @@ -156,8 +156,7 @@ public class TestDFSPropertiesConfiguration { .getResource("props/test.properties") .getPath() ) - ) - ); + )); TypedProperties props = cfg.getProps();
