zhedoubushishi commented on a change in pull request #3416:
URL: https://github.com/apache/hudi/pull/3416#discussion_r751692944
##########
File path:
hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
##########
@@ -43,72 +47,110 @@
private static final Logger LOG =
LogManager.getLogger(DFSPropertiesConfiguration.class);
+ private static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf";
+
+ public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR";
+
+ public static final String DEFAULT_CONF_FILE_DIR = "file:/etc/hudi/conf";
+
+ // props read from hudi-defaults.conf
+ private static TypedProperties GLOBAL_PROPS = loadGlobalProps();
+
private final FileSystem fs;
- private final Path rootFile;
+ private Path currentFilePath;
- private final TypedProperties props;
+ // props read from user defined configuration file or input stream
+ private final HoodieConfig hoodieConfig;
// Keep track of files visited, to detect loops
- private final Set<String> visitedFiles;
+ private final Set<String> visitedFilePaths;
- public DFSPropertiesConfiguration(FileSystem fs, Path rootFile,
TypedProperties defaults) {
+ public DFSPropertiesConfiguration(FileSystem fs, Path filePath) {
this.fs = fs;
- this.rootFile = rootFile;
- this.props = defaults;
- this.visitedFiles = new HashSet<>();
- visitFile(rootFile);
- }
-
- public DFSPropertiesConfiguration(FileSystem fs, Path rootFile) {
- this(fs, rootFile, new TypedProperties());
+ this.currentFilePath = filePath;
+ this.hoodieConfig = new HoodieConfig();
+ this.visitedFilePaths = new HashSet<>();
+ addPropsFromFile(filePath);
}
public DFSPropertiesConfiguration() {
this.fs = null;
- this.rootFile = null;
- this.props = new TypedProperties();
- this.visitedFiles = new HashSet<>();
+ this.currentFilePath = null;
+ this.hoodieConfig = new HoodieConfig();
+ this.visitedFilePaths = new HashSet<>();
}
- private String[] splitProperty(String line) {
- int ind = line.indexOf('=');
- String k = line.substring(0, ind).trim();
- String v = line.substring(ind + 1).trim();
- return new String[] {k, v};
+ /**
+ * Load global props from hudi-defaults.conf which is under
CONF_FILE_DIR_ENV_NAME.
+ * @return Typed Properties
+ */
+ public static TypedProperties loadGlobalProps() {
+ DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration();
+ Option<Path> defaultConfPath = getConfPathFromEnv();
+ if (defaultConfPath.isPresent()) {
+ conf.addPropsFromFile(defaultConfPath.get());
+ } else {
+ try {
+ conf.addPropsFromFile(new Path(DEFAULT_CONF_FILE_DIR));
+ } catch (Exception ignored) {
+ LOG.debug("Didn't find config file under default conf file dir: " +
DEFAULT_CONF_FILE_DIR);
+ }
+ }
+ return conf.getProps();
+ }
+
+ public static void refreshGlobalProps() {
+ GLOBAL_PROPS = loadGlobalProps();
+ }
+
+ public static void clearGlobalProps() {
+ GLOBAL_PROPS = new TypedProperties();
}
- private void visitFile(Path file) {
+ /**
+ * Add properties from external configuration files.
+ *
+ * @param filePath File path for configuration file
+ */
+ public void addPropsFromFile(Path filePath) {
+ if (visitedFilePaths.contains(filePath.toString())) {
+ throw new IllegalStateException("Loop detected; file " + filePath + "
already referenced");
+ }
+ FileSystem fileSystem;
try {
- if (visitedFiles.contains(file.getName())) {
- throw new IllegalStateException("Loop detected; file " + file + "
already referenced");
- }
- visitedFiles.add(file.getName());
- BufferedReader reader = new BufferedReader(new
InputStreamReader(fs.open(file)));
- addProperties(reader);
+ fileSystem = fs != null ? fs : filePath.getFileSystem(new
Configuration());
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Cannot get the file system from file
path", e);
+ }
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(fileSystem.open(filePath)))) {
Review comment:
When ```fs``` exists, if I put ```FileSystem fs = ...``` into the
try-with-resources clause, it would close the class variable ```fs```, which
causes fileSystem is closed exception in the runtime
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
