nsivabalan commented on code in PR #8609:
URL: https://github.com/apache/hudi/pull/8609#discussion_r1186572588
##########
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java:
##########
@@ -323,22 +326,43 @@ public HoodieTableConfig() {
super();
}
- private void fetchConfigs(FileSystem fs, String metaPath) throws IOException
{
+ private static TypedProperties fetchConfigs(FileSystem fs, String metaPath)
throws IOException {
Path cfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE);
- try (FSDataInputStream is = fs.open(cfgPath)) {
- props.load(is);
- } catch (IOException ioe) {
- if (!fs.exists(cfgPath)) {
- LOG.warn("Run `table recover-configs` if config update/delete failed
midway. Falling back to backed up configs.");
- // try the backup. this way no query ever fails if update fails midway.
- Path backupCfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE_BACKUP);
- try (FSDataInputStream is = fs.open(backupCfgPath)) {
+ Path backupCfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE_BACKUP);
+ int readRetryCount = 0;
+ boolean found = false;
+
+ TypedProperties props = new TypedProperties();
+ while (readRetryCount++ < MAX_READ_RETRIES) {
+ for (Path path : Arrays.asList(cfgPath, backupCfgPath)) {
+ // Read the properties and validate that it is a valid file
+ try (FSDataInputStream is = fs.open(path)) {
+ props.clear();
props.load(is);
+ found = true;
+ ValidationUtils.checkArgument(props.containsKey(TYPE) &&
props.containsKey(NAME));
Review Comment:
we can validate using checksum. we already have a method
```
validateChecksum()
```
##########
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java:
##########
@@ -323,22 +326,43 @@ public HoodieTableConfig() {
super();
}
- private void fetchConfigs(FileSystem fs, String metaPath) throws IOException
{
+ private static TypedProperties fetchConfigs(FileSystem fs, String metaPath)
throws IOException {
Path cfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE);
- try (FSDataInputStream is = fs.open(cfgPath)) {
- props.load(is);
- } catch (IOException ioe) {
- if (!fs.exists(cfgPath)) {
- LOG.warn("Run `table recover-configs` if config update/delete failed
midway. Falling back to backed up configs.");
- // try the backup. this way no query ever fails if update fails midway.
- Path backupCfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE_BACKUP);
- try (FSDataInputStream is = fs.open(backupCfgPath)) {
+ Path backupCfgPath = new Path(metaPath, HOODIE_PROPERTIES_FILE_BACKUP);
+ int readRetryCount = 0;
+ boolean found = false;
+
+ TypedProperties props = new TypedProperties();
+ while (readRetryCount++ < MAX_READ_RETRIES) {
+ for (Path path : Arrays.asList(cfgPath, backupCfgPath)) {
+ // Read the properties and validate that it is a valid file
+ try (FSDataInputStream is = fs.open(path)) {
+ props.clear();
props.load(is);
+ found = true;
+ ValidationUtils.checkArgument(props.containsKey(TYPE) &&
props.containsKey(NAME));
+ return props;
+ } catch (IOException e) {
+ LOG.warn(String.format("Could not read properties from %s: %s",
path, e));
+ } catch (IllegalArgumentException e) {
+ LOG.warn(String.format("Invalid properties file %s: %s", path,
props));
}
- } else {
- throw ioe;
+ }
+
+ // Failed to read all files so wait before retrying. This can happen in
cases of parallel updates to the properties.
+ try {
Review Comment:
is there any way we can add test for this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
