This is an automated email from the ASF dual-hosted git repository. jlli pushed a commit to branch handle-delete-aged-segments in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit ea1412c887f16d404c8a7e23bac02548ee8669ba Author: Jack Li(Analytics Engineering) <[email protected]> AuthorDate: Fri Jan 11 14:03:40 2019 -0800 Add touch method in PinotFS; Call touch when moving deleted segments. --- .../pinot/controller/helix/core/SegmentDeletionManager.java | 4 +++- .../main/java/org/apache/pinot/filesystem/LocalPinotFS.java | 9 +++++++++ .../src/main/java/org/apache/pinot/filesystem/PinotFS.java | 8 +++++++- .../java/org/apache/pinot/filesystem/HadoopPinotFS.java | 13 +++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java index f405e68..5554da6 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java @@ -185,6 +185,8 @@ public class SegmentDeletionManager { if (pinotFS.exists(fileToMoveURI)) { // Overwrites the file if it already exists in the target directory. pinotFS.move(fileToMoveURI, deletedSegmentDestURI, true); + // Updates last modified + pinotFS.touch(deletedSegmentDestURI); LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToMoveURI.toString(), deletedSegmentDestURI.toString()); } else { if (!SegmentName.isHighLevelConsumerSegmentName(segmentId)) { @@ -248,7 +250,7 @@ public class SegmentDeletionManager { } } } catch (IOException e) { - LOGGER.error("Had trouble deleting directories", deletedDirURI.toString()); + LOGGER.error("Had trouble deleting directories: {}", deletedDirURI.toString(), e.toString()); } } else { LOGGER.info("dataDir is not configured, won't delete any expired segments from deleted directory."); diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java index 987125f..e060d88 100644 --- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java +++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java @@ -161,6 +161,15 @@ public class LocalPinotFS extends PinotFS { return file.lastModified(); } + @Override + public boolean touch(URI uri) throws IOException { + File file = new File(decodeURI(uri.getRawPath())); + if (!exists(uri)) { + return file.createNewFile(); + } + return file.setLastModified(System.currentTimeMillis()); + } + private String encodeURI(String uri) { String encodedStr; try { diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java index abcf494..4a5815a 100644 --- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java +++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java @@ -137,7 +137,7 @@ public abstract class PinotFS implements Closeable { /** * Returns the age of the file - * @param uri + * @param uri location of file or directory * @return A long value representing the time the file was last modified, measured in milliseconds since epoch * (00:00:00 GMT, January 1, 1970) or 0L if the file does not exist or if an I/O error occurs * @throws Exception if uri is not valid or present @@ -145,6 +145,12 @@ public abstract class PinotFS implements Closeable { public abstract long lastModified(URI uri); /** + * Creates a new empty file if it doesn't exist, or updates the last modified on existing file or directory. + * @param uri location of file or directory + */ + public abstract boolean touch(URI uri) throws IOException; + + /** * For certain filesystems, we may need to close the filesystem and do relevant operations to prevent leaks. * By default, this method does nothing. * @throws IOException diff --git a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java index c58a16a..fb64770 100644 --- a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java +++ b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java @@ -19,6 +19,7 @@ package org.apache.pinot.filesystem; import com.google.common.base.Strings; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.pinot.common.utils.retry.RetryPolicies; import org.apache.pinot.common.utils.retry.RetryPolicy; import java.io.File; @@ -191,6 +192,18 @@ public class HadoopPinotFS extends PinotFS { } } + @Override + public boolean touch(URI uri) throws IOException { + Path path = new Path(uri); + if (!exists(uri)) { + FSDataOutputStream fos = _hadoopFS.create(path); + fos.close(); + } else { + _hadoopFS.setTimes(path, System.currentTimeMillis(), -1); + } + return true; + } + private void authenticate(org.apache.hadoop.conf.Configuration hadoopConf, org.apache.commons.configuration.Configuration configs) { String principal = configs.getString(PRINCIPAL); String keytab = configs.getString(KEYTAB); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
