saintstack commented on a change in pull request #2504:
URL: https://github.com/apache/hbase/pull/2504#discussion_r501849460
##########
File path:
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
##########
@@ -301,40 +303,56 @@ public WALKey getCurrentKey() throws IOException,
InterruptedException {
inpDirs.split(conf.get(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ",")));
}
+ /**
+ * @param startTime If file looks like it has a timestamp in its name, we'll
check if newer
+ * or equal to this value else we will filter out the file.
If name does not
+ * seem to have a timestamp, we will just return it w/o
filtering.
+ * @param endTime If file looks like it has a timestamp in its name, we'll
check if older or equal
+ * to this value else we will filter out the file. If name
does not seem to
+ * have a timestamp, we will just return it w/o filtering.
+ */
private List<FileStatus> getFiles(FileSystem fs, Path dir, long startTime,
long endTime)
throws IOException {
List<FileStatus> result = new ArrayList<>();
LOG.debug("Scanning " + dir.toString() + " for WAL files");
-
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(dir);
- if (!iter.hasNext()) return Collections.emptyList();
+ if (!iter.hasNext()) {
+ return Collections.emptyList();
+ }
while (iter.hasNext()) {
LocatedFileStatus file = iter.next();
if (file.isDirectory()) {
- // recurse into sub directories
+ // Recurse into sub directories
result.addAll(getFiles(fs, file.getPath(), startTime, endTime));
} else {
- String name = file.getPath().toString();
- int idx = name.lastIndexOf('.');
- if (idx > 0) {
- try {
- long fileStartTime = Long.parseLong(name.substring(idx+1));
- if (fileStartTime <= endTime) {
- LOG.info("Found: " + file);
- result.add(file);
- }
- } catch (NumberFormatException x) {
- idx = 0;
- }
- }
- if (idx == 0) {
- LOG.warn("File " + name + " does not appear to be an WAL file.
Skipping...");
- }
+ addFile(result, file, startTime, endTime);
}
}
+ // TODO: These results should be sorted? Results could be content of
recovered.edits directory
+ // -- null padded increasing numeric -- or a WAL file w/ timestamp suffix
or timestamp and
+ // then meta suffix. See AbstractFSWALProvider#WALStartTimeComparator
return result;
}
+ static void addFile(List<FileStatus> result, LocatedFileStatus lfs, long
startTime,
+ long endTime) {
+ long timestamp = WAL.getTimestamp(lfs.getPath().getName());
+ if (timestamp > 0) {
+ // Looks like a valid timestamp.
+ if (timestamp <= endTime && timestamp >= startTime) {
+ LOG.info("Found {}", lfs.getPath());
+ result.add(lfs);
+ } else {
+ LOG.debug("Skipped {}, outside range [{}/{} - {}/{}]", lfs.getPath(),
Review comment:
Made it info.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
