Apache9 commented on a change in pull request #4202:
URL: https://github.com/apache/hbase/pull/4202#discussion_r829710259
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
##########
@@ -114,23 +127,96 @@ private int select(StoreFileList[] lists) {
return lists[0].getTimestamp() >= lists[1].getTimestamp() ? 0 : 1;
}
- StoreFileList load() throws IOException {
+ // file sequence id to path
+ private NavigableMap<Long, List<Path>> listFiles() throws IOException {
+ FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
+ FileStatus[] statuses;
+ try {
+ statuses = fs.listStatus(trackFileDir);
+ } catch (FileNotFoundException e) {
+ LOG.debug("Track file directory {} does not exist", trackFileDir, e);
+ return Collections.emptyNavigableMap();
+ }
+ if (statuses == null || statuses.length == 0) {
+ return Collections.emptyNavigableMap();
+ }
+ TreeMap<Long, List<Path>> map = new TreeMap<>((l1, l2) ->
l2.compareTo(l1));
+ for (FileStatus status : statuses) {
+ Path file = status.getPath();
+ if (!status.isFile()) {
+ LOG.warn("Found invalid track file {}, which is not a file", file);
+ continue;
+ }
+ if (!TRACK_FILE_PATTERN.matcher(file.getName()).matches()) {
+ LOG.warn("Found invalid track file {}, skip", file);
+ continue;
+ }
+ List<String> parts =
Splitter.on(TRACK_FILE_SEPARATOR).splitToList(file.getName());
+ map.computeIfAbsent(Long.parseLong(parts.get(1)), k -> new
ArrayList<>()).add(file);
+ }
+ return map;
+ }
+
+ private void initializeTrackFiles(long seqId) {
+ trackFiles[0] = new Path(trackFileDir, TRACK_FILE_PREFIX +
TRACK_FILE_SEPARATOR + seqId);
+ trackFiles[1] = new Path(trackFileDir, TRACK_FILE_ROTATE_PREFIX +
TRACK_FILE_SEPARATOR + seqId);
+ }
+
+ private void cleanUpTrackFiles(long loadedSeqId,
+ NavigableMap<Long, List<Path>> seqId2TrackFiles) {
+ FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
+ NavigableMap<Long, List<Path>> toDelete =
+ loadedSeqId >= 0 ? seqId2TrackFiles.tailMap(loadedSeqId, false) :
seqId2TrackFiles;
+ toDelete.values().stream().flatMap(l -> l.stream()).forEach(file -> {
+ ForkJoinPool.commonPool().execute(() -> {
+ try {
+ fs.delete(file, false);
+ } catch (IOException e) {
+ LOG.warn("failed to delete unused track file {}", file, e);
+ }
+ });
+ });
+ }
+
+ StoreFileList load(boolean readOnly) throws IOException {
+ NavigableMap<Long, List<Path>> seqId2TrackFiles = listFiles();
+ long seqId = -1L;
StoreFileList[] lists = new StoreFileList[2];
- for (int i = 0; i < 2; i++) {
- try {
- lists[i] = load(trackFiles[i]);
- } catch (FileNotFoundException | EOFException e) {
- // this is normal case, so use info and do not log stacktrace
- LOG.info("Failed to load track file {}: {}", trackFiles[i],
e.toString());
+ for (Map.Entry<Long, List<Path>> entry : seqId2TrackFiles.entrySet()) {
+ List<Path> files = entry.getValue();
+ // should not have more than 2 files, if not, it means that the track
files are broken, just
+ // throw exception out and fail the region open.
Review comment:
Sounds reasonable. Will do.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
