Author: liyin Date: Tue Oct 15 18:18:20 2013 New Revision: 1532464 URL: http://svn.apache.org/r1532464 Log: [master] Let OldLogsCleaner find out whether logs are archived to hourly directory
Author: fan Summary: There was a static boolean initialized by HLog constructor to indicate whether logs should be archived to hourly directory. But OldLogsCleaner only runs on HMaster, which does not have HLog instance. It could not get the correct configuration so cleanup was still using the legacy way. Test Plan: 1. Unit test passes. 2. Replaced jar on master of os 8. The cleaner has expected behavior now. Reviewers: adela, aaiyer Reviewed By: adela CC: HBase Eng@ Differential Revision: https://phabricator.fb.com/D1010092 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Tue Oct 15 18:18:20 2013 @@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.regex.Pattern; @@ -47,6 +47,7 @@ public class OldLogsCleaner extends Chor static final Log LOG = LogFactory.getLog(OldLogsCleaner.class.getName()); private final static Pattern datePattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}"); + private static boolean ARCHIVE_TO_HOURLY_DIR = false; // Max number we can delete on every chore, this is to make sure we don't // issue thousands of delete commands around the same time @@ -75,6 +76,7 @@ public class OldLogsCleaner extends Chor conf.set("hbase.master.logcleanerplugin.impl", "org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner"); } + ARCHIVE_TO_HOURLY_DIR = conf.getBoolean("hbase.hlog.archive.hourlydir", false); this.maxDeletedLogs = conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20); this.fs = fs; @@ -102,19 +104,14 @@ public class OldLogsCleaner extends Chor * @throws IOException */ private void cleanHourlyDirectories(List<FileStatus> hourly) throws IOException { - FileStatus[] files = this.fs.listStatus(this.oldLogDir); - if (files == null || files.length == 0) { - LOG.debug("Old log folder is empty"); - return; - } - Arrays.sort(files); + Collections.sort(hourly); // Only delete one hourly sub-directory in one iteration. So we won't delete // too many directories/files in a short period of time. // When the system generates 10000-12000 log files per hour, // around 4GB data is deleted. - Path path = files[0].getPath(); + Path path = hourly.get(0).getPath(); if (logCleaner.isLogDeletable(path)) { - LOG.info("Removing old logs in " + path.toString()); + LOG.info("Removing old logs directory " + path.toString()); this.fs.delete(path, true); } else { LOG.debug("Current hourly directories are not old enough. Oldest directory: " + path.toString()); @@ -169,7 +166,7 @@ public class OldLogsCleaner extends Chor @Override protected void chore() { try { - if (HLog.shouldArchiveToHourlyDir()) { + if (ARCHIVE_TO_HOURLY_DIR) { FileStatus[] subdirs = this.fs.listStatus(this.oldLogDir); List<FileStatus> hourly = new ArrayList<FileStatus>(); List<FileStatus> legacy = new ArrayList<FileStatus>(); @@ -206,4 +203,8 @@ public class OldLogsCleaner extends Chor public static boolean isMatchDatePattern(Path file) { return datePattern.matcher(file.getName()).matches(); } + + public static boolean isOldLogsArchivedToHourlyDir() { + return ARCHIVE_TO_HOURLY_DIR; + } } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java Tue Oct 15 18:18:20 2013 @@ -41,7 +41,6 @@ public class TimeToLiveLogCleaner implem private Configuration conf; // Configured time a log can be kept after it was closed private long ttl; - private boolean parseTimeFromPathName; @Override public boolean isLogDeletable(Path filePath) { @@ -49,7 +48,7 @@ public class TimeToLiveLogCleaner implem long currentTime = System.currentTimeMillis(); try { // If the path name is in hourly format, skip getting modification time - if (HLog.shouldArchiveToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) { + if (OldLogsCleaner.isOldLogsArchivedToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) { time = HLog.DATE_FORMAT.parse(filePath.getName()).getTime(); } else { FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath); Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1532464&r1=1532463&r2=1532464&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 15 18:18:20 2013 @@ -1884,14 +1884,6 @@ public class HLog implements Syncable { return new Path(oldLogsSubDir, filename); } - public static boolean shouldArchiveToHourlyDir() { - return ARCHIVE_TO_HOURLY_DIR; - } - - public static void setArchiveToHourlyDir(boolean b) { - ARCHIVE_TO_HOURLY_DIR = b; - } - /** * Takes splitLogsMap and concurrently writes them to region directories using a thread pool * Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Tue Oct 15 18:18:20 2013 @@ -125,14 +125,13 @@ public class TestOldLogsCleaner { // set TTL to delete 5 hours c.setLong("hbase.master.logcleaner.ttl", 4 * 3600 * 1000); c.setBoolean("hbase.hlog.archive.hourlydir", true); - HLog.setArchiveToHourlyDir(true); Path oldLogDir = new Path(TEST_UTIL.getTestDir(), HConstants.HREGION_OLDLOGDIR_NAME); String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8"); FileSystem fs = FileSystem.get(c); StoppableImpl stop = new StoppableImpl(); - OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop,c, fs, oldLogDir); + OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop, c, fs, oldLogDir); // Create 1 invalid directory (considering legacy logs), 10 directories representing // recent 10 hours respectively
