Author: stack
Date: Fri Sep 23 18:35:17 2011
New Revision: 1174942
URL: http://svn.apache.org/viewvc?rev=1174942&view=rev
Log:
HBASE-4132 Extend the WALActionsListener API to accomodate log archival
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 23 18:35:17 2011
@@ -1,5 +1,8 @@
HBase Change Log
Release 0.93.0 - Unreleased
+ IMPROVEMENT
+ HBASE-4132 Extend the WALActionsListener API to accomodate log archival
+ (dhruba borthakur)
Release 0.92.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
Fri Sep 23 18:35:17 2011
@@ -154,7 +154,22 @@ class LogRoller extends Thread implement
}
@Override
- public void logRolled(Path newFile) {
+ public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+ // Not interested
+ }
+
+ @Override
+ public void postLogRoll(Path oldPath, Path newPath) throws IOException {
+ // Not interested
+ }
+
+ @Override
+ public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+ // Not interested
+ }
+
+ @Override
+ public void postLogArchive(Path oldPath, Path newPath) throws IOException {
// Not interested
}
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
Fri Sep 23 18:35:17 2011
@@ -548,11 +548,22 @@ public class HLog implements Syncable {
// Do all the preparation outside of the updateLock to block
// as less as possible the incoming writes
long currentFilenum = this.filenum;
+ Path oldPath = null;
+ if (currentFilenum > 0) {
+ oldPath = computeFilename(currentFilenum);
+ }
this.filenum = System.currentTimeMillis();
Path newPath = computeFilename();
if (LOG.isDebugEnabled()) {
LOG.debug("Enabling new writer for "+FSUtils.getPath(newPath));
}
+
+ // Tell our listeners that a new log is about to be created
+ if (!this.listeners.isEmpty()) {
+ for (WALActionsListener i : this.listeners) {
+ i.preLogRoll(oldPath, newPath);
+ }
+ }
HLog.Writer nextWriter = this.createWriterInstance(fs, newPath, conf);
// Can we get at the dfsclient outputstream? If an instance of
// SFLW, it'll have done the necessary reflection to get at the
@@ -564,7 +575,7 @@ public class HLog implements Syncable {
// Tell our listeners that a new log was created
if (!this.listeners.isEmpty()) {
for (WALActionsListener i : this.listeners) {
- i.logRolled(newPath);
+ i.postLogRoll(oldPath, newPath);
}
}
@@ -814,9 +825,22 @@ public class HLog implements Syncable {
LOG.info("moving old hlog file " + FSUtils.getPath(p) +
" whose highest sequenceid is " + seqno + " to " +
FSUtils.getPath(newPath));
+
+ // Tell our listeners that a log is going to be archived.
+ if (!this.listeners.isEmpty()) {
+ for (WALActionsListener i : this.listeners) {
+ i.preLogArchive(p, newPath);
+ }
+ }
if (!this.fs.rename(p, newPath)) {
throw new IOException("Unable to rename " + p + " to " + newPath);
}
+ // Tell our listeners that a log has been archived.
+ if (!this.listeners.isEmpty()) {
+ for (WALActionsListener i : this.listeners) {
+ i.postLogArchive(p, newPath);
+ }
+ }
}
/**
@@ -851,10 +875,24 @@ public class HLog implements Syncable {
if (!fs.exists(this.dir)) return;
FileStatus[] files = fs.listStatus(this.dir);
for(FileStatus file : files) {
+
Path p = getHLogArchivePath(this.oldLogDir, file.getPath());
+ // Tell our listeners that a log is going to be archived.
+ if (!this.listeners.isEmpty()) {
+ for (WALActionsListener i : this.listeners) {
+ i.preLogArchive(file.getPath(), p);
+ }
+ }
+
if (!fs.rename(file.getPath(),p)) {
throw new IOException("Unable to rename " + file.getPath() + " to " +
p);
}
+ // Tell our listeners that a log was archived.
+ if (!this.listeners.isEmpty()) {
+ for (WALActionsListener i : this.listeners) {
+ i.postLogArchive(file.getPath(), p);
+ }
+ }
}
LOG.debug("Moved " + files.length + " log files to " +
FSUtils.getPath(this.oldLogDir));
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
Fri Sep 23 18:35:17 2011
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
+import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -28,11 +29,36 @@ import org.apache.hadoop.hbase.HTableDes
* so make sure your implementation is fast else you'll slow hbase.
*/
public interface WALActionsListener {
+
+ /**
+ * The WAL is going to be rolled. The oldPath can be null if this is
+ * the first log file from the regionserver.
+ * @param oldPath the path to the old hlog
+ * @param newPath the path to the new hlog
+ */
+ public void preLogRoll(Path oldPath, Path newPath) throws IOException;
+
+ /**
+ * The WAL has been rolled. The oldPath can be null if this is
+ * the first log file from the regionserver.
+ * @param oldPath the path to the old hlog
+ * @param newPath the path to the new hlog
+ */
+ public void postLogRoll(Path oldPath, Path newPath) throws IOException;
+
+ /**
+ * The WAL is going to be archived.
+ * @param oldPath the path to the old hlog
+ * @param newPath the path to the new hlog
+ */
+ public void preLogArchive(Path oldPath, Path newPath) throws IOException;
+
/**
- * The WAL was rolled.
- * @param newFile the path to the new hlog
+ * The WAL has been archived.
+ * @param oldPath the path to the old hlog
+ * @param newPath the path to the new hlog
*/
- public void logRolled(Path newFile);
+ public void postLogArchive(Path oldPath, Path newPath) throws IOException;
/**
* A request was made that the WAL be rolled.
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
Fri Sep 23 18:35:17 2011
@@ -156,8 +156,23 @@ public class Replication implements WALA
}
@Override
- public void logRolled(Path p) {
- getReplicationManager().logRolled(p);
+ public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+ // Not interested
+ }
+
+ @Override
+ public void postLogRoll(Path oldPath, Path newPath) {
+ getReplicationManager().logRolled(newPath);
+ }
+
+ @Override
+ public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+ // Not interested
+ }
+
+ @Override
+ public void postLogArchive(Path oldPath, Path newPath) throws IOException {
+ // Not interested
}
/**
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
Fri Sep 23 18:35:17 2011
@@ -709,9 +709,23 @@ public class TestHLog {
}
@Override
- public void logRolled(Path newFile) {
+ public void preLogRoll(Path oldFile, Path newFile) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void postLogRoll(Path oldFile, Path newFile) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void preLogArchive(Path oldFile, Path newFile) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void postLogArchive(Path oldFile, Path newFile) {
// TODO Auto-generated method stub
-
}
@Override
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
Fri Sep 23 18:35:17 2011
@@ -416,13 +416,22 @@ public class TestLogRolling {
server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
this.log = server.getWAL();
final List<Path> paths = new ArrayList<Path>();
+ final List<Integer> preLogRolledCalled = new ArrayList<Integer>();
paths.add(log.computeFilename());
log.registerWALActionsListener(new WALActionsListener() {
@Override
- public void logRolled(Path newFile) {
+ public void preLogRoll(Path oldFile, Path newFile) {
+ preLogRolledCalled.add(new Integer(1));
+ }
+ @Override
+ public void postLogRoll(Path oldFile, Path newFile) {
paths.add(newFile);
}
@Override
+ public void preLogArchive(Path oldFile, Path newFile) {}
+ @Override
+ public void postLogArchive(Path oldFile, Path newFile) {}
+ @Override
public void logRollRequested() {}
@Override
public void logCloseRequested() {}
@@ -478,6 +487,7 @@ public class TestLogRolling {
// force a log roll to read back and verify previously written logs
log.rollWriter(true);
+ assertTrue(preLogRolledCalled.size() == 1);
// read back the data written
Set<String> loggedRows = new HashSet<String>();
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
Fri Sep 23 18:35:17 2011
@@ -108,8 +108,10 @@ public class TestWALActionsListener {
hlog.close();
hlog.closeAndDelete();
- assertEquals(11, observer.logRollCounter);
- assertEquals(5, laterobserver.logRollCounter);
+ assertEquals(11, observer.preLogRollCounter);
+ assertEquals(11, observer.postLogRollCounter);
+ assertEquals(5, laterobserver.preLogRollCounter);
+ assertEquals(5, laterobserver.postLogRollCounter);
assertEquals(2, observer.closedCount);
}
@@ -118,12 +120,28 @@ public class TestWALActionsListener {
* Just counts when methods are called
*/
static class DummyWALActionsListener implements WALActionsListener {
- public int logRollCounter = 0;
+ public int preLogRollCounter = 0;
+ public int postLogRollCounter = 0;
public int closedCount = 0;
@Override
- public void logRolled(Path newFile) {
- logRollCounter++;
+ public void preLogRoll(Path oldFile, Path newFile) {
+ preLogRollCounter++;
+ }
+
+ @Override
+ public void postLogRoll(Path oldFile, Path newFile) {
+ postLogRollCounter++;
+ }
+
+ @Override
+ public void preLogArchive(Path oldFile, Path newFile) {
+ // Not interested
+ }
+
+ @Override
+ public void postLogArchive(Path oldFile, Path newFile) {
+ // Not interested
}
@Override
