Author: kihwal
Date: Thu Aug 8 19:21:24 2013
New Revision: 1511968
URL: http://svn.apache.org/r1511968
Log:
HDFS-3020. Fix editlog to automatically sync when buffer is full. Contributed
by Todd Lipcon.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1511968&r1=1511967&r2=1511968&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Aug 8 19:21:24 2013
@@ -28,6 +28,8 @@ Release 0.23.10 - UNRELEASED
HDFS-5028. LeaseRenewer throws ConcurrentModificationException when
timeout.
(zhaoyunjiong via szetszwo)
+ HDFS-3020. Fix editlog to automatically sync when buffer is full. (todd)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java?rev=1511968&r1=1511967&r2=1511968&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
Thu Aug 8 19:21:24 2013
@@ -86,7 +86,7 @@ class EditsDoubleBuffer {
}
boolean shouldForceSync() {
- return bufReady.size() >= initBufferSize;
+ return bufCurrent.size() >= initBufferSize;
}
DataOutputBuffer getCurrentBuf() {
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1511968&r1=1511967&r2=1511968&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Thu Aug 8 19:21:24 2013
@@ -723,6 +723,14 @@ public class FSEditLog {
List<JournalAndStream> getJournals() {
return journals;
}
+
+ /**
+ * Used only by tests.
+ */
+ @VisibleForTesting
+ void setMetricsForTests(NameNodeMetrics metrics) {
+ this.metrics = metrics;
+ }
/**
* Return a manifest of what finalized edit logs are available
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1511968&r1=1511967&r2=1511968&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Thu Aug 8 19:21:24 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
+import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.hadoop.util.StringUtils;
@@ -771,6 +772,40 @@ public class TestEditLog extends TestCas
log.close();
}
}
+
+ /**
+ * Regression test for HDFS-1112/HDFS-3020. Ensures that, even if
+ * logSync isn't called periodically, the edit log will sync itself.
+ */
+ public void testAutoSync() throws Exception {
+ File logDir = new File(TEST_DIR, "testAutoSync");
+ logDir.mkdirs();
+ FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
+
+ String oneKB = StringUtils.byteToHexString(
+ new byte[500]);
+
+ try {
+ log.open();
+ NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
+ log.setMetricsForTests(mockMetrics);
+
+ for (int i = 0; i < 400; i++) {
+ log.logDelete(oneKB, 1L);
+ }
+ // After ~400KB, we're still within the 512KB buffer size
+ Mockito.verify(mockMetrics, Mockito.times(0)).addSync(Mockito.anyLong());
+
+ // After ~400KB more, we should have done an automatic sync
+ for (int i = 0; i < 400; i++) {
+ log.logDelete(oneKB, 1L);
+ }
+ Mockito.verify(mockMetrics, Mockito.times(1)).addSync(Mockito.anyLong());
+
+ } finally {
+ log.close();
+ }
+ }
/**
* Tests the getEditLogManifest function using mock storage for a number
