Author: todd
Date: Fri Apr 6 05:07:33 2012
New Revision: 1310174
URL: http://svn.apache.org/viewvc?rev=1310174&view=rev
Log:
Merge trunk into auto-failover branch.
Addressed two semantic conflicts after the commit of HADOOP-8077 -- test code
was referencing the old constant for the fencing methods config key.
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Apr 6 05:07:33 2012
@@ -52,6 +52,11 @@ Trunk (unreleased changes)
HADOOP-7994. Remove getProtocolVersion and getProtocolSignature from the
client side translator and server side implementation. (jitendra)
+ HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson
+ via atm)
+
+ HADOOP-7757. Test file reference count is at least 3x actual value (Jon
+ Eagles via bobby)
BUG FIXES
@@ -247,6 +252,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8007. Use substitution tokens for fencing argument (todd)
+ HADOOP-8077. HA: fencing method should be able to be configured on
+ a per-NN or per-NS basis (todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -319,6 +327,8 @@ Release 2.0.0 - UNRELEASED
HADOOP-8243. Security support broken in CLI (manual) failover controller
(todd)
+ HADOOP-8251. Fix SecurityUtil.fetchServiceTicket after HADOOP-6941 (todd)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
@@ -386,6 +396,15 @@ Release 0.23.3 - UNRELEASED
HADOOP-8088. User-group mapping cache incorrectly does negative caching on
transient failures (Khiwal Lee via bobby)
+ HADOOP-8179. risk of NPE in CopyCommands processArguments() (Daryn Sharp
+ via bobby)
+
+ HADOOP-6963. In FileUtil.getDU(..), neither include the size of directories
+ nor follow symbolic links. (Ravi Prakash via szetszwo)
+
+ HADOOP-8180. Remove hsqldb since its not needed from pom.xml (Ravi Prakash
+ via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1309568-1310173
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/pom.xml?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/pom.xml
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/pom.xml
Fri Apr 6 05:07:33 2012
@@ -79,6 +79,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<scope>compile</scope>
@@ -194,11 +199,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>hsqldb</groupId>
- <artifactId>hsqldb</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>compile</scope>
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1309568-1310173
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1309568-1310173
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
Fri Apr 6 05:07:33 2012
@@ -28,18 +28,28 @@ public interface ByteBufferReadable {
/**
* Reads up to buf.remaining() bytes into buf. Callers should use
* buf.limit(..) to control the size of the desired read.
- *
- * After the call, buf.position() should be unchanged, and therefore any data
- * can be immediately read from buf.
- *
+ * <p/>
+ * After a successful call, buf.position() and buf.limit() should be
+ * unchanged, and therefore any data can be immediately read from buf.
+ * buf.mark() may be cleared or updated.
+ * <p/>
+ * In the case of an exception, the values of buf.position() and buf.limit()
+ * are undefined, and callers should be prepared to recover from this
+ * eventuality.
+ * <p/>
* Many implementations will throw {@link UnsupportedOperationException}, so
* callers that are not confident in support for this method from the
* underlying filesystem should be prepared to handle that exception.
+ * <p/>
+ * Implementations should treat 0-length requests as legitimate, and must not
+ * signal an error upon their receipt.
*
* @param buf
- * the ByteBuffer to receive the results of the read operation
+ * the ByteBuffer to receive the results of the read operation. Up
to
+ * buf.limit() - buf.position() bytes may be read.
* @return the number of bytes available to read from buf
- * @throws IOException if there is some error performing the read
+ * @throws IOException
+ * if there is some error performing the read
*/
public int read(ByteBuffer buf) throws IOException;
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Fri Apr 6 05:07:33 2012
@@ -483,11 +483,18 @@ public class FileUtil {
if (!dir.isDirectory()) {
return dir.length();
} else {
- size = dir.length();
File[] allFiles = dir.listFiles();
if(allFiles != null) {
for (int i = 0; i < allFiles.length; i++) {
- size = size + getDU(allFiles[i]);
+ boolean isSymLink;
+ try {
+ isSymLink =
org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
+ } catch(IOException ioe) {
+ isSymLink = true;
+ }
+ if(!isSymLink) {
+ size += getDU(allFiles[i]);
+ }
}
}
return size;
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
Fri Apr 6 05:07:33 2012
@@ -81,10 +81,9 @@ class CopyCommands {
}
FSDataOutputStream out = dst.fs.create(dst.path);
try {
- FSDataInputStream in = null;
for (PathData src : srcs) {
+ FSDataInputStream in = src.fs.open(src.path);
try {
- in = src.fs.open(src.path);
IOUtils.copyBytes(in, out, getConf(), false);
if (delimiter != null) {
out.write(delimiter.getBytes("UTF-8"));
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
Fri Apr 6 05:07:33 2012
@@ -53,9 +53,6 @@ import com.google.common.collect.Lists;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class NodeFencer {
- public static final String CONF_METHODS_KEY =
- "dfs.ha.fencing.methods";
-
private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)";
private static final Pattern CLASS_WITH_ARGUMENT =
Pattern.compile(CLASS_RE + "\\((.+?)\\)");
@@ -76,18 +73,18 @@ public class NodeFencer {
private final List<FenceMethodWithArg> methods;
- public NodeFencer(Configuration conf)
+ NodeFencer(Configuration conf, String spec)
throws BadFencingConfigurationException {
- this.methods = parseMethods(conf);
+ this.methods = parseMethods(conf, spec);
}
- public static NodeFencer create(Configuration conf)
+ public static NodeFencer create(Configuration conf, String confKey)
throws BadFencingConfigurationException {
- String confStr = conf.get(CONF_METHODS_KEY);
+ String confStr = conf.get(confKey);
if (confStr == null) {
return null;
}
- return new NodeFencer(conf);
+ return new NodeFencer(conf, confStr);
}
public boolean fence(HAServiceTarget fromSvc) {
@@ -115,10 +112,10 @@ public class NodeFencer {
return false;
}
- private static List<FenceMethodWithArg> parseMethods(Configuration conf)
+ private static List<FenceMethodWithArg> parseMethods(Configuration conf,
+ String spec)
throws BadFencingConfigurationException {
- String confStr = conf.get(CONF_METHODS_KEY);
- String[] lines = confStr.split("\\s*\n\\s*");
+ String[] lines = spec.split("\\s*\n\\s*");
List<FenceMethodWithArg> methods = Lists.newArrayList();
for (String line : lines) {
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
Fri Apr 6 05:07:33 2012
@@ -171,7 +171,7 @@ public class SecurityUtil {
} else {
principalClass = Class.forName("sun.security.krb5.PrincipalName");
credentialsClass = Class.forName("sun.security.krb5.Credentials");
- krb5utilClass = Class.forName("sun.security.jgss.krb5");
+ krb5utilClass = Class.forName("sun.security.jgss.krb5.Krb5Util");
}
@SuppressWarnings("rawtypes")
Constructor principalConstructor =
principalClass.getConstructor(String.class,
Propchange:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1309568-1310173
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Fri Apr 6 05:07:33 2012
@@ -93,6 +93,9 @@ public class TestFileUtil {
// create files in partitioned directories
createFile(partitioned, "part-r-00000", "foo");
createFile(partitioned, "part-r-00001", "bar");
+
+ // create a cycle using symlinks. Cycles should be handled
+ FileUtil.symLink(del.toString(), dir1.toString() + "/cycle");
}
/**
@@ -458,4 +461,18 @@ public class TestFileUtil {
return result;
}
+
+ /**
+ * Test that getDU is able to handle cycles caused due to symbolic links
+ * and that directory sizes are not added to the final calculated size
+ * @throws IOException
+ */
+ @Test
+ public void testGetDU() throws IOException {
+ setupDirs();
+
+ long du = FileUtil.getDU(TEST_DIR);
+ //Only two files (in partitioned) have 4 bytes each
+ Assert.assertEquals(du, 8);
+ }
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
Fri Apr 6 05:07:33 2012
@@ -55,8 +55,8 @@ class DummyHAService extends HAServiceTa
this.proxy = makeMock();
try {
Configuration conf = new Configuration();
- conf.set(NodeFencer.CONF_METHODS_KEY, DummyFencer.class.getName());
- this.fencer = Mockito.spy(NodeFencer.create(conf));
+ this.fencer = Mockito.spy(NodeFencer.create(conf,
+ DummyFencer.class.getName()));
} catch (BadFencingConfigurationException e) {
throw new RuntimeException(e);
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
Fri Apr 6 05:07:33 2012
@@ -132,8 +132,7 @@ public class TestNodeFencer {
throws BadFencingConfigurationException {
System.err.println("Testing configuration:\n" + confStr);
Configuration conf = new Configuration();
- conf.set(NodeFencer.CONF_METHODS_KEY, confStr);
- return new NodeFencer(conf);
+ return new NodeFencer(conf, confStr);
}
/**
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java?rev=1310174&r1=1310173&r2=1310174&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java
Fri Apr 6 05:07:33 2012
@@ -71,8 +71,7 @@ public class TestShellCommandFencer {
public void testCheckNoArgs() {
try {
Configuration conf = new Configuration();
- conf.set(NodeFencer.CONF_METHODS_KEY, "shell");
- new NodeFencer(conf);
+ new NodeFencer(conf, "shell");
fail("Didn't throw when passing no args to shell");
} catch (BadFencingConfigurationException confe) {
assertTrue(
@@ -85,8 +84,7 @@ public class TestShellCommandFencer {
public void testCheckParensNoArgs() {
try {
Configuration conf = new Configuration();
- conf.set(NodeFencer.CONF_METHODS_KEY, "shell()");
- new NodeFencer(conf);
+ new NodeFencer(conf, "shell()");
fail("Didn't throw when passing no args to shell");
} catch (BadFencingConfigurationException confe) {
assertTrue(
