Author: szetszwo
Date: Fri Apr 5 02:43:29 2013
New Revision: 1464815
URL: http://svn.apache.org/r1464815
Log:
Merge r1462698 through r1464807 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Apr 5 02:43:29 2013
@@ -165,6 +165,9 @@ Trunk (Unreleased)
BUG FIXES
+ HADOOP-9451. Fault single-layer config if node group topology is enabled.
+ (Junping Du via llu)
+
HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
@@ -536,6 +539,9 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9358. "Auth failed" log should include exception string (todd)
+ HADOOP-9401. CodecPool: Add counters for number of (de)compressors
+ leased out. (kkambatl via tucu)
+
OPTIMIZATIONS
HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
@@ -604,8 +610,8 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
idle time. (Kai Zheng via atm)
- HADOOP-9357. Fallback to default authority if not specified in FileContext.
- (Andrew Wang via eli)
+ HADOOP-9429. TestConfiguration fails with IBM JAVA. (Amir Sanjar via
+ suresh)
Release 2.0.4-alpha - UNRELEASED
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1462698-1464807
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1462698-1464807
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1462698-1464807
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Fri Apr 5 02:43:29 2013
@@ -244,33 +244,17 @@ public final class FileContext {
}
/*
- * Resolve a relative path passed from the user.
- *
- * Relative paths are resolved against the current working directory
- * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
- * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
+ * Remove relative part - return "absolute":
+ * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
+ * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
* ("/foo/bar") are returned unchanged.
*
- * Additionally, we fix malformed URIs that specify a scheme but not an
- * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
- * if it matches the default FS, and let the default FS add in the default
- * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
- *
* Applications that use FileContext should use #makeQualified() since
- * they really want a fully-qualified URI.
+ * they really want a fully qualified URI.
* Hence this method is not called makeAbsolute() and
* has been deliberately declared private.
*/
private Path fixRelativePart(Path p) {
- // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
- if (p.hasSchemeAndNoAuthority()) {
- String scheme = p.toUri().getScheme();
- if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
- p = new Path(p.toUri().getSchemeSpecificPart());
- }
- }
- // Absolute paths are unchanged. Relative paths are resolved against the
- // current working directory.
if (p.isUriPathAbsolute()) {
return p;
} else {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Fri Apr 5 02:43:29 2013
@@ -256,10 +256,6 @@ public class Path implements Comparable
return (isUriPathAbsolute() &&
uri.getScheme() == null && uri.getAuthority() == null);
}
-
- public boolean hasSchemeAndNoAuthority() {
- return uri.getScheme() != null && uri.getAuthority() == null;
- }
/**
* True if the path component (i.e. directory) of this URI is absolute.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
Fri Apr 5 02:43:29 2013
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -29,6 +30,10 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
/**
* A global compressor/decompressor pool used to save and reuse
* (possibly native) compression/decompression codecs.
@@ -52,6 +57,29 @@ public class CodecPool {
private static final Map<Class<Decompressor>, List<Decompressor>>
decompressorPool =
new HashMap<Class<Decompressor>, List<Decompressor>>();
+ private static <T> LoadingCache<Class<T>, AtomicInteger> createCache(
+ Class<T> klass) {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Class<T>, AtomicInteger>() {
+ @Override
+ public AtomicInteger load(Class<T> key) throws Exception {
+ return new AtomicInteger();
+ }
+ });
+ }
+
+ /**
+ * Map to track the number of leased compressors
+ */
+ private static final LoadingCache<Class<Compressor>, AtomicInteger>
compressorCounts =
+ createCache(Compressor.class);
+
+ /**
+ * Map to tracks the number of leased decompressors
+ */
+ private static final LoadingCache<Class<Decompressor>, AtomicInteger>
decompressorCounts =
+ createCache(Decompressor.class);
+
private static <T> T borrow(Map<Class<T>, List<T>> pool,
Class<? extends T> codecClass) {
T codec = null;
@@ -90,6 +118,21 @@ public class CodecPool {
}
}
+ @SuppressWarnings("unchecked")
+ private static <T> int getLeaseCount(
+ LoadingCache<Class<T>, AtomicInteger> usageCounts,
+ Class<? extends T> codecClass) {
+ return usageCounts.getUnchecked((Class<T>) codecClass).get();
+ }
+
+ private static <T> void updateLeaseCount(
+ LoadingCache<Class<T>, AtomicInteger> usageCounts, T codec, int delta) {
+ if (codec != null) {
+ Class<T> codecClass = ReflectionUtils.getClass(codec);
+ usageCounts.getUnchecked(codecClass).addAndGet(delta);
+ }
+ }
+
/**
* Get a {@link Compressor} for the given {@link CompressionCodec} from the
* pool or a new one.
@@ -111,6 +154,7 @@ public class CodecPool {
LOG.debug("Got recycled compressor");
}
}
+ updateLeaseCount(compressorCounts, compressor, 1);
return compressor;
}
@@ -137,6 +181,7 @@ public class CodecPool {
LOG.debug("Got recycled decompressor");
}
}
+ updateLeaseCount(decompressorCounts, decompressor, 1);
return decompressor;
}
@@ -155,6 +200,7 @@ public class CodecPool {
}
compressor.reset();
payback(compressorPool, compressor);
+ updateLeaseCount(compressorCounts, compressor, -1);
}
/**
@@ -173,5 +219,24 @@ public class CodecPool {
}
decompressor.reset();
payback(decompressorPool, decompressor);
+ updateLeaseCount(decompressorCounts, decompressor, -1);
+ }
+
+ /**
+ * Return the number of leased {@link Compressor}s for this
+ * {@link CompressionCodec}
+ */
+ public static int getLeasedCompressorsCount(CompressionCodec codec) {
+ return (codec == null) ? 0 : getLeaseCount(compressorCounts,
+ codec.getCompressorType());
+ }
+
+ /**
+ * Return the number of leased {@link Decompressor}s for this
+ * {@link CompressionCodec}
+ */
+ public static int getLeasedDecompressorsCount(CompressionCodec codec) {
+ return (codec == null) ? 0 : getLeaseCount(decompressorCounts,
+ codec.getDecompressorType());
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
Fri Apr 5 02:43:29 2013
@@ -191,7 +191,12 @@ public class NetworkTopologyWithNodeGrou
}
rack = getNode(nodeGroup.getNetworkLocation());
- if (rack != null && !(rack instanceof InnerNode)) {
+ // rack should be an innerNode and with parent.
+ // note: rack's null parent case is: node's topology only has one layer,
+ // so rack is recognized as "/" and no parent.
+ // This will be recognized as a node with fault topology.
+ if (rack != null &&
+ (!(rack instanceof InnerNode) || rack.getParent() == null)) {
throw new IllegalArgumentException("Unexpected data node "
+ node.toString()
+ " at an illegal network location");
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
Fri Apr 5 02:43:29 2013
@@ -249,7 +249,7 @@ Hadoop MapReduce Next Generation - Clust
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.scheduler.class>>> | | |
| | <<<ResourceManager>>> Scheduler class. | |
-| | | <<<CapacityScheduler>>> (recommended) or <<<FifoScheduler>>> |
+| | | <<<CapacityScheduler>>> (recommended), <<<FairScheduler>>> (also
recommended), or <<<FifoScheduler>>> |
*-------------------------+-------------------------+------------------------+
| <<<yarn.scheduler.minimum-allocation-mb>>> | | |
| | Minimum limit of memory to allocate to each container request at the
<<<Resource Manager>>>. | |
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1462698-1464807
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
Fri Apr 5 02:43:29 2013
@@ -51,6 +51,10 @@ public class TestConfiguration extends T
final static String CONFIG = new File("./test-config.xml").getAbsolutePath();
final static String CONFIG2 = new
File("./test-config2.xml").getAbsolutePath();
final static Random RAN = new Random();
+ final static boolean IBMJAVA =
System.getProperty("java.vendor").contains("IBM");
+ final static String XMLHEADER =
+ IBMJAVA?"<?xml version=\"1.0\"
encoding=\"UTF-8\"?><configuration>":
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"
standalone=\"no\"?><configuration>";
@Override
protected void setUp() throws Exception {
@@ -327,8 +331,8 @@ public class TestConfiguration extends T
ByteArrayOutputStream baos = new ByteArrayOutputStream();
conf.writeXml(baos);
String result = baos.toString();
- assertTrue("Result has proper header", result.startsWith(
- "<?xml version=\"1.0\" encoding=\"UTF-8\"
standalone=\"no\"?><configuration>"));
+ assertTrue("Result has proper header", result.startsWith(XMLHEADER));
+
assertTrue("Result has proper footer",
result.endsWith("</configuration>"));
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
Fri Apr 5 02:43:29 2013
@@ -21,8 +21,6 @@ package org.apache.hadoop.fs;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.EnumSet;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1166,40 +1164,6 @@ public abstract class FileContextMainOpe
Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file));
}
}
-
- /**
- * Test that URIs with a scheme, no authority, and absolute path component
- * resolve with the authority of the default FS.
- */
- @Test(timeout=30000)
- public void testAbsolutePathSchemeNoAuthority() throws IOException,
- URISyntaxException {
- Path file = getTestRootPath(fc, "test/file");
- createFile(file);
- URI uri = file.toUri();
- URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(),
- uri.getQuery(), uri.getFragment());
- Path noAuthority = new Path(noAuthorityUri);
- Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
- }
-
- /**
- * Test that URIs with a scheme, no authority, and relative path component
- * resolve with the authority of the default FS.
- */
- @Test(timeout=30000)
- public void testRelativePathSchemeNoAuthority() throws IOException,
- URISyntaxException {
- Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
- fc.setWorkingDirectory(workDir);
- Path file = new Path(workDir, "file");
- createFile(file);
- URI uri = file.toUri();
- URI noAuthorityUri = new URI(uri.getScheme() + ":file");
- System.out.println(noAuthorityUri);
- Path noAuthority = new Path(noAuthorityUri);
- Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
- }
protected void createFile(Path path) throws IOException {
FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),
