Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Fri Feb 7 01:57:21 2014 @@ -32,12 +32,13 @@ import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; -import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -45,13 +46,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; -import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -68,8 +71,10 @@ import org.apache.hadoop.hdfs.util.ReadO import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.StringUtils; import com.google.common.base.Preconditions; +import com.google.common.annotations.VisibleForTesting; /** * Contains inner classes for reading or writing the on-disk format for @@ -468,7 +473,8 @@ public class FSImageFormat { } /** - * load fsimage files assuming only local names are stored + * load fsimage files assuming only local names are stored. Used when + * snapshots are not supported by the layout version. * * @param numFiles number of files expected to be read * @param in image input stream @@ -584,6 +590,8 @@ public class FSImageFormat { */ private int loadDirectory(DataInput in, Counter counter) throws IOException { String parentPath = FSImageSerialization.readString(in); + // Rename .snapshot paths if we're doing an upgrade + parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion()); final INodeDirectory parent = INodeDirectory.valueOf( namesystem.dir.rootDir.getNode(parentPath, true), parentPath); return loadChildren(parent, in, counter); @@ -643,11 +651,9 @@ public class FSImageFormat { */ private void addToParent(INodeDirectory parent, INode child) { FSDirectory fsDir = namesystem.dir; - if (parent == fsDir.rootDir && FSDirectory.isReservedName(child)) { - throw new HadoopIllegalArgumentException("File name \"" - + child.getLocalName() + "\" is reserved. Please " - + " change the name of the existing file or directory to another " - + "name before upgrading to this release."); + if (parent == fsDir.rootDir) { + child.setLocalName(renameReservedRootComponentOnUpgrade( + child.getLocalNameBytes(), getLayoutVersion())); } // NOTE: This does not update space counts for parents if (!parent.addChild(child)) { @@ -684,7 +690,9 @@ public class FSImageFormat { public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in, boolean updateINodeMap, Counter counter) throws IOException { - final byte[] localName = FSImageSerialization.readLocalName(in); + byte[] localName = FSImageSerialization.readLocalName(in); + localName = + renameReservedComponentOnUpgrade(localName, getLayoutVersion()); INode inode = loadINode(localName, isSnapshotINode, in, counter); if (updateINodeMap && LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) { @@ -989,7 +997,156 @@ public class FSImageFormat { return snapshotMap.get(in.readInt()); } } - + + @VisibleForTesting + public static TreeMap<String, String> renameReservedMap = + new TreeMap<String, String>(); + + /** + * Use the default key-value pairs that will be used to determine how to + * rename reserved paths on upgrade. + */ + @VisibleForTesting + public static void useDefaultRenameReservedPairs() { + renameReservedMap.clear(); + for (String key: HdfsConstants.RESERVED_PATH_COMPONENTS) { + renameReservedMap.put( + key, + key + "." + LayoutVersion.getCurrentLayoutVersion() + "." + + "UPGRADE_RENAMED"); + } + } + + /** + * Set the key-value pairs that will be used to determine how to rename + * reserved paths on upgrade. + */ + @VisibleForTesting + public static void setRenameReservedPairs(String renameReserved) { + // Clear and set the default values + useDefaultRenameReservedPairs(); + // Overwrite with provided values + setRenameReservedMapInternal(renameReserved); + } + + private static void setRenameReservedMapInternal(String renameReserved) { + Collection<String> pairs = + StringUtils.getTrimmedStringCollection(renameReserved); + for (String p : pairs) { + String[] pair = StringUtils.split(p, '/', '='); + Preconditions.checkArgument(pair.length == 2, + "Could not parse key-value pair " + p); + String key = pair[0]; + String value = pair[1]; + Preconditions.checkArgument(DFSUtil.isReservedPathComponent(key), + "Unknown reserved path " + key); + Preconditions.checkArgument(DFSUtil.isValidNameForComponent(value), + "Invalid rename path for " + key + ": " + value); + LOG.info("Will rename reserved path " + key + " to " + value); + renameReservedMap.put(key, value); + } + } + + /** + * When upgrading from an old version, the filesystem could contain paths + * that are now reserved in the new version (e.g. .snapshot). This renames + * these new reserved paths to a user-specified value to avoid collisions + * with the reserved name. + * + * @param path Old path potentially containing a reserved path + * @return New path with reserved path components renamed to user value + */ + static String renameReservedPathsOnUpgrade(String path, + final int layoutVersion) { + final String oldPath = path; + // If any known LVs aren't supported, we're doing an upgrade + if (!LayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Only need to worry about the root directory + if (components.length > 1) { + components[1] = DFSUtil.bytes2String( + renameReservedRootComponentOnUpgrade( + DFSUtil.string2Bytes(components[1]), + layoutVersion)); + path = DFSUtil.strings2PathString(components); + } + } + if (!LayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Special case the root path + if (components.length == 0) { + return path; + } + for (int i=0; i<components.length; i++) { + components[i] = DFSUtil.bytes2String( + renameReservedComponentOnUpgrade( + DFSUtil.string2Bytes(components[i]), + layoutVersion)); + } + path = DFSUtil.strings2PathString(components); + } + + if (!path.equals(oldPath)) { + LOG.info("Upgrade process renamed reserved path " + oldPath + " to " + + path); + } + return path; + } + + private final static String RESERVED_ERROR_MSG = + FSDirectory.DOT_RESERVED_PATH_PREFIX + " is a reserved path and " + + HdfsConstants.DOT_SNAPSHOT_DIR + " is a reserved path component in" + + " this version of HDFS. Please rollback and delete or rename" + + " this path, or upgrade with the " + + StartupOption.RENAMERESERVED.getName() + + " [key-value pairs]" + + " option to automatically rename these paths during upgrade."; + + /** + * Same as {@link #renameReservedPathsOnUpgrade(String)}, but for a single + * byte array path component. + */ + private static byte[] renameReservedComponentOnUpgrade(byte[] component, + final int layoutVersion) { + // If the LV doesn't support snapshots, we're doing an upgrade + if (!LayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) { + if (Arrays.equals(component, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES)) { + Preconditions.checkArgument( + renameReservedMap != null && + renameReservedMap.containsKey(HdfsConstants.DOT_SNAPSHOT_DIR), + RESERVED_ERROR_MSG); + component = + DFSUtil.string2Bytes(renameReservedMap + .get(HdfsConstants.DOT_SNAPSHOT_DIR)); + } + } + return component; + } + + /** + * Same as {@link #renameReservedPathsOnUpgrade(String)}, but for a single + * byte array path component. + */ + private static byte[] renameReservedRootComponentOnUpgrade(byte[] component, + final int layoutVersion) { + // If the LV doesn't support inode IDs, we're doing an upgrade + if (!LayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) { + if (Arrays.equals(component, FSDirectory.DOT_RESERVED)) { + Preconditions.checkArgument( + renameReservedMap != null && + renameReservedMap.containsKey(FSDirectory.DOT_RESERVED_STRING), + RESERVED_ERROR_MSG); + final String renameString = renameReservedMap + .get(FSDirectory.DOT_RESERVED_STRING); + component = + DFSUtil.string2Bytes(renameString); + LOG.info("Renamed root path " + FSDirectory.DOT_RESERVED_STRING + + " to " + renameString); + } + } + return component; + } + /** * A one-shot class responsible for writing an image file. * The write() function should be called once, after which the getter
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Feb 7 01:57:21 2014 @@ -1164,7 +1164,8 @@ public class FSNamesystem implements Nam if (isInSafeMode()) { SafeModeException se = new SafeModeException(errorMsg, safeMode); if (haEnabled && haContext != null - && haContext.getState().getServiceState() == HAServiceState.ACTIVE) { + && haContext.getState().getServiceState() == HAServiceState.ACTIVE + && shouldRetrySafeMode(this.safeMode)) { throw new RetriableException(se); } else { throw se; @@ -1172,6 +1173,18 @@ public class FSNamesystem implements Nam } } + /** + * We already know that the safemode is on. We will throw a RetriableException + * if the safemode is not manual or caused by low resource. + */ + private boolean shouldRetrySafeMode(SafeModeInfo safeMode) { + if (safeMode == null) { + return false; + } else { + return !safeMode.isManual() && !safeMode.areResourcesLow(); + } + } + public static Collection<URI> getNamespaceDirs(Configuration conf) { return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY); } @@ -3803,7 +3816,7 @@ public class FSNamesystem implements Nam final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes(); if (diff > 0) { try { - String path = leaseManager.findPath(fileINode); + String path = fileINode.getFullPathName(); dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication()); } catch (IOException e) { LOG.warn("Unexpected exception while updating disk space.", e); @@ -4005,7 +4018,7 @@ public class FSNamesystem implements Nam @VisibleForTesting String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock) throws IOException { - String src = leaseManager.findPath(pendingFile); + String src = pendingFile.getFullPathName(); // commit the last block and complete it if it has minimum replicas commitOrCompleteLastBlock(pendingFile, storedBlock); @@ -4027,7 +4040,7 @@ public class FSNamesystem implements Nam @VisibleForTesting String persistBlocks(INodeFile pendingFile, boolean logRetryCache) throws IOException { - String src = leaseManager.findPath(pendingFile); + String src = pendingFile.getFullPathName(); dir.persistBlocks(src, pendingFile, logRetryCache); return src; } @@ -5955,7 +5968,7 @@ public class FSNamesystem implements Nam .getDatanodeStorageInfos(newNodes, newStorageIDs); blockinfo.setExpectedLocations(storages); - String src = leaseManager.findPath(pendingFile); + String src = pendingFile.getFullPathName(); dir.persistBlocks(src, pendingFile, logRetryCache); } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Fri Feb 7 01:57:21 2014 @@ -180,24 +180,6 @@ public class LeaseManager { } /** - * Finds the pathname for the specified pendingFile - */ - public synchronized String findPath(INodeFile pendingFile) - throws IOException { - FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature(); - Preconditions.checkArgument(uc != null); - Lease lease = getLease(uc.getClientName()); - if (lease != null) { - String src = lease.findPath(pendingFile); - if (src != null) { - return src; - } - } - throw new IOException("pendingFile (=" + pendingFile + ") not found." - + "(lease=" + lease + ")"); - } - - /** * Renew the lease(s) held by the given client */ synchronized void renewLease(String holder) { @@ -252,24 +234,6 @@ public class LeaseManager { return now() - lastUpdate > softLimit; } - /** - * @return the path associated with the pendingFile and null if not found. - */ - private String findPath(INodeFile pendingFile) { - try { - for (String src : paths) { - INode node = fsnamesystem.dir.getINode(src); - if (node == pendingFile - || (node.isFile() && node.asFile() == pendingFile)) { - return src; - } - } - } catch (UnresolvedLinkException e) { - throw new AssertionError("Lease files should reside on this FS"); - } - return null; - } - /** Does this lease contain any path? */ boolean hasPath() {return !paths.isEmpty();} Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Feb 7 01:57:21 2014 @@ -212,7 +212,9 @@ public class NameNode implements NameNod + StartupOption.CLUSTERID.getName() + " cid ] [" + StartupOption.FORCE.getName() + "] [" + StartupOption.NONINTERACTIVE.getName() + "] ] | [" - + StartupOption.UPGRADE.getName() + "] | [" + + StartupOption.UPGRADE.getName() + + " [" + StartupOption.CLUSTERID.getName() + " cid]" + + " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + StartupOption.IMPORT.getName() + "] | [" @@ -1056,7 +1058,8 @@ public class NameNode implements NameNod out.println(USAGE + "\n"); } - private static StartupOption parseArguments(String args[]) { + @VisibleForTesting + static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for(int i=0; i < argsLen; i++) { @@ -1103,11 +1106,33 @@ public class NameNode implements NameNod startOpt = StartupOption.CHECKPOINT; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.UPGRADE; - // might be followed by two args - if (i + 2 < argsLen - && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { - i += 2; - startOpt.setClusterId(args[i]); + /* Can be followed by CLUSTERID with a required parameter or + * RENAMERESERVED with an optional parameter + */ + while (i + 1 < argsLen) { + String flag = args[i + 1]; + if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { + if (i + 2 < argsLen) { + i += 2; + startOpt.setClusterId(args[i]); + } else { + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED + .getName())) { + if (i + 2 < argsLen) { + FSImageFormat.setRenameReservedPairs(args[i + 2]); + i += 2; + } else { + FSImageFormat.useDefaultRenameReservedPairs(); + i += 1; + } + } else { + LOG.fatal("Unknown upgrade flag " + flag); + return null; + } } } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLBACK; Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Fri Feb 7 01:57:21 2014 @@ -982,13 +982,18 @@ class NameNodeRpcServer implements Namen + "from " + nodeReg + ", reports.length=" + reports.length); } final BlockManager bm = namesystem.getBlockManager(); + boolean hasStaleStorages = true; for(StorageBlockReport r : reports) { final BlockListAsLongs blocks = new BlockListAsLongs(r.getBlocks()); - bm.processReport(nodeReg, r.getStorage(), poolId, blocks); + hasStaleStorages = bm.processReport(nodeReg, r.getStorage(), poolId, blocks); } - if (nn.getFSImage().isUpgradeFinalized() && !nn.isStandbyState()) + if (nn.getFSImage().isUpgradeFinalized() && + !nn.isStandbyState() && + !hasStaleStorages) { return new FinalizeCommand(poolId); + } + return null; } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Fri Feb 7 01:57:21 2014 @@ -962,9 +962,8 @@ public class CacheAdmin extends Configur if (numResults > 0) { System.out.print(listing); } - // If there are no results, we return 1 (failure exit code); - // otherwise we return 0 (success exit code). - return (numResults == 0) ? 1 : 0; + // If list pools succeed, we return 0 (success exit code) + return 0; } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Fri Feb 7 01:57:21 2014 @@ -188,6 +188,9 @@ public class WebHdfsFileSystem extends F int maxFailoverAttempts = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + int maxRetryAttempts = conf.getInt( + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); int failoverSleepBaseMillis = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); @@ -197,7 +200,7 @@ public class WebHdfsFileSystem extends F this.retryPolicy = RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, - maxFailoverAttempts, failoverSleepBaseMillis, + maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis); } Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1562962-1565513 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Fri Feb 7 01:57:21 2014 @@ -483,6 +483,20 @@ </property> <property> + <name>dfs.blockreport.split.threshold</name> + <value>1000000</value> + <description>If the number of blocks on the DataNode is below this + threshold then it will send block reports for all Storage Directories + in a single message. + + If the number of blocks exceeds this threshold then the DataNode will + send block reports for each Storage Directory in separate messages. + + Set to zero to always split. + </description> +</property> + +<property> <name>dfs.datanode.directoryscan.interval</name> <value>21600</value> <description>Interval in seconds for Datanode to scan data directories and Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1562962-1565513 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1562962-1565513 Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1562962-1565513 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm Fri Feb 7 01:57:21 2014 @@ -435,7 +435,7 @@ HDFS Users Guide state it was in before the upgrade. HDFS upgrade is described in more detail in {{{http://wiki.apache.org/hadoop/Hadoop_Upgrade}Hadoop Upgrade}} Wiki page. HDFS can have one such backup at a time. Before upgrading, - administrators need to remove existing backupusing bin/hadoop dfsadmin + administrators need to remove existing backup using bin/hadoop dfsadmin <<<-finalizeUpgrade>>> command. The following briefly describes the typical upgrade procedure: @@ -459,6 +459,33 @@ HDFS Users Guide * start the cluster with rollback option. (<<<bin/start-dfs.sh -rollback>>>). + When upgrading to a new version of HDFS, it is necessary to rename or + delete any paths that are reserved in the new version of HDFS. If the + NameNode encounters a reserved path during upgrade, it will print an + error like the following: + + <<< /.reserved is a reserved path and .snapshot is a + reserved path component in this version of HDFS. Please rollback and delete + or rename this path, or upgrade with the -renameReserved [key-value pairs] + option to automatically rename these paths during upgrade.>>> + + Specifying <<<-upgrade -renameReserved [optional key-value pairs]>>> causes + the NameNode to automatically rename any reserved paths found during + startup. For example, to rename all paths named <<<.snapshot>>> to + <<<.my-snapshot>>> and <<<.reserved>>> to <<<.my-reserved>>>, a user would + specify <<<-upgrade -renameReserved + .snapshot=.my-snapshot,.reserved=.my-reserved>>>. + + If no key-value pairs are specified with <<<-renameReserved>>>, the + NameNode will then suffix reserved paths with + <<<.<LAYOUT-VERSION>.UPGRADE_RENAMED>>>, e.g. + <<<.snapshot.-51.UPGRADE_RENAMED>>>. + + There are some caveats to this renaming process. It's recommended, + if possible, to first <<<hdfs dfsadmin -saveNamespace>>> before upgrading. + This is because data inconsistency can result if an edit log operation + refers to the destination of an automatically renamed file. + * File Permissions and Security The file permissions are designed to be similar to file permissions on Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml Fri Feb 7 01:57:21 2014 @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd";> <properties> - <title>HFDS Snapshots</title> + <title>HDFS Snapshots</title> </properties> <body> @@ -99,15 +99,22 @@ <li>Copying a file from snapshot <code>s0</code>: <source>hdfs dfs -cp /foo/.snapshot/s0/bar /tmp</source></li> </ul> - <p> - <b>Note</b> that the name ".snapshot" is now a reserved file name in HDFS - so that users cannot create a file/directory with ".snapshot" as the name. - If ".snapshot" is used in a previous version of HDFS, it must be renamed before upgrade; - otherwise, upgrade will fail. - </p> </subsection> </section> + <section name="Upgrading to a version of HDFS with snapshots" id="Upgrade"> + + <p> + The HDFS snapshot feature introduces a new reserved path name used to + interact with snapshots: <tt>.snapshot</tt>. When upgrading from an + older version of HDFS, existing paths named <tt>.snapshot</tt> need + to first be renamed or deleted to avoid conflicting with the reserved path. + See the upgrade section in + <a href="HdfsUserGuide.html#Upgrade_and_Rollback">the HDFS user guide</a> + for more information. </p> + + </section> + <section name="Snapshot Operations" id="SnapshotOperations"> <subsection name="Administrator Operations" id="AdministratorOperations"> <p> Propchange: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1562962-1565513 Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Fri Feb 7 01:57:21 2014 @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.TreeMap; @@ -43,7 +44,9 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -67,6 +70,7 @@ public class TestDFSUpgradeFromImage { private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt"; private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz"; private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz"; + private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz"; private static class ReferenceFileInfo { String path; @@ -320,6 +324,87 @@ public class TestDFSUpgradeFromImage { assertEquals("Upgrade did not fail with bad MD5", 1, md5failures); } } + + /** + * Test upgrade from 2.0 image with a variety of .snapshot and .reserved + * paths to test renaming on upgrade + */ + @Test + public void testUpgradeFromRel2ReservedImage() throws IOException { + unpackStorage(HADOOP2_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + try { + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "reserved path component in this version", + e); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + // Make sure the paths were renamed as expected + DistributedFileSystem dfs = cluster.getFileSystem(); + ArrayList<Path> toList = new ArrayList<Path>(); + ArrayList<String> found = new ArrayList<String>(); + toList.add(new Path("/")); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + String[] expected = new String[] { + "/edits", + "/edits/.reserved", + "/edits/.user-snapshot", + "/edits/.user-snapshot/editsdir", + "/edits/.user-snapshot/editsdir/editscontents", + "/edits/.user-snapshot/editsdir/editsdir2", + "/image", + "/image/.reserved", + "/image/.user-snapshot", + "/image/.user-snapshot/imagedir", + "/image/.user-snapshot/imagedir/imagecontents", + "/image/.user-snapshot/imagedir/imagedir2", + "/.my-reserved", + "/.my-reserved/edits-touch", + "/.my-reserved/image-touch" + }; + + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } static void recoverAllLeases(DFSClient dfs, Path path) throws IOException { Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java Fri Feb 7 01:57:21 2014 @@ -19,16 +19,19 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Supplier; + import java.io.InputStream; import java.io.PrintWriter; -import java.net.InetSocketAddress; -import java.net.Socket; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -37,10 +40,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ReflectionUtils; import org.junit.After; import org.junit.Before; @@ -51,10 +52,7 @@ import com.google.common.io.NullOutputSt public class TestDataTransferKeepalive { Configuration conf = new HdfsConfiguration(); private MiniDFSCluster cluster; - private FileSystem fs; - private InetSocketAddress dnAddr; private DataNode dn; - private DFSClient dfsClient; private static Path TEST_FILE = new Path("/test"); private static final int KEEPALIVE_TIMEOUT = 1000; @@ -69,15 +67,7 @@ public class TestDataTransferKeepalive { cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1).build(); - fs = cluster.getFileSystem(); - dfsClient = ((DistributedFileSystem)fs).dfs; - dfsClient.peerCache.clear(); - - String poolId = cluster.getNamesystem().getBlockPoolId(); dn = cluster.getDataNodes().get(0); - DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP( - dn, poolId); - dnAddr = NetUtils.createSocketAddr(dnReg.getXferAddr()); } @After @@ -90,34 +80,86 @@ public class TestDataTransferKeepalive { * its configured keepalive timeout. */ @Test(timeout=30000) - public void testKeepaliveTimeouts() throws Exception { + public void testDatanodeRespectsKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 60000L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Clients that write aren't currently re-used. - assertEquals(0, dfsClient.peerCache.size()); + assertEquals(0, fs.dfs.peerCache.size()); assertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.readFile(fs, TEST_FILE); - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); assertXceiverCount(1); // Sleep for a bit longer than the keepalive timeout // and make sure the xceiver died. - Thread.sleep(KEEPALIVE_TIMEOUT * 2); + Thread.sleep(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT + 1); assertXceiverCount(0); // The socket is still in the cache, because we don't // notice that it's closed until we try to read // from it again. - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); // Take it out of the cache - reading should // give an EOF. - Peer peer = dfsClient.peerCache.get(dn.getDatanodeId(), false); + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); assertNotNull(peer); assertEquals(-1, peer.getInputStream().read()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); + } + + /** + * Test that the client respects its keepalive timeout. + */ + @Test(timeout=30000) + public void testClientResponsesKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much shorter than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 10L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); + + // Clients that write aren't currently re-used. + assertEquals(0, fs.dfs.peerCache.size()); + assertXceiverCount(0); + + // Reads the file, so we should get a + // cached socket, and should have an xceiver on the other side. + DFSTestUtil.readFile(fs, TEST_FILE); + assertEquals(1, fs.dfs.peerCache.size()); + assertXceiverCount(1); + + // Sleep for a bit longer than the client keepalive timeout. + Thread.sleep(CLIENT_EXPIRY_MS + 1); + + // Taking out a peer which is expired should give a null. + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); + assertTrue(peer == null); + + // The socket cache is now empty. + assertEquals(0, fs.dfs.peerCache.size()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); } /** @@ -125,8 +167,17 @@ public class TestDataTransferKeepalive { * read bytes off the stream quickly. The datanode should time out sending the * chunks and the transceiver should die, even if it has a long keepalive. */ - @Test(timeout=30000) + @Test(timeout=300000) public void testSlowReader() throws Exception { + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 600000L; + Configuration clientConf = new Configuration(conf); + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); // Restart the DN with a shorter write timeout. DataNodeProperties props = cluster.stopDataNode(0); props.conf.setInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY, @@ -134,38 +185,31 @@ public class TestDataTransferKeepalive { props.conf.setInt(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY, 120000); assertTrue(cluster.restartDataNode(props, true)); + dn = cluster.getDataNodes().get(0); // Wait for heartbeats to avoid a startup race where we // try to write the block while the DN is still starting. cluster.triggerHeartbeats(); - dn = cluster.getDataNodes().get(0); - DFSTestUtil.createFile(fs, TEST_FILE, 1024*1024*8L, (short)1, 0L); FSDataInputStream stm = fs.open(TEST_FILE); - try { - stm.read(); - assertXceiverCount(1); + stm.read(); + assertXceiverCount(1); - // Poll for 0 running xceivers. Allow up to 5 seconds for some slack. - long totalSleepTime = 0; - long sleepTime = WRITE_TIMEOUT + 100; - while (getXceiverCountWithoutServer() > 0 && totalSleepTime < 5000) { - Thread.sleep(sleepTime); - totalSleepTime += sleepTime; - sleepTime = 100; + GenericTestUtils.waitFor(new Supplier<Boolean>() { + public Boolean get() { + // DN should time out in sendChunks, and this should force + // the xceiver to exit. + return getXceiverCountWithoutServer() == 0; } + }, 500, 50000); - // DN should time out in sendChunks, and this should force - // the xceiver to exit. - assertXceiverCount(0); - } finally { - IOUtils.closeStream(stm); - } + IOUtils.closeStream(stm); } @Test(timeout=30000) public void testManyClosedSocketsInCache() throws Exception { // Make a small file + DistributedFileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Insert a bunch of dead sockets in the cache, by opening Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Fri Feb 7 01:57:21 2014 @@ -65,7 +65,7 @@ public class TestBalancerWithNodeGroup { ClientProtocol client; - static final long TIMEOUT = 20000L; //msec + static final long TIMEOUT = 40000L; //msec static final double CAPACITY_ALLOWED_VARIANCE = 0.005; // 0.5% static final double BALANCE_ALLOWED_VARIANCE = 0.11; // 10%+delta static final int DEFAULT_BLOCK_SIZE = 10; Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Fri Feb 7 01:57:21 2014 @@ -124,6 +124,8 @@ public class TestReplicationPolicyWithNo CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, NetworkTopologyWithNodeGroup.class.getName()); + CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); + File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class); CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Fri Feb 7 01:57:21 2014 @@ -18,12 +18,16 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.DataOutputStream; import java.io.File; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.nio.channels.ClosedChannelException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -196,4 +200,16 @@ public class TestDiskError { } } } + + @Test + public void testNetworkErrorsIgnored() { + DataNode dn = cluster.getDataNodes().iterator().next(); + + assertTrue(dn.isNetworkRelatedException(new SocketException())); + assertTrue(dn.isNetworkRelatedException(new SocketTimeoutException())); + assertTrue(dn.isNetworkRelatedException(new ClosedChannelException())); + assertTrue(dn.isNetworkRelatedException(new Exception("Broken pipe foo bar"))); + assertFalse(dn.isNetworkRelatedException(new Exception())); + assertFalse(dn.isNetworkRelatedException(new Exception("random problem"))); + } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java Fri Feb 7 01:57:21 2014 @@ -85,6 +85,7 @@ public class TestNameNodeHttpServer { @Test public void testHttpPolicy() throws Exception { conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); InetSocketAddress addr = InetSocketAddress.createUnresolved("localhost", 0); NameNodeHttpServer server = null; @@ -103,7 +104,9 @@ public class TestNameNodeHttpServer { server.getHttpsAddress() == null)); } finally { - server.stop(); + if (server != null) { + server.stop(); + } } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java Fri Feb 7 01:57:21 2014 @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.protocol.H import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.SafeModeInfo; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IOUtils; @@ -65,6 +66,7 @@ import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; import com.google.common.base.Supplier; import com.google.common.collect.Lists; @@ -124,6 +126,9 @@ public class TestHASafeMode { final Path test = new Path("/test"); // let nn0 enter safemode NameNodeAdapter.enterSafeMode(nn0, false); + SafeModeInfo safeMode = (SafeModeInfo) Whitebox.getInternalState( + nn0.getNamesystem(), "safeMode"); + Whitebox.setInternalState(safeMode, "extension", Integer.valueOf(30000)); LOG.info("enter safemode"); new Thread() { @Override Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java Fri Feb 7 01:57:21 2014 @@ -52,6 +52,7 @@ public class TestHttpsFileSystem { conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); FileUtil.fullyDelete(base);
