Author: szetszwo
Date: Wed Nov 21 21:08:45 2012
New Revision: 1412297
URL: http://svn.apache.org/viewvc?rev=1412297&view=rev
Log:
Merge r1410998 through r1412282 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1410998-1412282
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Nov 21 21:08:45 2012
@@ -164,6 +164,9 @@ Trunk (Unreleased)
HDFS-4206. Change the fields in INode and its subclasses to private.
(szetszwo)
+ HDFS-4215. Remove locking from addToParent(..) since it is used in image
+ loading, and add INode.isFile(). (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -617,6 +620,13 @@ Release 2.0.3-alpha - Unreleased
HDFS-4171. WebHDFS and HttpFs should accept only valid Unix user
names. (tucu)
+ HDFS-4178. Shell scripts should not close stderr (Andy Isaacson via daryn)
+
+ HDFS-4179. BackupNode: allow reads, fix checkpointing, safeMode. (shv)
+
+ HDFS-4216. Do not ignore QuotaExceededException when adding symlinks.
+ (szetszwo)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
Wed Nov 21 21:08:45 2012
@@ -74,7 +74,7 @@ fi
#---------------------------------------------------------
# secondary namenodes (if any)
-SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>&-)
+SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes
2>/dev/null)
if [ -n "$SECONDARY_NAMENODES" ]; then
echo "Starting secondary namenodes [$SECONDARY_NAMENODES]"
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
Wed Nov 21 21:08:45 2012
@@ -50,7 +50,7 @@ fi
#---------------------------------------------------------
# secondary namenodes (if any)
-SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>&-)
+SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes
2>/dev/null)
if [ -n "$SECONDARY_NAMENODES" ]; then
echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]"
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1410998-1412282
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
Wed Nov 21 21:08:45 2012
@@ -82,12 +82,8 @@ import org.xml.sax.helpers.XMLReaderFact
@InterfaceStability.Evolving
public class HftpFileSystem extends FileSystem
implements DelegationTokenRenewer.Renewable {
- private static final DelegationTokenRenewer<HftpFileSystem> dtRenewer
- = new DelegationTokenRenewer<HftpFileSystem>(HftpFileSystem.class);
-
static {
HttpURLConnection.setFollowRedirects(true);
- dtRenewer.start();
}
public static final Text TOKEN_KIND = new Text("HFTP delegation");
@@ -106,6 +102,16 @@ public class HftpFileSystem extends File
private static final HftpDelegationTokenSelector hftpTokenSelector =
new HftpDelegationTokenSelector();
+ private DelegationTokenRenewer dtRenewer = null;
+
+ private synchronized void addRenewAction(final HftpFileSystem hftpFs) {
+ if (dtRenewer == null) {
+ dtRenewer = DelegationTokenRenewer.getInstance();
+ }
+
+ dtRenewer.addRenewAction(hftpFs);
+ }
+
public static final SimpleDateFormat getDateFormat() {
final SimpleDateFormat df = new SimpleDateFormat(HFTP_DATE_FORMAT);
df.setTimeZone(TimeZone.getTimeZone(HFTP_TIMEZONE));
@@ -202,7 +208,7 @@ public class HftpFileSystem extends File
if (token != null) {
setDelegationToken(token);
if (createdToken) {
- dtRenewer.addRenewAction(this);
+ addRenewAction(this);
LOG.debug("Created new DT for " + token.getService());
} else {
LOG.debug("Found existing DT for " + token.getService());
@@ -395,6 +401,14 @@ public class HftpFileSystem extends File
return new FSDataInputStream(new RangeHeaderInputStream(u));
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (dtRenewer != null) {
+ dtRenewer.removeRenewAction(this); // blocks
+ }
+ }
+
/** Class to parse and store a listing reply from the server. */
class LsParser extends DefaultHandler {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
Wed Nov 21 21:08:45 2012
@@ -69,6 +69,8 @@ public class BackupNode extends NameNode
private static final String BN_HTTP_ADDRESS_NAME_KEY =
DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY;
private static final String BN_HTTP_ADDRESS_DEFAULT =
DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_DEFAULT;
private static final String BN_SERVICE_RPC_ADDRESS_KEY =
DFSConfigKeys.DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY;
+ private static final float BN_SAFEMODE_THRESHOLD_PCT_DEFAULT = 1.5f;
+ private static final int BN_SAFEMODE_EXTENSION_DEFAULT =
Integer.MAX_VALUE;
/** Name-node proxy */
NamenodeProtocol namenode;
@@ -127,6 +129,10 @@ public class BackupNode extends NameNode
@Override // NameNode
protected void loadNamesystem(Configuration conf) throws IOException {
+ conf.setFloat(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,
+ BN_SAFEMODE_THRESHOLD_PCT_DEFAULT);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY,
+ BN_SAFEMODE_EXTENSION_DEFAULT);
BackupImage bnImage = new BackupImage(conf);
this.namesystem = new FSNamesystem(conf, bnImage);
bnImage.setNamesystem(namesystem);
@@ -423,9 +429,9 @@ public class BackupNode extends NameNode
return;
}
if (OperationCategory.JOURNAL != op &&
- !(OperationCategory.READ == op && allowStaleStandbyReads)) {
+ !(OperationCategory.READ == op && !isRole(NamenodeRole.CHECKPOINT)))
{
String msg = "Operation category " + op
- + " is not supported at the BackupNode";
+ + " is not supported at " + getRole();
throw new StandbyException(msg);
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
Wed Nov 21 21:08:45 2012
@@ -206,6 +206,7 @@ class Checkpointer extends Daemon {
RemoteEditLogManifest manifest =
getRemoteNamenodeProxy().getEditLogManifest(bnImage.getLastAppliedTxId()
+ 1);
+ boolean needReloadImage = false;
if (!manifest.getLogs().isEmpty()) {
RemoteEditLog firstRemoteLog = manifest.getLogs().get(0);
// we don't have enough logs to roll forward using only logs. Need
@@ -218,13 +219,10 @@ class Checkpointer extends Daemon {
bnStorage, true);
bnImage.saveDigestAndRenameCheckpointImage(
sig.mostRecentCheckpointTxId, downloadedHash);
-
- LOG.info("Loading image with txid " + sig.mostRecentCheckpointTxId);
- File file = bnStorage.findImageFile(sig.mostRecentCheckpointTxId);
- bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
+ lastApplied = sig.mostRecentCheckpointTxId;
+ needReloadImage = true;
}
-
- lastApplied = bnImage.getLastAppliedTxId();
+
if (firstRemoteLog.getStartTxId() > lastApplied + 1) {
throw new IOException("No logs to roll forward from " + lastApplied);
}
@@ -234,7 +232,12 @@ class Checkpointer extends Daemon {
TransferFsImage.downloadEditsToStorage(
backupNode.nnHttpAddress, log, bnStorage);
}
-
+
+ if(needReloadImage) {
+ LOG.info("Loading image with txid " + sig.mostRecentCheckpointTxId);
+ File file = bnStorage.findImageFile(sig.mostRecentCheckpointTxId);
+ bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
+ }
rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem());
}
@@ -243,8 +246,9 @@ class Checkpointer extends Daemon {
backupNode.namesystem.writeLock();
try {
backupNode.namesystem.dir.setReady();
- backupNode.namesystem.setBlockTotal();
-
+ if(backupNode.namesystem.getBlocksTotal() > 0) {
+ backupNode.namesystem.setBlockTotal();
+ }
bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
bnStorage.writeAll();
} finally {
@@ -284,12 +288,12 @@ class Checkpointer extends Daemon {
List<EditLogInputStream> editsStreams = Lists.newArrayList();
for (RemoteEditLog log : manifest.getLogs()) {
- File f = dstStorage.findFinalizedEditsFile(
- log.getStartTxId(), log.getEndTxId());
- if (log.getStartTxId() > dstImage.getLastAppliedTxId()) {
+ if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
+ File f = dstStorage.findFinalizedEditsFile(
+ log.getStartTxId(), log.getEndTxId());
editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(),
log.getEndTxId(), true));
- }
+ }
}
LOG.info("Checkpointer about to load edits from " +
editsStreams.size() + " stream(s).");
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Wed Nov 21 21:08:45 2012
@@ -77,6 +77,12 @@ import com.google.common.base.Preconditi
*
*************************************************/
public class FSDirectory implements Closeable {
+ private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) {
+ final INodeDirectoryWithQuota r = new INodeDirectoryWithQuota(
+ INodeDirectory.ROOT_NAME,
+ namesystem.createFsOwnerPermissions(new FsPermission((short)0755)));
+ return INodeDirectorySnapshottable.newInstance(r, 0);
+ }
INodeDirectoryWithQuota rootDir;
FSImage fsImage;
@@ -125,16 +131,7 @@ public class FSDirectory implements Clos
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
this.dirLock = new ReentrantReadWriteLock(true); // fair
this.cond = dirLock.writeLock().newCondition();
-
- this.namesystem = ns;
- int threshold = conf.getInt(
- DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
- DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT);
- NameNode.LOG.info("Caching file names occuring more than " + threshold
- + " times");
- this.nameCache = new NameCache<ByteArray>(threshold);
- reset();
-
+ rootDir = createRoot(ns);
this.fsImage = fsImage;
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
@@ -148,6 +145,14 @@ public class FSDirectory implements Clos
this.maxDirItems = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT);
+
+ int threshold = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT);
+ NameNode.LOG.info("Caching file names occuring more than " + threshold
+ + " times");
+ nameCache = new NameCache<ByteArray>(threshold);
+ namesystem = ns;
}
private FSNamesystem getFSNamesystem() {
@@ -309,35 +314,6 @@ public class FSDirectory implements Clos
return newNode;
}
- INodeDirectory addToParent(INodeDirectory parentINode,
- INode newNode, boolean propagateModTime) {
- // NOTE: This does not update space counts for parents
- INodeDirectory newParent = null;
- writeLock();
- try {
- try {
- newParent = rootDir.addToParent(newNode, parentINode,
- propagateModTime);
- cacheName(newNode);
- } catch (FileNotFoundException e) {
- return null;
- }
- if(newParent == null)
- return null;
- if(!newNode.isDirectory() && !newNode.isSymlink()) {
- // Add file->block mapping
- INodeFile newF = (INodeFile)newNode;
- BlockInfo[] blocks = newF.getBlocks();
- for (int i = 0; i < blocks.length; i++) {
- newF.setBlock(i, getBlockManager().addBlockCollection(blocks[i],
newF));
- }
- }
- } finally {
- writeUnlock();
- }
- return newParent;
- }
-
/**
* Add a block to the file. Returns a reference to the added block.
*/
@@ -845,11 +821,7 @@ public class FSDirectory implements Clos
final INodesInPath inodesInPath = rootDir.getMutableINodesInPath(src,
true);
final INode[] inodes = inodesInPath.getINodes();
INode inode = inodes[inodes.length - 1];
- if (inode == null) {
- return null;
- }
- assert !inode.isSymlink();
- if (inode.isDirectory()) {
+ if (inode == null || !inode.isFile()) {
return null;
}
INodeFile fileNode = (INodeFile)inode;
@@ -868,22 +840,15 @@ public class FSDirectory implements Clos
}
/**
- * Get the blocksize of a file
- * @param filename the filename
- * @return the number of bytes
+ * @param path the file path
+ * @return the block size of the file.
*/
- long getPreferredBlockSize(String filename) throws UnresolvedLinkException,
+ long getPreferredBlockSize(String path) throws UnresolvedLinkException,
FileNotFoundException, IOException {
readLock();
try {
- INode inode = rootDir.getNode(filename, false);
- if (inode == null) {
- throw new FileNotFoundException("File does not exist: " + filename);
- }
- if (inode.isDirectory() || inode.isSymlink()) {
- throw new IOException("Getting block size of non-file: "+ filename);
- }
- return ((INodeFile)inode).getPreferredBlockSize();
+ return INodeFile.valueOf(rootDir.getNode(path, false), path
+ ).getPreferredBlockSize();
} finally {
readUnlock();
}
@@ -897,9 +862,7 @@ public class FSDirectory implements Clos
if (inode == null) {
return false;
}
- return inode.isDirectory() || inode.isSymlink()
- ? true
- : ((INodeFile)inode).getBlocks() != null;
+ return !inode.isFile() || ((INodeFile)inode).getBlocks() != null;
} finally {
readUnlock();
}
@@ -1336,14 +1299,8 @@ public class FSDirectory implements Clos
waitForReady();
readLock();
try {
- INode targetNode = rootDir.getNode(src, false);
- if (targetNode == null)
- return null;
- if (targetNode.isDirectory())
- return null;
- if (targetNode.isSymlink())
- return null;
- return ((INodeFile)targetNode).getBlocks();
+ final INode i = rootDir.getNode(src, false);
+ return i != null && i.isFile()? ((INodeFile)i).getBlocks(): null;
} finally {
readUnlock();
}
@@ -2151,11 +2108,7 @@ public class FSDirectory implements Clos
writeLock();
try {
setReady(false);
- final INodeDirectoryWithQuota r = new INodeDirectoryWithQuota(
- INodeDirectory.ROOT_NAME,
- getFSNamesystem().createFsOwnerPermissions(new
FsPermission((short)0755)),
- Long.MAX_VALUE, UNKNOWN_DISK_SPACE);
- rootDir = INodeDirectorySnapshottable.newInstance(r, 0);
+ rootDir = createRoot(getFSNamesystem());
nameCache.reset();
} finally {
writeUnlock();
@@ -2250,7 +2203,7 @@ public class FSDirectory implements Clos
INodeSymlink addSymlink(String path, String target,
PermissionStatus dirPerms, boolean createParent)
throws UnresolvedLinkException, FileAlreadyExistsException,
- QuotaExceededException, IOException {
+ QuotaExceededException, SnapshotAccessControlException {
waitForReady();
final long modTime = now();
@@ -2264,7 +2217,7 @@ public class FSDirectory implements Clos
INodeSymlink newNode = null;
writeLock();
try {
- newNode = unprotectedSymlink(path, target, modTime, modTime,
+ newNode = unprotectedAddSymlink(path, target, modTime, modTime,
new PermissionStatus(userName, null, FsPermission.getDefault()));
} finally {
writeUnlock();
@@ -2284,23 +2237,12 @@ public class FSDirectory implements Clos
/**
* Add the specified path into the namespace. Invoked from edit log
processing.
*/
- INodeSymlink unprotectedSymlink(String path, String target, long modTime,
+ INodeSymlink unprotectedAddSymlink(String path, String target, long mtime,
long atime, PermissionStatus perm)
- throws UnresolvedLinkException {
+ throws UnresolvedLinkException, QuotaExceededException {
assert hasWriteLock();
- INodeSymlink newNode = new INodeSymlink(target, modTime, atime, perm);
- try {
- newNode = addNode(path, newNode, UNKNOWN_DISK_SPACE);
- } catch (UnresolvedLinkException e) {
- /* All UnresolvedLinkExceptions should have been resolved by now, but we
- * should re-throw them in case that changes so they are not swallowed
- * by catching IOException below.
- */
- throw e;
- } catch (IOException e) {
- return null;
- }
- return newNode;
+ final INodeSymlink symlink = new INodeSymlink(target, mtime, atime, perm);
+ return addNode(path, symlink, UNKNOWN_DISK_SPACE);
}
/**
@@ -2309,7 +2251,7 @@ public class FSDirectory implements Clos
*/
void cacheName(INode inode) {
// Name is cached only for files
- if (inode.isDirectory() || inode.isSymlink()) {
+ if (!inode.isFile()) {
return;
}
ByteArray name = new ByteArray(inode.getLocalNameBytes());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Wed Nov 21 21:08:45 2012
@@ -426,7 +426,7 @@ public class FSEditLogLoader {
}
case OP_SYMLINK: {
SymlinkOp symlinkOp = (SymlinkOp)op;
- fsDir.unprotectedSymlink(symlinkOp.path, symlinkOp.value,
+ fsDir.unprotectedAddSymlink(symlinkOp.path, symlinkOp.value,
symlinkOp.mtime, symlinkOp.atime,
symlinkOp.permissionStatus);
break;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Wed Nov 21 21:08:45 2012
@@ -38,7 +38,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable;
@@ -1020,6 +1019,7 @@ public class FSImage implements Closeabl
NamenodeCommand startCheckpoint(NamenodeRegistration bnReg, // backup node
NamenodeRegistration nnReg) // active
name-node
throws IOException {
+ LOG.info("Start checkpoint at txid " + getEditLog().getLastWrittenTxId());
String msg = null;
// Verify that checkpoint is allowed
if(bnReg.getNamespaceID() != storage.getNamespaceID())
@@ -1059,6 +1059,7 @@ public class FSImage implements Closeabl
* @throws IOException if the checkpoint fields are inconsistent
*/
void endCheckpoint(CheckpointSignature sig) throws IOException {
+ LOG.info("End checkpoint at txid " + getEditLog().getLastWrittenTxId());
sig.validateStorageInfo(this);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
Wed Nov 21 21:08:45 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.io.MD5Hash;
@@ -202,7 +203,7 @@ class FSImageFormat {
fsDir.rootDir.setQuota(nsQuota, dsQuota);
}
fsDir.rootDir.setModificationTime(root.getModificationTime());
- fsDir.rootDir.setPermissionStatus(root.getPermissionStatus());
+ fsDir.rootDir.clonePermissionStatus(root);
}
/**
@@ -258,7 +259,7 @@ class FSImageFormat {
// add to parent
newNode.setLocalName(localName);
- namesystem.dir.addToParent(parent, newNode, false);
+ addToParent(parent, newNode);
}
return numChildren;
}
@@ -293,7 +294,30 @@ class FSImageFormat {
// add new inode
newNode.setLocalName(pathComponents[pathComponents.length-1]);
- parentINode = fsDir.addToParent(parentINode, newNode, false);
+ addToParent(parentINode, newNode);
+ }
+ }
+
+ /**
+ * Add the child node to parent and, if child is a file, update block map.
+ * This method is only used for image loading so that synchronization,
+ * modification time update and space count update are not needed.
+ */
+ void addToParent(INodeDirectory parent, INode child) {
+ // NOTE: This does not update space counts for parents
+ if (parent.addChild(child, false) == null) {
+ return;
+ }
+ namesystem.dir.cacheName(child);
+
+ if (child.isFile()) {
+ // Add file->block mapping
+ final INodeFile file = (INodeFile)child;
+ final BlockInfo[] blocks = file.getBlocks();
+ final BlockManager bm = namesystem.getBlockManager();
+ for (int i = 0; i < blocks.length; i++) {
+ file.setBlock(i, bm.addBlockCollection(blocks[i], file));
+ }
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Wed Nov 21 21:08:45 2012
@@ -4048,7 +4048,8 @@ public class FSNamesystem implements Nam
// of the number of total blocks in the system.
this.shouldIncrementallyTrackBlocks = true;
}
-
+ if(blockSafe < 0)
+ this.blockSafe = 0;
checkMode();
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
Wed Nov 21 21:08:45 2012
@@ -90,17 +90,17 @@ public abstract class INode implements C
return (record & ~MASK) | (bits << OFFSET);
}
- /** Set the {@link PermissionStatus} */
+ /** Encode the {@link PermissionStatus} to a long. */
static long toLong(PermissionStatus ps) {
long permission = 0L;
final int user = SerialNumberManager.INSTANCE.getUserSerialNumber(
ps.getUserName());
- permission = PermissionStatusFormat.USER.combine(user, permission);
+ permission = USER.combine(user, permission);
final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber(
ps.getGroupName());
- permission = PermissionStatusFormat.GROUP.combine(group, permission);
+ permission = GROUP.combine(group, permission);
final int mode = ps.getPermission().toShort();
- permission = PermissionStatusFormat.MODE.combine(mode, permission);
+ permission = MODE.combine(mode, permission);
return permission;
}
}
@@ -114,8 +114,9 @@ public abstract class INode implements C
*/
private byte[] name = null;
/**
- * Permission encoded using PermissionStatusFormat.
- * Codes other than {@link #updatePermissionStatus(PermissionStatusFormat,
long)}.
+ * Permission encoded using {@link PermissionStatusFormat}.
+ * Codes other than {@link #clonePermissionStatus(INode)}
+ * and {@link #updatePermissionStatus(PermissionStatusFormat, long)}
* should not modify it.
*/
private long permission = 0L;
@@ -159,11 +160,9 @@ public abstract class INode implements C
return name.length == 0;
}
- /** Set the {@link PermissionStatus} */
- protected void setPermissionStatus(PermissionStatus ps) {
- setUser(ps.getUserName());
- setGroup(ps.getGroupName());
- setPermission(ps.getPermission());
+ /** Clone the {@link PermissionStatus}. */
+ void clonePermissionStatus(INode that) {
+ this.permission = that.permission;
}
/** Get the {@link PermissionStatus} */
public PermissionStatus getPermissionStatus() {
@@ -206,6 +205,13 @@ public abstract class INode implements C
}
/**
+ * Check whether it's a file.
+ */
+ public boolean isFile() {
+ return false;
+ }
+
+ /**
* Check whether it's a directory
*/
public boolean isDirectory() {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
Wed Nov 21 21:08:45 2012
@@ -395,23 +395,6 @@ public class INodeDirectory extends INod
return addToParent(pathComponents, newNode, true) == null? null: newNode;
}
- /**
- * Add new inode to the parent if specified.
- * Optimized version of addNode() if parent is not null.
- *
- * @return parent INode if new inode is inserted
- * or null if it already exists.
- * @throws FileNotFoundException if parent does not exist or
- * is not a directory.
- */
- INodeDirectory addToParent(INode newNode, INodeDirectory parent,
- boolean propagateModTime) throws FileNotFoundException {
- // insert into the parent children list
- if(parent.addChild(newNode, propagateModTime) == null)
- return null;
- return parent;
- }
-
INodeDirectory getParent(byte[][] pathComponents
) throws FileNotFoundException, UnresolvedLinkException {
if (pathComponents.length < 2) // add root
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
Wed Nov 21 21:08:45 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -26,9 +27,13 @@ import org.apache.hadoop.hdfs.protocol.Q
* Directory INode class that has a quota restriction
*/
public class INodeDirectoryWithQuota extends INodeDirectory {
- private long nsQuota; /// NameSpace quota
+ /** Name space quota */
+ private long nsQuota = Long.MAX_VALUE;
+ /** Name space count */
private long nsCount = 1L;
- private long dsQuota; /// disk space quota
+ /** Disk space quota */
+ private long dsQuota = HdfsConstants.QUOTA_RESET;
+ /** Disk space count */
private long diskspace = 0L;
/** Convert an existing directory inode to one with the given quota
@@ -57,11 +62,8 @@ public class INodeDirectoryWithQuota ext
}
/** constructor with no quota verification */
- INodeDirectoryWithQuota(String name, PermissionStatus permissions,
- long nsQuota, long dsQuota) {
+ INodeDirectoryWithQuota(String name, PermissionStatus permissions) {
super(name, permissions);
- this.nsQuota = nsQuota;
- this.dsQuota = dsQuota;
}
/** Get this directory's namespace quota
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Wed Nov 21 21:08:45 2012
@@ -100,6 +100,12 @@ public class INodeFile extends INode imp
this.setLocalName(f.getLocalNameBytes());
}
+ /** @return true unconditionally. */
+ @Override
+ public final boolean isFile() {
+ return true;
+ }
+
/**
* Set the {@link FsPermission} of this {@link INodeFile}.
* Since this is a file,
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Wed Nov 21 21:08:45 2012
@@ -124,15 +124,14 @@ public class WebHdfsFileSystem extends F
public static final WebHdfsDelegationTokenSelector DT_SELECTOR
= new WebHdfsDelegationTokenSelector();
- private static DelegationTokenRenewer<WebHdfsFileSystem> DT_RENEWER = null;
+ private DelegationTokenRenewer dtRenewer = null;
- private static synchronized void addRenewAction(final WebHdfsFileSystem
webhdfs) {
- if (DT_RENEWER == null) {
- DT_RENEWER = new
DelegationTokenRenewer<WebHdfsFileSystem>(WebHdfsFileSystem.class);
- DT_RENEWER.start();
+ private synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
+ if (dtRenewer == null) {
+ dtRenewer = DelegationTokenRenewer.getInstance();
}
- DT_RENEWER.addRenewAction(webhdfs);
+ dtRenewer.addRenewAction(webhdfs);
}
/** Is WebHDFS enabled in conf? */
@@ -766,6 +765,14 @@ public class WebHdfsFileSystem extends F
new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (dtRenewer != null) {
+ dtRenewer.removeRenewAction(this); // blocks
+ }
+ }
+
class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
OffsetUrlOpener(final URL url) {
super(url);
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1410998-1412282
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1410998-1412282
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1410998-1412282
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1410998-1412282
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1410998-1412282
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
Wed Nov 21 21:08:45 2012
@@ -28,9 +28,11 @@ import org.apache.commons.logging.impl.L
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
@@ -51,6 +53,7 @@ public class TestFcHdfsSymlink extends F
private static MiniDFSCluster cluster;
private static WebHdfsFileSystem webhdfs;
+ private static DistributedFileSystem dfs;
@Override
@@ -89,6 +92,7 @@ public class TestFcHdfsSymlink extends F
cluster = new MiniDFSCluster.Builder(conf).build();
fc = FileContext.getFileContext(cluster.getURI(0));
webhdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
+ dfs = cluster.getFileSystem();
}
@AfterClass
@@ -317,4 +321,27 @@ public class TestFcHdfsSymlink extends F
assertEquals(2, fc.getFileStatus(link).getReplication());
assertEquals(2, fc.getFileStatus(file).getReplication());
}
+
+ @Test
+ /** Test craeteSymlink(..) with quota. */
+ public void testQuota() throws IOException {
+ final Path dir = new Path(testBaseDir1());
+ dfs.setQuota(dir, 3, HdfsConstants.QUOTA_DONT_SET);
+
+ final Path file = new Path(dir, "file");
+ createAndWriteFile(file);
+
+ //creating the first link should succeed
+ final Path link1 = new Path(dir, "link1");
+ fc.createSymlink(file, link1, false);
+
+ try {
+ //creating the second link should fail with QuotaExceededException.
+ final Path link2 = new Path(dir, "link2");
+ fc.createSymlink(file, link2, false);
+ fail("Created symlink despite quota violation");
+ } catch(QuotaExceededException qee) {
+ //expected
+ }
+ }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
Wed Nov 21 21:08:45 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -33,6 +34,8 @@ public class TestHDFSFileSystemContract
@Override
protected void setUp() throws Exception {
Configuration conf = new HdfsConfiguration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+ FileSystemContractBaseTest.TEST_UMASK);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
fs = cluster.getFileSystem();
defaultWorkingDirectory = "/user/" +
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
Wed Nov 21 21:08:45 2012
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
@@ -99,7 +100,10 @@ public class TestBackupNode {
c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
"127.0.0.1:0");
- return (BackupNode)NameNode.createNameNode(new
String[]{startupOpt.getName()}, c);
+ BackupNode bn = (BackupNode)NameNode.createNameNode(
+ new String[]{startupOpt.getName()}, c);
+ assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
+ return bn;
}
void waitCheckpointDone(MiniDFSCluster cluster, long txid) {
@@ -358,11 +362,22 @@ public class TestBackupNode {
DFSTestUtil.createFile(bnFS, file3, fileSize, fileSize, blockSize,
replication, seed);
} catch (IOException eio) {
- LOG.info("Write to BN failed as expected: ", eio);
+ LOG.info("Write to " + backup.getRole() + " failed as expected: ",
eio);
canWrite = false;
}
assertFalse("Write to BackupNode must be prohibited.", canWrite);
+ // Reads are allowed for BackupNode, but not for CheckpointNode
+ boolean canRead = true;
+ try {
+ bnFS.exists(file2);
+ } catch (IOException eio) {
+ LOG.info("Read from " + backup.getRole() + " failed: ", eio);
+ canRead = false;
+ }
+ assertEquals("Reads to BackupNode are allowed, but not CheckpointNode.",
+ canRead, backup.isRole(NamenodeRole.BACKUP));
+
DFSTestUtil.createFile(fileSys, file3, fileSize, fileSize, blockSize,
replication, seed);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
Wed Nov 21 21:08:45 2012
@@ -73,7 +73,7 @@ public class TestFsLimits {
fileAsURI(new File(MiniDFSCluster.getBaseDirectory(),
"namenode")).toString());
- rootInode = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME, perms,
0L, 0L);
+ rootInode = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME, perms);
inodes = new INode[]{ rootInode, null };
fs = null;
fsIsReady = true;
