Author: cutting Date: Fri Jun 22 15:55:34 2007 New Revision: 549977 URL: http://svn.apache.org/viewvc?view=rev&rev=549977 Log: HADOOP-1377. Add support for modification time to FileSystem. Contributed by Dhruba.
Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileStatus.java lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestModTime.java Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFileInfo.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java lucene/hadoop/trunk/src/webapps/datanode/browseDirectory.jsp Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Fri Jun 22 15:55:34 2007 @@ -246,6 +246,11 @@ a temporary name, then rename them to their final name, so that failures don't leave partial files. (Tsz Wo Sze via cutting) + 76. HADOOP-1377. Add support for modification time to FileSystem and + implement in HDFS and local implementations. Also, alter access + to file properties to be through a new FileStatus interface. + (Dhruba Borthakur via cutting) + Release 0.13.0 - 2007-06-08 Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java Fri Jun 22 15:55:34 2007 @@ -30,10 +30,8 @@ /** * Compared to the previous version the following changes have been introduced: - * 12: open() prototype changed; - * getBlockLocations() added; - * DFSFileInfo format changed; - * getHints() removed. + * 13: getListing returns file creation times and modification times. + * getFileInfo added. * DatanodeInfo serialization has hostname. */ public static final long versionID = 13L; @@ -370,4 +368,11 @@ * @throws IOException */ public void metaSave(String filename) throws IOException; + + /* Get the file info for a specific file or directory. + * @param src The string representation of the path to the file + * @throws IOException if file does not exist + * @return object containing information regarding the file + */ + public DFSFileInfo getFileInfo(String src) throws IOException; } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Fri Jun 22 15:55:34 2007 @@ -399,6 +399,11 @@ return namenode.getListing(src.toString()); } + public DFSFileInfo getFileInfo(UTF8 src) throws IOException { + checkOpen(); + return namenode.getFileInfo(src.toString()); + } + /** */ public long totalRawCapacity() throws IOException { Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFileInfo.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFileInfo.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFileInfo.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFileInfo.java Fri Jun 22 15:55:34 2007 @@ -19,6 +19,7 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; import java.io.*; @@ -30,7 +31,7 @@ * Block locations are sorted by the distance to the current client. * ******************************************************/ -class DFSFileInfo implements Writable { +class DFSFileInfo implements Writable, FileStatus { static { // register a ctor WritableFactories.setFactory (DFSFileInfo.class, @@ -44,7 +45,8 @@ boolean isDir; short blockReplication; long blockSize; - + long modificationTime; + /** */ public DFSFileInfo() { @@ -59,6 +61,7 @@ this.len = isDir ? node.computeContentsLength() : node.computeFileLength(); this.blockReplication = node.getReplication(); blockSize = node.getBlockSize(); + modificationTime = node.getModificationTime(); } /** @@ -112,6 +115,14 @@ public long getBlockSize() { return blockSize; } + + /** + * Get the last modification time of the file. + * @return the number of milliseconds since January 1, 1970 UTC. + */ + public long getModificationTime() { + return modificationTime; + } ////////////////////////////////////////////////// // Writable @@ -122,6 +133,7 @@ out.writeBoolean(isDir); out.writeShort(blockReplication); out.writeLong(blockSize); + out.writeLong(modificationTime); } public void readFields(DataInput in) throws IOException { @@ -131,5 +143,6 @@ this.isDir = in.readBoolean(); this.blockReplication = in.readShort(); blockSize = in.readLong(); + modificationTime = in.readLong(); } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java Fri Jun 22 15:55:34 2007 @@ -55,4 +55,7 @@ public long getBlockSize() { return info.getBlockSize(); } + public long getModificationTime() { + return info.getModificationTime(); + } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Fri Jun 22 15:55:34 2007 @@ -76,14 +76,6 @@ return dfs.getDefaultBlockSize(); } - public long getBlockSize(Path f) throws IOException { - // if we already know the answer, use it. - if (f instanceof DfsPath) { - return ((DfsPath) f).getBlockSize(); - } - return dfs.getBlockSize(getPath(f)); - } - public short getDefaultReplication() { return dfs.getDefaultReplication(); } @@ -172,22 +164,6 @@ return dfs.exists(getPath(f)); } - public boolean isDirectory(Path f) throws IOException { - if (f instanceof DfsPath) { - return ((DfsPath)f).isDirectory(); - } - return dfs.isDirectory(getPath(f)); - } - - public long getLength(Path f) throws IOException { - if (f instanceof DfsPath) { - return ((DfsPath)f).length(); - } - - DFSFileInfo info[] = dfs.listPaths(getPath(f)); - return (info == null) ? 0 : info[0].getLen(); - } - public long getContentLength(Path f) throws IOException { if (f instanceof DfsPath) { return ((DfsPath)f).getContentsLength(); @@ -197,15 +173,6 @@ return (info == null) ? 0 : info[0].getLen(); } - public short getReplication(Path f) throws IOException { - if (f instanceof DfsPath) { - return ((DfsPath)f).getReplication(); - } - - DFSFileInfo info[] = dfs.listPaths(getPath(f)); - return info[0].getReplication(); - } - public Path[] listPaths(Path f) throws IOException { DFSFileInfo info[] = dfs.listPaths(getPath(f)); if (info == null) { @@ -367,6 +334,20 @@ return true; } + + /** + * Returns the stat information about the file. + */ + public FileStatus getFileStatus(Path f) throws IOException { + if (f instanceof DfsPath) { + DfsPath p = (DfsPath) f; + return p.info; + } + else { + DFSFileInfo p = dfs.getFileInfo(getPath(f)); + return p; + } + } } public DistributedFileSystem() { @@ -443,5 +424,13 @@ FSDataInputStream sums, long sumsPos) { return ((RawDistributedFileSystem)fs).reportChecksumFailure( f, in, inPos, sums, sumsPos); + } + + /** + * Returns the stat information about the file. + */ + @Override + public FileStatus getFileStatus(Path f) throws IOException { + return ((RawDistributedFileSystem)fs).getFileStatus(f); } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java Fri Jun 22 15:55:34 2007 @@ -136,8 +136,7 @@ // Version is reflected in the data storage file. // Versions are negative. // Decrement LAYOUT_VERSION to define a new version. - public static final int LAYOUT_VERSION = -4; + public static final int LAYOUT_VERSION = -5; // Current version: - // Top level directory is reorganized to allow file system state - // transitions: upgrade, rollback, and finalize. + // File modification times added. } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java Fri Jun 22 15:55:34 2007 @@ -48,6 +48,7 @@ private TreeMap<String, INode> children = null; private Block blocks[] = null; private short blockReplication; + private long modificationTime; /** */ @@ -56,6 +57,7 @@ this.parent = null; this.blocks = blocks; this.blockReplication = replication; + this.modificationTime = 0; } /** @@ -65,6 +67,26 @@ this.parent = null; this.blocks = null; this.blockReplication = 0; + this.modificationTime = 0; + } + + INode(String name, long modifictionTime) { + this.name = name; + this.parent = null; + this.blocks = null; + this.blockReplication = 0; + this.modificationTime = modifictionTime; + } + + /** + */ + INode(String name, Block blocks[], short replication, + long modificationTime) { + this.name = name; + this.parent = null; + this.blocks = blocks; + this.blockReplication = replication; + this.modificationTime = modificationTime; } /** @@ -107,6 +129,24 @@ } /** + * Get last modification time of inode. + * @return access time + */ + long getModificationTime() { + return this.modificationTime; + } + + /** + * Set last modification time of inode. + */ + void setModificationTime(long modtime) { + assert isDir(); + if (this.modificationTime <= modtime) { + this.modificationTime = modtime; + } + } + + /** * Get children iterator * @return Iterator of children */ @@ -215,7 +255,7 @@ return true; } } - + /** * Collect all the blocks at this INode and all its children. * This operation is performed after a node is removed from the tree, @@ -385,12 +425,14 @@ public boolean addFile(String path, Block[] blocks, short replication) { waitForReady(); - // Always do an implicit mkdirs for parent directory tree - if (!mkdirs(new Path(path).getParent().toString())) { + // Always do an implicit mkdirs for parent directory tree. + long modTime = FSNamesystem.now(); + if (!mkdirs(new Path(path).getParent().toString(), modTime)) { return false; } - INode newNode = new INode(new File(path).getName(), blocks, replication); - if (!unprotectedAddFile(path, newNode)) { + INode newNode = new INode(new File(path).getName(), blocks, + replication, modTime); + if (!unprotectedAddFile(path, newNode, modTime)) { NameNode.stateChangeLog.info("DIR* FSDirectory.addFile: " +"failed to add "+path+" with " +blocks.length+" blocks to the file system"); @@ -405,14 +447,17 @@ /** */ - boolean unprotectedAddFile(String path, INode newNode) { + private boolean unprotectedAddFile(String path, INode newNode, long parentModTime) { synchronized (rootDir) { try { if (rootDir.addNode(path, newNode) != null) { int nrBlocks = (newNode.blocks == null) ? 0 : newNode.blocks.length; // Add file->block mapping - for (int i = 0; i < nrBlocks; i++) + for (int i = 0; i < nrBlocks; i++) { namesystem.blocksMap.addINode(newNode.blocks[i], newNode); + } + // update modification time of parent directory + newNode.getParent().setModificationTime(parentModTime); return true; } else { return false; @@ -423,9 +468,12 @@ } } - boolean unprotectedAddFile(String path, Block[] blocks, short replication) { + boolean unprotectedAddFile(String path, Block[] blocks, short replication, + long modificationTime) { return unprotectedAddFile(path, - new INode(path, blocks, replication)); + new INode(path, blocks, replication, + modificationTime), + modificationTime); } /** @@ -435,15 +483,16 @@ NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " +src+" to "+dst); waitForReady(); - if (!unprotectedRenameTo(src, dst)) + long now = namesystem.now(); + if (!unprotectedRenameTo(src, dst, now)) return false; - fsImage.getEditLog().logRename(src, dst); + fsImage.getEditLog().logRename(src, dst, now); return true; } /** */ - boolean unprotectedRenameTo(String src, String dst) { + boolean unprotectedRenameTo(String src, String dst, long timestamp) { synchronized(rootDir) { INode renamedNode = rootDir.getNode(src); if (renamedNode == null) { @@ -459,6 +508,7 @@ +"failed to rename "+src+" to "+dst+ " because destination exists"); return false; } + INode oldParent = renamedNode.getParent(); renamedNode.removeNode(); // the renamed node can be reused now @@ -466,6 +516,10 @@ if (rootDir.addNode(dst, renamedNode) != null) { NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedRenameTo: " +src+" is renamed to "+dst); + + // update modification time of old parent as well as new parent dir + oldParent.setModificationTime(timestamp); + renamedNode.getParent().setModificationTime(timestamp); return true; } } catch (FileNotFoundException e) { @@ -476,7 +530,6 @@ }catch(FileNotFoundException e2) { } } - return false; } } @@ -549,15 +602,16 @@ NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " +src); waitForReady(); - Block[] blocks = unprotectedDelete(src); + long now = namesystem.now(); + Block[] blocks = unprotectedDelete(src, now); if (blocks != null) - fsImage.getEditLog().logDelete(src); + fsImage.getEditLog().logDelete(src, now); return blocks; } /** */ - Block[] unprotectedDelete(String src) { + Block[] unprotectedDelete(String src, long modificationTime) { synchronized (rootDir) { INode targetNode = rootDir.getNode(src); if (targetNode == null) { @@ -576,6 +630,7 @@ } else { NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: " +src+" is removed"); + targetNode.getParent().setModificationTime(modificationTime); Vector<Block> v = new Vector<Block>(); targetNode.collectSubtreeBlocks(v); for (Block b : v) { @@ -648,6 +703,24 @@ } } + /* Get the file info for a specific file. + * @param src The string representation of the path to the file + * @throws IOException if file does not exist + * @return object containing information regarding the file + */ + DFSFileInfo getFileInfo(String src) throws IOException { + String srcs = normalizePath(src); + synchronized (rootDir) { + INode targetNode = rootDir.getNode(srcs); + if (targetNode == null) { + throw new IOException("File does not exist"); + } + else { + return new DFSFileInfo(targetNode); + } + } + } + /** * Get the blocks associated with the file. */ @@ -702,7 +775,7 @@ /** * Create directory entries for every item */ - boolean mkdirs(String src) { + boolean mkdirs(String src, long now) { src = normalizePath(src); // Use this to collect all the dirs we need to construct @@ -724,7 +797,7 @@ for (int i = numElts - 1; i >= 0; i--) { String cur = v.elementAt(i); try { - INode inserted = unprotectedMkdir(cur); + INode inserted = unprotectedMkdir(cur, now); if (inserted != null) { NameNode.stateChangeLog.debug("DIR* FSDirectory.mkdirs: " +"created directory "+cur); @@ -747,9 +820,10 @@ /** */ - INode unprotectedMkdir(String src) throws FileNotFoundException { + INode unprotectedMkdir(String src, long timestamp) throws FileNotFoundException { synchronized (rootDir) { - return rootDir.addNode(src, new INode(new File(src).getName())); + return rootDir.addNode(src, new INode(new File(src).getName(), + timestamp)); } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java Fri Jun 22 15:55:34 2007 @@ -227,6 +227,9 @@ short replication = fsNamesys.getDefaultReplication(); try { while (true) { + long timestamp = 0; + long ctime = 0; + long mtime = 0; byte opcode = -1; try { opcode = in.readByte(); @@ -247,13 +250,18 @@ aw = new ArrayWritable(UTF8.class); aw.readFields(in); writables = aw.get(); - if (writables.length != 2) - throw new IOException("Incorrect data fortmat. " - + "Name & replication pair expected"); + if (logVersion >= -4 && writables.length != 2 || + logVersion < -4 && writables.length != 3) { + throw new IOException("Incorrect data fortmat. " + + "Name & replication pair expected"); + } name = (UTF8) writables[0]; replication = Short.parseShort( ((UTF8)writables[1]).toString()); replication = adjustReplication(replication); + if (logVersion < -4) { + mtime = Long.parseLong(((UTF8)writables[2]).toString()); + } } // get blocks aw = new ArrayWritable(Block.class); @@ -262,7 +270,7 @@ Block blocks[] = new Block[writables.length]; System.arraycopy(writables, 0, blocks, 0, blocks.length); // add to the file tree - fsDir.unprotectedAddFile(name.toString(), blocks, replication); + fsDir.unprotectedAddFile(name.toString(), blocks, replication, mtime); break; } case OP_SET_REPLICATION: { @@ -277,23 +285,70 @@ break; } case OP_RENAME: { - UTF8 src = new UTF8(); - UTF8 dst = new UTF8(); - src.readFields(in); - dst.readFields(in); - fsDir.unprotectedRenameTo(src.toString(), dst.toString()); + UTF8 src = null; + UTF8 dst = null; + if (logVersion >= -4) { + src = new UTF8(); + dst = new UTF8(); + src.readFields(in); + dst.readFields(in); + } else { + ArrayWritable aw = null; + Writable writables[]; + aw = new ArrayWritable(UTF8.class); + aw.readFields(in); + writables = aw.get(); + if (writables.length != 3) { + throw new IOException("Incorrect data fortmat. " + + "Mkdir operation."); + } + src = (UTF8) writables[0]; + dst = (UTF8) writables[1]; + timestamp = Long.parseLong(((UTF8)writables[2]).toString()); + } + fsDir.unprotectedRenameTo(src.toString(), dst.toString(), timestamp); break; } case OP_DELETE: { - UTF8 src = new UTF8(); - src.readFields(in); - fsDir.unprotectedDelete(src.toString()); + UTF8 src = null; + if (logVersion >= -4) { + src = new UTF8(); + src.readFields(in); + } else { + ArrayWritable aw = null; + Writable writables[]; + aw = new ArrayWritable(UTF8.class); + aw.readFields(in); + writables = aw.get(); + if (writables.length != 2) { + throw new IOException("Incorrect data fortmat. " + + "delete operation."); + } + src = (UTF8) writables[0]; + timestamp = Long.parseLong(((UTF8)writables[1]).toString()); + } + fsDir.unprotectedDelete(src.toString(), timestamp); break; } case OP_MKDIR: { - UTF8 src = new UTF8(); - src.readFields(in); - fsDir.unprotectedMkdir(src.toString()); + UTF8 src = null; + if (logVersion >= -4) { + src = new UTF8(); + src.readFields(in); + } else { + ArrayWritable aw = null; + Writable writables[]; + aw = new ArrayWritable(UTF8.class); + aw.readFields(in); + writables = aw.get(); + if (writables.length != 2) { + throw new IOException("Incorrect data fortmat. " + + "Mkdir operation."); + } + src = (UTF8) writables[0]; + timestamp = Long.parseLong(((UTF8)writables[1]).toString()); + } + fsDir.unprotectedMkdir(src.toString(), timestamp); break; } case OP_DATANODE_ADD: { @@ -422,7 +477,8 @@ void logCreateFile(FSDirectory.INode newNode) { UTF8 nameReplicationPair[] = new UTF8[] { new UTF8(newNode.computeName()), - FSEditLog.toLogReplication(newNode.getReplication())}; + FSEditLog.toLogReplication(newNode.getReplication()), + FSEditLog.toLogTimeStamp(newNode.getModificationTime())}; logEdit(OP_ADD, new ArrayWritable(UTF8.class, nameReplicationPair), new ArrayWritable(Block.class, newNode.getBlocks())); @@ -432,15 +488,23 @@ * Add create directory record to edit log */ void logMkDir(FSDirectory.INode newNode) { - logEdit(OP_MKDIR, new UTF8(newNode.computeName()), null); + UTF8 info[] = new UTF8[] { + new UTF8(newNode.computeName()), + FSEditLog.toLogTimeStamp(newNode.getModificationTime()) + }; + logEdit(OP_MKDIR, new ArrayWritable(UTF8.class, info), null); } /** * Add rename record to edit log * TODO: use String parameters until just before writing to disk */ - void logRename(String src, String dst) { - logEdit(OP_RENAME, new UTF8(src), new UTF8(dst)); + void logRename(String src, String dst, long timestamp) { + UTF8 info[] = new UTF8[] { + new UTF8(src), + new UTF8(dst), + FSEditLog.toLogTimeStamp(timestamp)}; + logEdit(OP_RENAME, new ArrayWritable(UTF8.class, info), null); } /** @@ -455,8 +519,11 @@ /** * Add delete file record to edit log */ - void logDelete(String src) { - logEdit(OP_DELETE, new UTF8(src), null); + void logDelete(String src, long timestamp) { + UTF8 info[] = new UTF8[] { + new UTF8(src), + FSEditLog.toLogTimeStamp(timestamp)}; + logEdit(OP_DELETE, new ArrayWritable(UTF8.class, info), null); } /** @@ -481,6 +548,10 @@ static short fromLogReplication(UTF8 replication) { return Short.parseShort(replication.toString()); + } + + static UTF8 toLogTimeStamp(long timestamp) { + return new UTF8(Long.toString(timestamp)); } /** Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Fri Jun 22 15:55:34 2007 @@ -647,12 +647,16 @@ short replication = FSNamesystem.getFSNamesystem().getDefaultReplication(); for (int i = 0; i < numFiles; i++) { UTF8 name = new UTF8(); + long modificationTime = 0; name.readFields(in); // version 0 does not support per file replication if (!(imgVersion >= 0)) { replication = in.readShort(); // other versions do replication = FSEditLog.adjustReplication(replication); } + if (imgVersion <= -5) { + modificationTime = in.readLong(); + } int numBlocks = in.readInt(); Block blocks[] = null; if (numBlocks > 0) { @@ -662,7 +666,8 @@ blocks[j].readFields(in); } } - fsDir.unprotectedAddFile(name.toString(), blocks, replication); + fsDir.unprotectedAddFile(name.toString(), blocks, replication, + modificationTime); } // load datanode info @@ -784,6 +789,7 @@ fullName = parentPrefix + "/" + root.getLocalName(); new UTF8(fullName).write(out); out.writeShort(root.getReplication()); + out.writeLong(root.getModificationTime()); if (root.isDir()) { out.writeInt(0); } else { Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri Jun 22 15:55:34 2007 @@ -1169,6 +1169,15 @@ return dir.isDir(src); } + /* Get the file info for a specific file. + * @param src The string representation of the path to the file + * @throws IOException if file does not exist + * @return object containing information regarding the file + */ + DFSFileInfo getFileInfo(String src) throws IOException { + return dir.getFileInfo(src); + } + /** * Whether the pathname is valid. Currently prohibits relative paths, * and names which contain a ":" or "/" @@ -1213,7 +1222,7 @@ if (!isValidName(src)) { throw new IOException("Invalid directory name: " + src); } - success = dir.mkdirs(src); + success = dir.mkdirs(src, now()); if (!success) { throw new IOException("Invalid directory name: " + src); } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Fri Jun 22 15:55:34 2007 @@ -470,6 +470,15 @@ return files; } + /* Get the file info for a specific file. + * @param src The string representation of the path to the file + * @throws IOException if file does not exist + * @return object containing information regarding the file + */ + public DFSFileInfo getFileInfo(String src) throws IOException { + return namesystem.getFileInfo(src); + } + /** */ public long[] getStats() throws IOException { Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileStatus.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileStatus.java?view=auto&rev=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileStatus.java (added) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileStatus.java Fri Jun 22 15:55:34 2007 @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + + +import org.apache.hadoop.io.*; + +/** Interface that represents the client side information for a file. + */ +public interface FileStatus { + + /* + * @return the length of this file, in blocks + */ + public long getLen(); + + /** + * Is this a directory? + * @return true if this is a directory + */ + public boolean isDir(); + + /** + * Get the block size of the file. + * @return the number of bytes + */ + public long getBlockSize(); + + /** + * Get the replication factor of a file. + * @return the replication factor of a file. + */ + public short getReplication(); + + /** + * Get the modification time of the file. + * @return the modification time of file. + */ + public long getModificationTime(); +} Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Fri Jun 22 15:55:34 2007 @@ -368,11 +368,15 @@ /** * Get replication. * + * @deprecated Use getFileStatus() instead * @param src file name * @return file replication * @throws IOException - */ - public abstract short getReplication(Path src) throws IOException; + */ + @Deprecated + public short getReplication(Path src) throws IOException { + return getFileStatus(src).getReplication(); + } /** * Set replication for an existing file. @@ -400,7 +404,14 @@ public abstract boolean exists(Path f) throws IOException; /** True iff the named path is a directory. */ - public abstract boolean isDirectory(Path f) throws IOException; + /** @deprecated Use getFileStatus() instead */ @Deprecated + public boolean isDirectory(Path f) throws IOException { + try { + return getFileStatus(f).isDir(); + } catch (IOException e) { + return false; // f does not exist + } + } /** True iff the named path is a regular file. */ public boolean isFile(Path f) throws IOException { @@ -412,7 +423,10 @@ } /** The number of bytes in a file. */ - public abstract long getLength(Path f) throws IOException; + /** @deprecated Use getFileStatus() instead */ @Deprecated + public long getLength(Path f) throws IOException { + return getFileStatus(f).getLen(); + } /** Return the number of bytes of the given path * If <i>f</i> is a file, return the size of the file; @@ -817,7 +831,10 @@ * @param f the filename * @return the number of bytes in a block */ - public abstract long getBlockSize(Path f) throws IOException; + /** @deprecated Use getFileStatus() instead */ @Deprecated + public long getBlockSize(Path f) throws IOException { + return getFileStatus(f).getBlockSize(); + } /** Return the number of bytes that large input files should be optimally * be split into to minimize i/o time. */ @@ -831,4 +848,12 @@ */ public abstract short getDefaultReplication(); + /* + * Return a file status object that represents the + * file. + * @param f The path to the file we want information from + * @return filestatus object + * @throws IOException see specific implementation + */ + public abstract FileStatus getFileStatus(Path f) throws IOException; } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Fri Jun 22 15:55:34 2007 @@ -124,17 +124,6 @@ } /** - * Get replication. - * - * @param src file name - * @return file replication - * @throws IOException - */ - public short getReplication(Path src) throws IOException { - return fs.getReplication(src); - } - - /** * Set replication for an existing file. * * @param src file name @@ -167,16 +156,6 @@ return fs.exists(f); } - /** True iff the named path is a directory. */ - public boolean isDirectory(Path f) throws IOException { - return fs.isDirectory(f); - } - - /** The number of bytes in a file. */ - public long getLength(Path f) throws IOException { - return fs.getLength(f); - } - /** List files in a directory. */ public Path[] listPaths(Path f) throws IOException { return fs.listPaths(f); @@ -272,15 +251,6 @@ fs.completeLocalOutput(fsOutputFile, tmpLocalFile); } - /** - * Get the block size for a particular file. - * @param f the filename - * @return the number of bytes in a block - */ - public long getBlockSize(Path f) throws IOException { - return fs.getBlockSize(f); - } - /** Return the number of bytes that large input files should be optimally * be split into to minimize i/o time. */ public long getDefaultBlockSize() { @@ -292,6 +262,13 @@ */ public short getDefaultReplication() { return fs.getDefaultReplication(); + } + + /** + * Get file status. + */ + public FileStatus getFileStatus(Path f) throws IOException { + return fs.getFileStatus(f); } @Override Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java Fri Jun 22 15:55:34 2007 @@ -19,6 +19,7 @@ import java.io.*; import java.util.*; +import java.text.SimpleDateFormat; import org.apache.hadoop.conf.*; import org.apache.hadoop.ipc.*; @@ -29,6 +30,8 @@ protected FileSystem fs; private Trash trash; + public static final SimpleDateFormat dateForm = + new SimpleDateFormat("yyyy-MM-dd HH:mm"); /** */ @@ -405,12 +408,15 @@ } for (int i = 0; i < items.length; i++) { Path cur = items[i]; + FileStatus stat = fs.getFileStatus(cur); + String mdate = dateForm.format(new Date(stat.getModificationTime())); System.out.println(cur.toUri().getPath() + "\t" - + (fs.isDirectory(cur) ? - "<dir>" : - ("<r " + fs.getReplication(cur) - + ">\t" + fs.getLength(cur)))); - if (recursive && fs.isDirectory(cur)) { + + (stat.isDir() ? + "<dir>\t" : + ("<r " + stat.getReplication() + + ">\t" + stat.getLen())) + + "\t" + mdate); + if (recursive && stat.isDir()) { ls(cur, recursive, printHeader); } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java Fri Jun 22 15:55:34 2007 @@ -233,7 +233,7 @@ /** * Replication is not supported for the inmemory file system. */ - public short getReplication(Path src) throws IOException { + public short getDefaultReplication() { return 1; } @@ -275,23 +275,6 @@ /** * Directory operations are not supported */ - public boolean isDirectory(Path f) throws IOException { - return !isFile(f); - } - - public boolean isFile(Path f) throws IOException { - return exists(f); - } - - public long getLength(Path f) throws IOException { - synchronized (this) { - return pathToFileAttribs.get(getPath(f)).size; - } - } - - /** - * Directory operations are not supported - */ public Path[] listPaths(Path f) throws IOException { return null; } @@ -330,16 +313,14 @@ throws IOException { } - public long getBlockSize(Path f) throws IOException { - return getDefaultBlockSize(); - } - public long getDefaultBlockSize() { return 32 * 1024; //some random large number. can be anything actually } - public short getDefaultReplication() { - return 1; + public FileStatus getFileStatus(Path f) throws IOException { + synchronized (this) { + return new InMemoryFileStatus(pathToFileAttribs.get(getPath(f))); + } } /** Some APIs exclusively for InMemoryFileSystem */ @@ -428,6 +409,29 @@ public FileAttributes(int size) { this.size = size; this.data = new byte[size]; + } + } + + private class InMemoryFileStatus implements FileStatus { + private long length; + + InMemoryFileStatus(FileAttributes attr) throws IOException { + length = attr.size; + } + public long getLen() { + return length; + } + public boolean isDir() { + return false; + } + public long getBlockSize() { + return getDefaultBlockSize(); + } + public short getReplication() { + return 1; + } + public long getModificationTime() { + return 0; // not supported yet } } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Fri Jun 22 15:55:34 2007 @@ -195,10 +195,6 @@ /** * Replication is not supported for the local file system. */ - public short getReplication(Path f) throws IOException { - return 1; - } - /** Set the replication of the given file */ public boolean setReplication(Path src, short replication @@ -223,14 +219,6 @@ return pathToFile(f).exists(); } - public boolean isDirectory(Path f) throws IOException { - return pathToFile(f).isDirectory(); - } - - public long getLength(Path f) throws IOException { - return pathToFile(f).length(); - } - public Path[] listPaths(Path f) throws IOException { File localf = pathToFile(f); Path[] results; @@ -366,12 +354,38 @@ return "LocalFS"; } - public long getBlockSize(Path filename) { - // local doesn't really do blocks, so just use the global number - return getDefaultBlockSize(); + public FileStatus getFileStatus(Path f) throws IOException { + return new RawLocalFileStatus(pathToFile(f)); } - + public short getDefaultReplication() { return 1; + } + + private class RawLocalFileStatus implements FileStatus { + private long length; + private boolean isDir; + private long mtime; + + RawLocalFileStatus(File f) throws IOException { + length = f.length(); + isDir = f.isDirectory(); + mtime = f.lastModified(); + } + public long getLen() { + return length; + } + public boolean isDir() { + return isDir; + } + public long getBlockSize() { + return getDefaultBlockSize(); + } + public short getReplication() { + return 1; + } + public long getModificationTime() { + return mtime; + } } } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Fri Jun 22 15:55:34 2007 @@ -13,6 +13,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; @@ -122,15 +123,6 @@ } @Override - public boolean isDirectory(Path path) throws IOException { - INode inode = store.retrieveINode(makeAbsolute(path)); - if (inode == null) { - return false; - } - return inode.isDirectory(); - } - - @Override public boolean isFile(Path path) throws IOException { INode inode = store.retrieveINode(makeAbsolute(path)); if (inode == null) { @@ -270,28 +262,25 @@ return true; } - @Override - public long getLength(Path path) throws IOException { - INode inode = checkFile(path); - long length = 0; - for (Block block : inode.getBlocks()) { - length += block.getLength(); - } - return length; - } - /** * Replication is not supported for S3 file systems since S3 handles it for * us. */ @Override - public short getReplication(Path path) throws IOException { + public short getDefaultReplication() { return 1; } + /** + * FileStatus for S3 file systems. + */ @Override - public short getDefaultReplication() { - return 1; + public FileStatus getFileStatus(Path f) throws IOException { + INode inode = store.retrieveINode(makeAbsolute(f)); + if (inode == null) { + throw new IOException(f.toString() + ": No such file or directory."); + } + return new S3FileStatus(inode); } /** @@ -305,19 +294,6 @@ } @Override - public long getBlockSize(Path path) throws IOException { - INode inode = store.retrieveINode(makeAbsolute(path)); - if (inode == null) { - throw new IOException(path.toString() + ": No such file or directory."); - } - Block[] blocks = inode.getBlocks(); - if (blocks == null || blocks.length == 0) { - return 0; - } - return blocks[0].getLength(); - } - - @Override public long getDefaultBlockSize() { return getConf().getLong("fs.s3.block.size", DEFAULT_BLOCK_SIZE); } @@ -379,4 +355,35 @@ store.purge(); } + private static class S3FileStatus implements FileStatus { + private long length = 0, blockSize = 0; + private boolean isDir; + + S3FileStatus(INode inode) throws IOException { + isDir = inode.isDirectory(); + if (!isDir) { + for (Block block : inode.getBlocks()) { + length += block.getLength(); + if (blockSize == 0) { + blockSize = block.getLength(); + } + } + } + } + public long getLen() { + return length; + } + public boolean isDir() { + return isDir; + } + public long getBlockSize() { + return blockSize; + } + public short getReplication() { + return 1; + } + public long getModificationTime() { + return 0; // not supported yet + } + } } Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestModTime.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestModTime.java?view=auto&rev=549977 ============================================================================== --- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestModTime.java (added) +++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestModTime.java Fri Jun 22 15:55:34 2007 @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.dfs; + +import junit.framework.TestCase; +import java.io.*; +import java.util.Random; +import java.net.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; + +/** + * This class tests the decommissioning of nodes. + * @author Dhruba Borthakur + */ +public class TestModTime extends TestCase { + static final long seed = 0xDEADBEEFL; + static final int blockSize = 8192; + static final int fileSize = 16384; + static final int numDatanodes = 6; + + + Random myrand = new Random(); + Path hostsFile; + Path excludeFile; + + private void writeFile(FileSystem fileSys, Path name, int repl) + throws IOException { + // create and write a file that contains three blocks of data + FSDataOutputStream stm = fileSys.create(name, true, + fileSys.getConf().getInt("io.file.buffer.size", 4096), + (short)repl, (long)blockSize); + byte[] buffer = new byte[fileSize]; + Random rand = new Random(seed); + rand.nextBytes(buffer); + stm.write(buffer); + stm.close(); + } + + private void cleanupFile(FileSystem fileSys, Path name) throws IOException { + assertTrue(fileSys.exists(name)); + fileSys.delete(name); + assertTrue(!fileSys.exists(name)); + } + + private void printDatanodeReport(DatanodeInfo[] info) { + System.out.println("-------------------------------------------------"); + for (int i = 0; i < info.length; i++) { + System.out.println(info[i].getDatanodeReport()); + System.out.println(); + } + } + + /** + * Tests modification time in DFS. + */ + public void testModTime() throws IOException { + Configuration conf = new Configuration(); + + MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, true, null); + cluster.waitActive(); + InetSocketAddress addr = new InetSocketAddress("localhost", + cluster.getNameNodePort()); + DFSClient client = new DFSClient(addr, conf); + DatanodeInfo[] info = client.datanodeReport(); + assertEquals("Number of Datanodes ", numDatanodes, info.length); + FileSystem fileSys = cluster.getFileSystem(); + int replicas = numDatanodes - 1; + assertTrue(fileSys instanceof DistributedFileSystem); + + try { + + // + // create file and record ctime and mtime of test file + // + System.out.println("Creating testdir1 and testdir1/test1.dat."); + Path dir1 = new Path("testdir1"); + Path file1 = new Path(dir1, "test1.dat"); + writeFile(fileSys, file1, replicas); + FileStatus stat = fileSys.getFileStatus(file1); + long mtime1 = stat.getModificationTime(); + assertTrue(mtime1 != 0); + // + // record dir times + // + stat = fileSys.getFileStatus(dir1); + long mdir1 = stat.getModificationTime(); + + // + // create second test file + // + System.out.println("Creating testdir1/test2.dat."); + Path file2 = new Path(dir1, "test2.dat"); + writeFile(fileSys, file2, replicas); + stat = fileSys.getFileStatus(file2); + + // + // verify that mod time of dir remains the same + // as before. modification time of directory has increased. + // + stat = fileSys.getFileStatus(dir1); + assertTrue(stat.getModificationTime() >= mdir1); + mdir1 = stat.getModificationTime(); + // + // create another directory + // + Path dir2 = (new Path("testdir2/")).makeQualified(fileSys); + System.out.println("Creating testdir2 " + dir2); + assertTrue(fileSys.mkdirs(dir2)); + stat = fileSys.getFileStatus(dir2); + long mdir2 = stat.getModificationTime(); + // + // rename file1 from testdir into testdir2 + // + Path newfile = new Path(dir2, "testnew.dat"); + System.out.println("Moving " + file1 + " to " + newfile); + fileSys.rename(file1, newfile); + // + // verify that modification time of file1 did not change. + // + stat = fileSys.getFileStatus(newfile); + assertTrue(stat.getModificationTime() == mtime1); + // + // verify that modification time of testdir1 and testdir2 + // were changed. + // + stat = fileSys.getFileStatus(dir1); + assertTrue(stat.getModificationTime() != mdir1); + mdir1 = stat.getModificationTime(); + + stat = fileSys.getFileStatus(dir2); + assertTrue(stat.getModificationTime() != mdir2); + mdir2 = stat.getModificationTime(); + // + // delete newfile + // + System.out.println("Deleting testdir2/testnew.dat."); + assertTrue(fileSys.delete(newfile)); + // + // verify that modification time of testdir1 has not changed. + // + stat = fileSys.getFileStatus(dir1); + assertTrue(stat.getModificationTime() == mdir1); + // + // verify that modification time of testdir2 has changed. + // + stat = fileSys.getFileStatus(dir2); + assertTrue(stat.getModificationTime() != mdir2); + mdir2 = stat.getModificationTime(); + + cleanupFile(fileSys, file2); + cleanupFile(fileSys, dir1); + cleanupFile(fileSys, dir2); + } catch (IOException e) { + info = client.datanodeReport(); + printDatanodeReport(info); + throw e; + } finally { + fileSys.close(); + cluster.shutdown(); + } + } +} Modified: lucene/hadoop/trunk/src/webapps/datanode/browseDirectory.jsp URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/datanode/browseDirectory.jsp?view=diff&rev=549977&r1=549976&r2=549977 ============================================================================== --- lucene/hadoop/trunk/src/webapps/datanode/browseDirectory.jsp (original) +++ lucene/hadoop/trunk/src/webapps/datanode/browseDirectory.jsp Fri Jun 22 15:55:34 2007 @@ -69,9 +69,10 @@ // directory DFSFileInfo[] files = dfs.listPaths(target); //generate a table and dump the info - String [] headings = new String[5]; + String [] headings = new String[7]; headings[0] = "Name"; headings[1] = "Type"; headings[2] = "Size"; headings[3] = "Replication"; headings[4] = "BlockSize"; + headings[5] = "Creation Time"; headings[6] = "Modification Time"; out.print("<h3>Contents of directory "); JspHelper.printPathWithLinks(dir, out, namenodeInfoPort); out.print("</h3><hr>"); @@ -92,7 +93,7 @@ jspHelper.addTableHeader(out); int row=0; jspHelper.addTableRow(out, headings, row++); - String cols [] = new String[5]; + String cols [] = new String[6]; for (int i = 0; i < files.length; i++) { //Get the location of the first block of the file if (files[i].getPath().endsWith(".crc")) continue; @@ -113,6 +114,7 @@ cols[2] = FsShell.byteDesc(files[i].getLen()); cols[3] = Short.toString(files[i].getReplication()); cols[4] = FsShell.byteDesc(files[i].getBlockSize()); + cols[5] = FsShell.dateForm.format(new Date((files[i].getModificationTime()))); } else { String datanodeUrl = req.getRequestURL()+"?dir="+ @@ -123,6 +125,7 @@ cols[2] = ""; cols[3] = ""; cols[4] = ""; + cols[5] = FsShell.dateForm.format(new Date((files[i].getModificationTime()))); } jspHelper.addTableRow(out, cols, row++); }