Author: szetszwo
Date: Tue Sep 15 21:23:05 2009
New Revision: 815496
URL: http://svn.apache.org/viewvc?rev=815496&view=rev
Log:
HDFS-617. Support non-recursive create(). Contributed by Kan Zhang
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Sep 15 21:23:05 2009
@@ -155,6 +155,8 @@
HDFS-472. Update hdfsproxy documentation. Adds a setup guide and design
document. (Zhiyong Zhang via cdouglas)
+ HDFS-617. Support non-recursive create(). (Kan Zhang via szetszwo)
+
BUG FIXES
HDFS-76. Better error message to users when commands fail because of
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java Tue Sep 15
21:23:05 2009
@@ -522,6 +522,23 @@
}
/**
+ * Call
+ * {...@link
#create(String,FsPermission,EnumSet,boolean,short,long,Progressable,int)}
+ * with createParent set to true.
+ */
+ public OutputStream create(String src,
+ FsPermission permission,
+ EnumSet<CreateFlag> flag,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ int buffersize
+ ) throws IOException {
+ return create(src, permission, flag, true,
+ replication, blockSize, progress, buffersize);
+ }
+
+ /**
* Create a new dfs file with the specified block replication
* with write-progress reporting and return an output stream for writing
* into the file.
@@ -530,14 +547,16 @@
* @param permission The permission of the directory being created.
* If permission == null, use {...@link FsPermission#getDefault()}.
* @param flag do not check for file existence if true
+ * @param createParent create missing parent directory if true
* @param replication block replication
* @return output stream
* @throws IOException
- * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
short, long)
+ * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
boolean, short, long)
*/
public OutputStream create(String src,
FsPermission permission,
EnumSet<CreateFlag> flag,
+ boolean createParent,
short replication,
long blockSize,
Progressable progress,
@@ -550,7 +569,7 @@
FsPermission masked = permission.applyUMask(FsPermission.getUMask(conf));
LOG.debug(src + ": masked=" + masked);
OutputStream result = new DFSOutputStream(src, masked,
- flag, replication, blockSize, progress, buffersize,
+ flag, createParent, replication, blockSize, progress, buffersize,
conf.getInt("io.bytes.per.checksum", 512));
leasechecker.put(src, result);
return result;
@@ -3068,10 +3087,10 @@
/**
* Create a new output stream to the given DataNode.
- * @see ClientProtocol#create(String, FsPermission, String, boolean,
short, long)
+ * @see ClientProtocol#create(String, FsPermission, String,
EnumSetWritable, boolean, short, long)
*/
DFSOutputStream(String src, FsPermission masked, EnumSet<CreateFlag> flag,
- short replication, long blockSize, Progressable progress,
+ boolean createParent, short replication, long blockSize, Progressable
progress,
int buffersize, int bytesPerChecksum) throws IOException {
this(src, blockSize, progress, bytesPerChecksum);
@@ -3079,9 +3098,11 @@
try {
namenode.create(
- src, masked, clientName, new EnumSetWritable<CreateFlag>(flag),
replication, blockSize);
+ src, masked, clientName, new EnumSetWritable<CreateFlag>(flag),
createParent, replication, blockSize);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
+ FileAlreadyExistsException.class,
+ FileNotFoundException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class);
}
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
Tue Sep 15 21:23:05 2009
@@ -209,6 +209,18 @@
statistics);
}
+ /**
+ * Same as create(), except fails if parent directory doesn't already exist.
+ * @see #create(Path, FsPermission, EnumSet, int, short, long, Progressable)
+ */
+ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
+ EnumSet<CreateFlag> flag, int bufferSize, short replication,
+ long blockSize, Progressable progress) throws IOException {
+
+ return new FSDataOutputStream(dfs.create(getPathName(f), permission, flag,
+ false, replication, blockSize, progress, bufferSize), statistics);
+ }
+
@Override
public boolean setReplication(Path src,
short replication
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
Tue Sep 15 21:23:05 2009
@@ -44,9 +44,9 @@
* Compared to the previous version the following changes have been
introduced:
* (Only the latest change is reflected.
* The log of historical changes can be retrieved from the svn).
- * 46: added a new method getServerDefaults(), see HDFS-578
+ * 47: create() takes an additional boolean param createParent.
*/
- public static final long versionID = 46L;
+ public static final long versionID = 47L;
///////////////////////////////////////
// File contents
@@ -101,6 +101,7 @@
* @param clientName name of the current client.
* @param flag indicates whether the file should be
* overwritten if it already exists or create if it does not exist or append.
+ * @param createParent create missing parent directory if true
* @param replication block replication factor.
* @param blockSize maximum block size.
*
@@ -115,6 +116,7 @@
FsPermission masked,
String clientName,
EnumSetWritable<CreateFlag> flag,
+ boolean createParent,
short replication,
long blockSize
) throws IOException;
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Sep 15 21:23:05 2009
@@ -54,6 +54,7 @@
import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
@@ -813,6 +814,24 @@
return dir.getPreferredBlockSize(filename);
}
+ /*
+ * Verify that parent dir exists
+ */
+ private void verifyParentDir(String src) throws FileAlreadyExistsException,
+ FileNotFoundException {
+ Path parent = new Path(src).getParent();
+ if (parent != null) {
+ INode[] pathINodes = dir.getExistingPathINodes(parent.toString());
+ if (pathINodes[pathINodes.length - 1] == null) {
+ throw new FileNotFoundException("Parent directory doesn't exist: "
+ + parent.toString());
+ } else if (!pathINodes[pathINodes.length - 1].isDirectory()) {
+ throw new FileAlreadyExistsException("Parent path is not a directory: "
+ + parent.toString());
+ }
+ }
+ }
+
/**
* Create a new file entry in the namespace.
*
@@ -823,10 +842,11 @@
*/
void startFile(String src, PermissionStatus permissions,
String holder, String clientMachine,
- EnumSet<CreateFlag> flag, short replication, long blockSize
+ EnumSet<CreateFlag> flag, boolean createParent,
+ short replication, long blockSize
) throws IOException {
startFileInternal(src, permissions, holder, clientMachine, flag,
- replication, blockSize);
+ createParent, replication, blockSize);
getEditLog().logSync();
if (auditLog.isInfoEnabled()) {
final FileStatus stat = dir.getFileInfo(src);
@@ -841,6 +861,7 @@
String holder,
String clientMachine,
EnumSet<CreateFlag> flag,
+ boolean createParent,
short replication,
long blockSize
) throws IOException {
@@ -852,6 +873,7 @@
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: src=" + src
+ ", holder=" + holder
+ ", clientMachine=" + clientMachine
+ + ", createParent=" + createParent
+ ", replication=" + replication
+ ", overwrite=" + overwrite
+ ", append=" + append);
@@ -878,6 +900,10 @@
}
}
+ if (!createParent) {
+ verifyParentDir(src);
+ }
+
try {
INode myFile = dir.getFileINode(src);
if (myFile != null && myFile.isUnderConstruction()) {
@@ -935,7 +961,7 @@
else {
//append & create a nonexist file equals to overwrite
this.startFileInternal(src, permissions, holder, clientMachine,
- EnumSet.of(CreateFlag.OVERWRITE), replication, blockSize);
+ EnumSet.of(CreateFlag.OVERWRITE), createParent, replication,
blockSize);
return;
}
} else if (myFile.isDirectory()) {
@@ -1011,7 +1037,7 @@
" Please refer to dfs.support.append configuration
parameter.");
}
startFileInternal(src, null, holder, clientMachine,
EnumSet.of(CreateFlag.APPEND),
- (short)blockManager.maxReplication, (long)0);
+ false, (short)blockManager.maxReplication, (long)0);
getEditLog().logSync();
//
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Tue Sep 15 21:23:05 2009
@@ -556,6 +556,7 @@
FsPermission masked,
String clientName,
EnumSetWritable<CreateFlag> flag,
+ boolean createParent,
short replication,
long blockSize
) throws IOException {
@@ -571,7 +572,7 @@
namesystem.startFile(src,
new
PermissionStatus(UserGroupInformation.getCurrentUGI().getUserName(),
null, masked),
- clientName, clientMachine, flag.get(), replication, blockSize);
+ clientName, clientMachine, flag.get(), createParent, replication,
blockSize);
myMetrics.numFilesCreated.inc();
myMetrics.numCreateFileOps.inc();
}
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
Tue Sep 15 21:23:05 2009
@@ -157,7 +157,7 @@
public FsServerDefaults getServerDefaults() throws IOException { return
null; }
- public void create(String src, FsPermission masked, String clientName,
EnumSetWritable<CreateFlag> flag, short replication, long blockSize) throws
IOException {}
+ public void create(String src, FsPermission masked, String clientName,
EnumSetWritable<CreateFlag> flag, boolean createParent, short replication, long
blockSize) throws IOException {}
public LocatedBlock append(String src, String clientName) throws
IOException { return null; }
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
Tue Sep 15 21:23:05 2009
@@ -19,6 +19,7 @@
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -30,6 +31,7 @@
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
@@ -735,6 +737,99 @@
}
}
+ /**
+ * Test file creation using createNonRecursive().
+ */
+ public void testFileCreationNonRecursive() throws IOException {
+ Configuration conf = new Configuration();
+ if (simulatedStorage) {
+ conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
+ }
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
+ FileSystem fs = cluster.getFileSystem();
+ final Path path = new Path("/" + System.currentTimeMillis()
+ + "-testFileCreationNonRecursive");
+ FSDataOutputStream out = null;
+
+ try {
+ IOException expectedException = null;
+ final String nonExistDir = "/non-exist-" + System.currentTimeMillis();
+
+ fs.delete(new Path(nonExistDir), true);
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ // Create a new file in root dir, should succeed
+ out = createNonRecursive(fs, path, 1, createFlag);
+ out.close();
+ // Create a file when parent dir exists as file, should fail
+ expectedException = null;
+ try {
+ createNonRecursive(fs, new Path(path, "Create"), 1, createFlag);
+ } catch (IOException e) {
+ expectedException = e;
+ }
+ assertTrue("Create a file when parent directory exists as a file"
+ + " should throw FileAlreadyExistsException ",
+ expectedException != null
+ && expectedException instanceof FileAlreadyExistsException);
+ fs.delete(path, true);
+ // Create a file in a non-exist directory, should fail
+ final Path path2 = new Path(nonExistDir + "/testCreateNonRecursive");
+ expectedException = null;
+ try {
+ createNonRecursive(fs, path2, 1, createFlag);
+ } catch (IOException e) {
+ expectedException = e;
+ }
+ assertTrue("Create a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+
+ EnumSet<CreateFlag> overwriteFlag = EnumSet.of(CreateFlag.OVERWRITE);
+ // Overwrite a file in root dir, should succeed
+ out = createNonRecursive(fs, path, 1, overwriteFlag);
+ out.close();
+ // Overwrite a file when parent dir exists as file, should fail
+ expectedException = null;
+ try {
+ createNonRecursive(fs, new Path(path, "Overwrite"), 1, overwriteFlag);
+ } catch (IOException e) {
+ expectedException = e;
+ }
+ assertTrue("Overwrite a file when parent directory exists as a file"
+ + " should throw FileAlreadyExistsException ",
+ expectedException != null
+ && expectedException instanceof FileAlreadyExistsException);
+ fs.delete(path, true);
+ // Overwrite a file in a non-exist directory, should fail
+ final Path path3 = new Path(nonExistDir + "/testOverwriteNonRecursive");
+ expectedException = null;
+ try {
+ createNonRecursive(fs, path3, 1, overwriteFlag);
+ } catch (IOException e) {
+ expectedException = e;
+ }
+ assertTrue("Overwrite a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+ } finally {
+ fs.close();
+ cluster.shutdown();
+ }
+ }
+
+ // creates a file using DistributedFileSystem.createNonRecursive()
+ static FSDataOutputStream createNonRecursive(FileSystem fs, Path name,
+ int repl, EnumSet<CreateFlag> flag) throws IOException {
+ System.out.println("createNonRecursive: Created " + name + " with " + repl
+ + " replica.");
+ FSDataOutputStream stm = ((DistributedFileSystem) fs).createNonRecursive(
+ name, FsPermission.getDefault(), flag, fs.getConf().getInt(
+ "io.file.buffer.size", 4096), (short) repl, (long) blockSize,
null);
+ return stm;
+ }
+
// creates a file with the flag api
static FSDataOutputStream createFileWithFlag(FileSystem fileSys, Path name,
int repl, EnumSet<CreateFlag> flag)
throws IOException {
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=815496&r1=815495&r2=815496&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
Tue Sep 15 21:23:05 2009
@@ -516,7 +516,7 @@
// dummyActionNoSynch(fileIdx);
nameNode.create(fileNames[daemonId][inputIdx], FsPermission.getDefault(),
clientName, new EnumSetWritable<CreateFlag>(EnumSet
- .of(CreateFlag.OVERWRITE)), replication, BLOCK_SIZE);
+ .of(CreateFlag.OVERWRITE)), true, replication, BLOCK_SIZE);
long end = System.currentTimeMillis();
for(boolean written = !closeUponCreate; !written;
written = nameNode.complete(fileNames[daemonId][inputIdx],
clientName));
@@ -887,7 +887,7 @@
for(int idx=0; idx < nrFiles; idx++) {
String fileName = nameGenerator.getNextFileName("ThroughputBench");
nameNode.create(fileName, FsPermission.getDefault(), clientName,
- new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.OVERWRITE)),
replication,
+ new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.OVERWRITE)),
true, replication,
BLOCK_SIZE);
addBlocks(fileName, clientName);
nameNode.complete(fileName, clientName);
