Author: boryas
Date: Mon Jan 4 22:14:34 2010
New Revision: 895801
URL: http://svn.apache.org/viewvc?rev=895801&view=rev
Log:
HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
file with secret keys to a map reduce job. (boryas)
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=895801&r1=895800&r2=895801&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Mon Jan 4 22:14:34 2010
@@ -76,6 +76,9 @@
HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran
via tomwhite)
+
+ HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
+ file with secret keys to a map reduce job. (boryas)
OPTIMIZATIONS
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java?rev=895801&r1=895800&r2=895801&view=diff
==============================================================================
---
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
(original)
+++
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
Mon Jan 4 22:14:34 2010
@@ -233,6 +233,12 @@
.withDescription("comma separated archives to be unarchived" +
" on the compute machines.")
.create("archives");
+
+ // file with security tokens
+ Option tokensFile = OptionBuilder.withArgName("tokensFile")
+ .hasArg()
+ .withDescription("name of the file with the tokens")
+ .create("tokenCacheFile");
opts.addOption(fs);
opts.addOption(jt);
@@ -241,6 +247,7 @@
opts.addOption(libjars);
opts.addOption(files);
opts.addOption(archives);
+ opts.addOption(tokensFile);
return opts;
}
@@ -295,6 +302,19 @@
}
}
conf.setBoolean("mapred.used.genericoptionsparser", true);
+
+ // tokensFile
+ if(line.hasOption("tokenCacheFile")) {
+ String fileName = line.getOptionValue("tokenCacheFile");
+ // check if the local file exists
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Path p = new Path(fileName);
+ if (!localFs.exists(p)) {
+ throw new FileNotFoundException("File "+fileName+" does not exist.");
+ }
+ LOG.debug("setting conf tokensFile: " + fileName);
+ conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
+ }
}
/**
Modified:
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java?rev=895801&r1=895800&r2=895801&view=diff
==============================================================================
---
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
(original)
+++
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
Mon Jan 4 22:14:34 2010
@@ -19,22 +19,23 @@
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.net.URI;
+import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import junit.framework.TestCase;
-
public class TestGenericOptionsParser extends TestCase {
- private static File testDir =
- new File(System.getProperty("test.build.data", "/tmp"), "generic");
+ File testDir;
+ Configuration conf;
+ FileSystem localFs;
+
public void testFilesOption() throws Exception {
- Configuration conf = new Configuration();
File tmpFile = new File(testDir, "tmpfile");
- FileSystem localFs = FileSystem.getLocal(conf);
Path tmpPath = new Path(tmpFile.toString());
localFs.create(tmpPath);
String[] args = new String[2];
@@ -74,7 +75,62 @@
th instanceof FileNotFoundException);
files = conf2.get("tmpfiles");
assertNull("files is not null", files);
- testDir.delete();
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ conf = new Configuration();
+ localFs = FileSystem.getLocal(conf);
+ testDir = new File(System.getProperty("test.build.data", "/tmp"),
"generic");
+ if(testDir.exists())
+ localFs.delete(new Path(testDir.toString()), true);
}
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ if(testDir.exists()) {
+ localFs.delete(new Path(testDir.toString()), true);
+ }
+ }
+
+ /**
+ * testing -fileCache option
+ * @throws IOException
+ */
+ public void testTokenCacheOption() throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+
+ File tmpFile = new File(testDir, "tokenCacheFile");
+ if(tmpFile.exists()) {
+ tmpFile.delete();
+ }
+ String[] args = new String[2];
+ // pass a files option
+ args[0] = "-tokenCacheFile";
+ args[1] = tmpFile.toString();
+
+ // test non existing file
+ Throwable th = null;
+ try {
+ new GenericOptionsParser(conf, args);
+ } catch (Exception e) {
+ th = e;
+ }
+ assertNotNull(th);
+ assertTrue("FileNotFoundException is not thrown",
+ th instanceof FileNotFoundException);
+
+ // create file
+ Path tmpPath = new Path(tmpFile.toString());
+ localFs.create(tmpPath);
+ new GenericOptionsParser(conf, args);
+ String fileName = conf.get("tokenCacheFile");
+ assertNotNull("files is null", fileName);
+ assertEquals("files option does not match",
+ localFs.makeQualified(tmpPath).toString(), fileName);
+
+ localFs.delete(new Path(testDir.getAbsolutePath()), true);
+ }
}
