HADOOP-15885. Add base64 (urlString) support to DTUtil. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44e37b4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44e37b4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44e37b4f Branch: refs/heads/HDDS-4 Commit: 44e37b4fd9f441becf536368a89436afcd6dede8 Parents: 6d9c18c Author: Giovanni Matteo Fumarola <gif...@apache.org> Authored: Fri Nov 2 10:54:12 2018 -0700 Committer: Giovanni Matteo Fumarola <gif...@apache.org> Committed: Fri Nov 2 10:54:12 2018 -0700 ---------------------------------------------------------------------- .../hadoop/security/token/DtFileOperations.java | 28 ++++++++++++- .../hadoop/security/token/DtUtilShell.java | 37 +++++++++++++++- .../src/site/markdown/CommandsManual.md | 1 + .../hadoop/security/token/TestDtUtilShell.java | 44 ++++++++++++++++++++ 4 files changed, 107 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java index f154f2d..5f74f8b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java @@ -89,7 +89,7 @@ public final class DtFileOperations { /** Add the service prefix for a local filesystem. */ private static Path fileToPath(File f) { - return new Path("file:" + f.getAbsolutePath()); + return new Path(f.toURI().toString()); } /** Write out a Credentials object as a local file. @@ -294,4 +294,30 @@ public final class DtFileOperations { } doFormattedWrite(tokenFile, fileFormat, creds, conf); } + + /** Import a token from a base64 encoding into the local filesystem. + * @param tokenFile A local File object. + * @param fileFormat A string equal to FORMAT_PB or FORMAT_JAVA, for output. + * @param alias overwrite Service field of fetched token with this text. + * @param base64 urlString Encoding of the token to import. + * @param conf Configuration object passed along. + * @throws IOException Error to import the token into the file. + */ + public static void importTokenFile(File tokenFile, String fileFormat, + Text alias, String base64, Configuration conf) + throws IOException { + + Credentials creds = tokenFile.exists() ? + Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials(); + + Token<TokenIdentifier> token = new Token<>(); + token.decodeFromUrlString(base64); + if (alias != null) { + token.setService(alias); + } + creds.addToken(token.getService(), token); + LOG.info("Add token with service {}", token.getService()); + + doFormattedWrite(tokenFile, fileFormat, creds, conf); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java index 88db34f..bc2d1b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java @@ -55,6 +55,7 @@ public class DtUtilShell extends CommandShell { private static final String CANCEL = "cancel"; private static final String REMOVE = "remove"; private static final String RENEW = "renew"; + private static final String IMPORT = "import"; private static final String RENEWER = "-renewer"; private static final String SERVICE = "-service"; private static final String ALIAS = "-alias"; @@ -138,6 +139,8 @@ public class DtUtilShell extends CommandShell { setSubCommand(new Remove(false)); } else if (command.equals(RENEW)) { setSubCommand(new Renew()); + } else if (command.equals(IMPORT)) { + setSubCommand(new Import(args[++i])); } } else if (args[i].equals(ALIAS)) { alias = new Text(args[++i]); @@ -176,11 +179,11 @@ public class DtUtilShell extends CommandShell { @Override public String getCommandUsage() { return String.format( - "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n", + "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n", DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(), (new Edit()).getUsage(), (new Append()).getUsage(), (new Remove(true)).getUsage(), (new Remove(false)).getUsage(), - (new Renew()).getUsage()); + (new Renew()).getUsage(), (new Import()).getUsage()); } private class Print extends SubCommand { @@ -357,6 +360,36 @@ public class DtUtilShell extends CommandShell { } } + private class Import extends SubCommand { + public static final String IMPORT_USAGE = + "dtutil import <base64> [-alias <alias>] " + + FORMAT_SUBSTRING + " filename"; + + private String base64 = null; + + Import() { } + + Import(String arg) { + base64 = arg; + } + + @Override + public boolean validate() { + return true; + } + + @Override + public void execute() throws Exception { + DtFileOperations.importTokenFile( + firstFile, format, alias, base64, getConf()); + } + + @Override + public String getUsage() { + return IMPORT_USAGE; + } + } + public static void main(String[] args) throws Exception { System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md index ce904c5..f39a92d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md @@ -172,6 +172,7 @@ For every subcommand that connects to a service, convenience flags are provided | `remove -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. | | `cancel -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. | | `renew -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. | +| `import` *base64* <br/> `[-alias` *alias* `]` <br/> *filename* | Import a token from a base64 token. <br/> *alias* will overwrite the service field in the token. | ### `fs` http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java index 53b0d3d..08554fc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java @@ -57,6 +57,13 @@ public class TestDtUtilShell { public static Text KIND_GET = new Text("testTokenKindGet"); public static Token<?> MOCK_TOKEN = new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET); + + private static final Text SERVICE_IMPORT = + new Text("testTokenServiceImport"); + private static final Text KIND_IMPORT = new Text("testTokenKindImport"); + private static final Token<?> IMPORT_TOKEN = + new Token(IDENTIFIER, PASSWORD, KIND_IMPORT, SERVICE_IMPORT); + static { try { defaultConf.set("fs.defaultFS", "file:///"); @@ -73,9 +80,11 @@ public class TestDtUtilShell { private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2"); private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3"); private final Path tokenFileGet = new Path(workDir, "testGetTokenFile"); + private final Path tokenFileImport = new Path(workDir, "testImportTokenFile"); private final String tokenFilename = tokenFile.toString(); private final String tokenFilename2 = tokenFile2.toString(); private final String tokenFilenameGet = tokenFileGet.toString(); + private final String tokenFilenameImport = tokenFileImport.toString(); private String[] args = null; private DtUtilShell dt = null; private int rc = 0; @@ -283,4 +292,39 @@ public class TestDtUtilShell { spyCreds.readTokenStorageStream(in); Mockito.verify(spyCreds, Mockito.never()).readFields(in); } + + @Test + public void testImport() throws Exception { + String base64 = IMPORT_TOKEN.encodeToUrlString(); + args = new String[] {"import", base64, tokenFilenameImport}; + rc = dt.run(args); + assertEquals("test simple import print old exit code", 0, rc); + + args = new String[] {"print", tokenFilenameImport}; + rc = dt.run(args); + assertEquals("test simple import print old exit code", 0, rc); + assertTrue("test print after import output:\n" + outContent, + outContent.toString().contains(KIND_IMPORT.toString())); + assertTrue("test print after import output:\n" + outContent, + outContent.toString().contains(SERVICE_IMPORT.toString())); + assertTrue("test print after simple import output:\n" + outContent, + outContent.toString().contains(base64)); + } + + @Test + public void testImportWithAliasFlag() throws Exception { + String base64 = IMPORT_TOKEN.encodeToUrlString(); + args = new String[] {"import", base64, "-alias", alias, + tokenFilenameImport}; + rc = dt.run(args); + assertEquals("test import with alias print old exit code", 0, rc); + + args = new String[] {"print", tokenFilenameImport}; + rc = dt.run(args); + assertEquals("test simple import print old exit code", 0, rc); + assertTrue("test print after import output:\n" + outContent, + outContent.toString().contains(KIND_IMPORT.toString())); + assertTrue("test print after import with alias output:\n" + outContent, + outContent.toString().contains(alias)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
