This is an automated email from the ASF dual-hosted git repository.
gezapeti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 6e10bad OOZIE-3575 Add credential support for cloud file systems
(matijhs via gezapeti)
6e10bad is described below
commit 6e10bad6adb1c60707956e720f99bf2fc7922cd7
Author: Gezapeti Cseh <gezap...@apache.org>
AuthorDate: Wed Jan 15 15:49:16 2020 +0100
OOZIE-3575 Add credential support for cloud file systems (matijhs via
gezapeti)
---
.../action/hadoop/CredentialsProviderFactory.java | 14 +--
.../oozie/action/hadoop/DistcpActionExecutor.java | 58 +++++------
.../oozie/action/hadoop/FileSystemCredentials.java | 90 +++++++++++++++++
.../oozie/action/hadoop/JavaActionExecutor.java | 100 ++++++++++---------
.../oozie/action/hadoop/TestCredentials.java | 109 +++++++++++++++++++--
docs/src/site/markdown/DG_ActionAuthentication.md | 7 +-
release-log.txt | 1 +
7 files changed, 285 insertions(+), 94 deletions(-)
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
index 8c1bd40..4b576b6 100644
---
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
+++
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
@@ -18,10 +18,6 @@
package org.apache.oozie.action.hadoop;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
@@ -30,10 +26,16 @@ import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.util.StringUtils;
import org.apache.oozie.util.XLog;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
public class CredentialsProviderFactory {
public static final String CRED_KEY =
"oozie.credentials.credentialclasses";
private static final XLog LOG =
XLog.getLog(CredentialsProviderFactory.class);
- public static final String HDFS = "hdfs";
+ public static final String FS = "filesystem";
+ public static final String NAMENODE_FS = "namenode_fs";
+ public static final String WORKFLOW_APP_FS = "workflow_app_fs";
public static final String YARN = "yarnRM";
public static final String JHS = "jhs";
private static CredentialsProviderFactory instance;
@@ -76,7 +78,7 @@ public class CredentialsProviderFactory {
}
}
}
- providerCache.put(HDFS, HDFSCredentials.class);
+ providerCache.put(FS, FileSystemCredentials.class);
providerCache.put(YARN, YarnRMCredentials.class);
providerCache.put(JHS, JHSCredentials.class);
}
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
index a3daaa7..7b925af 100644
---
a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
+++
b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
@@ -22,25 +22,27 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.oozie.action.ActionExecutorException;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.util.StringUtils;
import org.apache.oozie.util.XLog;
import org.jdom.Element;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
-public class DistcpActionExecutor extends JavaActionExecutor{
- public static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS =
"org.apache.oozie.action.hadoop.DistcpMain";
+import static org.apache.oozie.action.hadoop.CredentialsProviderFactory.FS;
+import static
org.apache.oozie.action.hadoop.CredentialsProviderFactory.NAMENODE_FS;
+import static
org.apache.oozie.action.hadoop.FileSystemCredentials.FILESYSTEM_PATH;
+
+public class DistcpActionExecutor extends JavaActionExecutor {
+ private static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS =
"org.apache.oozie.action.hadoop.DistcpMain";
private static final String DISTCP_MAIN_CLASS_NAME =
"org.apache.hadoop.tools.DistCp";
public static final String CLASS_NAMES = "oozie.actions.main.classnames";
private static final XLog LOG = XLog.getLog(DistcpActionExecutor.class);
- public static final String DISTCP_TYPE = "distcp";
/**
* Comma separated list of NameNode hosts to obtain delegation token(s)
for.
*/
- private static final String OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS =
"oozie.launcher.mapreduce.job.hdfs-servers";
+ static final String OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS =
"oozie.launcher.mapreduce.job.hdfs-servers";
/**
* Comma separated list to instruct ResourceManagers on either cluster to
skip delegation token renewal for NameNode hosts.
@@ -67,35 +69,13 @@ public class DistcpActionExecutor extends
JavaActionExecutor{
List<Class<?>> classes = new ArrayList<>();
try {
classes.add(Class.forName(CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS));
- }
- catch (ClassNotFoundException e) {
+ } catch (ClassNotFoundException e) {
throw new RuntimeException("Class not found", e);
}
return classes;
}
/**
- * This function returns the Action classes names from the configuration
- *
- * @param type This is type of the action classes
- * @return Name of the class from the configuration
- */
- public static String getClassNamebyType(String type){
- String classname = null;
- for (String function : ConfigurationService.getStrings(CLASS_NAMES)) {
- function = StringUtils.trim(function);
- LOG.debug("class for Distcp Action: " + function);
- String[] str = function.split("=");
- if (str.length > 0) {
- if(type.equalsIgnoreCase(str[0])){
- classname = new String(str[1]);
- }
- }
- }
- return classname;
- }
-
- /**
* Return the sharelib name for the action.
*
* @return returns <code>distcp</code>.
@@ -114,17 +94,25 @@ public class DistcpActionExecutor extends
JavaActionExecutor{
/**
* Extracts information required for DistCp action between secure clusters
(in the same or distinct Kerberos realms)
*
- * @param jobconf workflow action configuration
+ * @param actionConf workflow action configuration
+ * @param credPropertiesMap Map of defined workflow credentials
*/
@Override
- protected void setActionTokenProperties(final Configuration jobconf) {
- final String hdfsServers =
jobconf.get(OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS);
+ void addNameNodeCredentials(Configuration actionConf, Map<String,
CredentialsProperties> credPropertiesMap) {
+ String hdfsServers =
actionConf.get(OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS);
if (hdfsServers != null) {
- jobconf.set(MRJobConfig.JOB_NAMENODES, hdfsServers);
- final String tokenRenewalExclude =
jobconf.get(OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS_TOKEN_RENEWAL_EXCLUDE);
+ LOG.info("Overriding {0} default value from action config with
{1}",
+ MRJobConfig.JOB_NAMENODES, hdfsServers);
+ actionConf.set(MRJobConfig.JOB_NAMENODES, hdfsServers);
+ final String tokenRenewalExclude =
actionConf.get(OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS_TOKEN_RENEWAL_EXCLUDE);
if (tokenRenewalExclude != null) {
- jobconf.set(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE,
tokenRenewalExclude);
+ actionConf.set(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE,
tokenRenewalExclude);
}
+ CredentialsProperties fsCredentialProperties = new
CredentialsProperties(NAMENODE_FS, FS);
+ fsCredentialProperties.getProperties().put(FILESYSTEM_PATH,
hdfsServers);
+ credPropertiesMap.put(NAMENODE_FS, fsCredentialProperties);
+ } else {
+ super.addNameNodeCredentials(actionConf, credPropertiesMap);
}
}
}
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemCredentials.java
b/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemCredentials.java
new file mode 100644
index 0000000..d1164c8
--- /dev/null
+++
b/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemCredentials.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.action.ActionExecutor;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.UserGroupInformationService;
+import org.apache.oozie.util.XLog;
+
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+
+
+public class FileSystemCredentials implements CredentialsProvider {
+ protected XLog LOG = XLog.getLog(getClass());
+
+ static final String FILESYSTEM_PATH = "filesystem.path";
+
+ /**
+ * Add an HDFS_DELEGATION_TOKEN to the {@link Credentials} provided.
+ * This is also important to ensure that log aggregation works correctly
from the NM.
+ *
+ * @param credentials the credentials object which is updated
+ * @param config launcher AM configuration
+ * @param props properties for getting credential token or
certificate
+ * @param context workflow context
+ * @throws Exception thrown if failed
+ */
+ @Override
+ public void updateCredentials(Credentials credentials, Configuration
config, CredentialsProperties props,
+ ActionExecutor.Context context) throws
Exception {
+ final String[] fileSystemPaths =
StringUtils.getStrings(props.getProperties().get(FILESYSTEM_PATH));
+ if (fileSystemPaths == null) {
+ throw new CredentialException(ErrorCode.E0510,
+ FILESYSTEM_PATH + " property is required to get filesystem
type credential");
+ }
+
+ final Path[] paths = new Path[fileSystemPaths.length];
+ for (int i = 0; i != fileSystemPaths.length; ++i) {
+ paths[i] = new Path(fileSystemPaths[i]);
+ }
+
+ final UserGroupInformation ugi =
Services.get().get(UserGroupInformationService.class)
+ .getProxyUser(context.getWorkflow().getUser());
+ LOG.info("Obtaining delegation tokens");
+ obtainTokens(credentials, config, paths, ugi);
+ }
+
+ private void obtainTokens(Credentials credentials, Configuration config,
Path[] paths, UserGroupInformation ugi)
+ throws java.io.IOException, InterruptedException {
+ ugi.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ TokenCache.obtainTokensForNamenodes(credentials,
paths, config);
+ Collection<Token<? extends TokenIdentifier>> creds =
credentials.getAllTokens();
+ for (Token tok : creds) {
+ LOG.debug("Tokens in TokenCache: {0}",
tok.getService());
+ }
+ return null;
+ }
+ }
+ );
+ }
+}
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index dd13744..0940703 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -19,33 +19,12 @@
package org.apache.oozie.action.hadoop;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Ints;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.ConnectException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -113,14 +92,37 @@ import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.ConnectException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-
+import static org.apache.oozie.action.hadoop.CredentialsProviderFactory.JHS;
+import static
org.apache.oozie.action.hadoop.CredentialsProviderFactory.NAMENODE_FS;
+import static
org.apache.oozie.action.hadoop.CredentialsProviderFactory.WORKFLOW_APP_FS;
+import static org.apache.oozie.action.hadoop.CredentialsProviderFactory.YARN;
+import static
org.apache.oozie.action.hadoop.FileSystemCredentials.FILESYSTEM_PATH;
public class JavaActionExecutor extends ActionExecutor {
public static final String RUNNING = "RUNNING";
@@ -1071,15 +1073,17 @@ public class JavaActionExecutor extends ActionExecutor {
Credentials credentials = new Credentials();
Configuration launcherConf = createLauncherConf(actionFs, context,
action, actionXml, actionConf);
yarnClient = createYarnClient(context, launcherConf);
- Map<String, CredentialsProperties> credentialsProperties =
setCredentialPropertyToActionConf(context,
+ Map<String, CredentialsProperties> credPropertiesMap =
setCredentialPropertyToActionConf(context,
action, actionConf);
if (UserGroupInformation.isSecurityEnabled()) {
-
addHadoopCredentialPropertiesToActionConf(credentialsProperties);
+ addNameNodeCredentials(actionConf, credPropertiesMap);
+ addWorkflowAppFileSystemCredentials(context,
credPropertiesMap);
+ addYarnCredentials(credPropertiesMap);
}
// Adding if action need to set more credential tokens
Configuration credentialsConf = new Configuration(false);
XConfiguration.copy(actionConf, credentialsConf);
- setCredentialTokens(credentials, credentialsConf, context, action,
credentialsProperties);
+ setCredentialTokens(credentials, credentialsConf, context, action,
credPropertiesMap);
// copy back new entries from credentialsConf
for (Entry<String, String> entry : credentialsConf) {
@@ -1189,11 +1193,25 @@ public class JavaActionExecutor extends ActionExecutor {
return context.getVar(OOZIE_ACTION_NAME);
}
- private void addHadoopCredentialPropertiesToActionConf(Map<String,
CredentialsProperties> credentialsProperties) {
- LOG.info("Adding default credentials for action: hdfs, yarn and jhs");
- addHadoopCredentialProperties(credentialsProperties,
CredentialsProviderFactory.HDFS);
- addHadoopCredentialProperties(credentialsProperties,
CredentialsProviderFactory.YARN);
- addHadoopCredentialProperties(credentialsProperties,
CredentialsProviderFactory.JHS);
+ void addNameNodeCredentials(Configuration actionConf, Map<String,
CredentialsProperties> credPropertiesMap) {
+ LOG.info("Adding default credentials for action: namenode");
+ final String jobNameNodes = actionConf.get(MRJobConfig.JOB_NAMENODES);
+ CredentialsProperties hdfsCredProps = new
CredentialsProperties(NAMENODE_FS, CredentialsProviderFactory.FS);
+ hdfsCredProps.getProperties().put(FILESYSTEM_PATH, jobNameNodes);
+ credPropertiesMap.put(NAMENODE_FS, hdfsCredProps);
+ }
+
+ void addWorkflowAppFileSystemCredentials(Context context, Map<String,
CredentialsProperties> credPropertiesMap) {
+ LOG.info("Adding workflow application file system credentials for
action");
+ CredentialsProperties fsCredProps = new
CredentialsProperties(WORKFLOW_APP_FS, CredentialsProviderFactory.FS);
+ fsCredProps.getProperties().put(FILESYSTEM_PATH,
context.getWorkflow().getAppPath());
+ credPropertiesMap.put(WORKFLOW_APP_FS, fsCredProps);
+ }
+
+ private void addYarnCredentials(Map<String, CredentialsProperties>
credPropertiesMap) {
+ LOG.info("Adding default credentials for action: yarn and jhs");
+ addHadoopCredentialProperties(credPropertiesMap, YARN);
+ addHadoopCredentialProperties(credPropertiesMap, JHS);
}
private void addHadoopCredentialProperties(Map<String,
CredentialsProperties> credentialsProperties, String type) {
@@ -1533,7 +1551,6 @@ public class JavaActionExecutor extends ActionExecutor {
return;
}
- setActionTokenProperties(jobconf);
// Make sure we're logged into Kerberos; if not, or near expiration,
it will relogin
CredentialsProviderFactory.ensureKerberosLogin();
for (Entry<String, CredentialsProperties> entry :
credPropertiesMap.entrySet()) {
@@ -1548,7 +1565,7 @@ public class JavaActionExecutor extends ActionExecutor {
} else {
LOG.debug("Credentials object is null for name= " +
credName + ", type=" + credProps.getType());
throw new
ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA020",
- "Could not load credentials of type [{0}] with
name [{1}]]; perhaps it was not defined"
+ "Could not load credentials of type [{0}] with
name [{1}]; perhaps it was not defined"
+ " in oozie-site.xml?",
credProps.getType(), credName);
}
}
@@ -1561,15 +1578,6 @@ public class JavaActionExecutor extends ActionExecutor {
return context != null && action != null && credPropertiesMap != null;
}
- /**
- * Subclasses may override this method in order to take additional actions
required for obtaining credential token(s).
- *
- * @param jobconf workflow action configuration
- */
- protected void setActionTokenProperties(final Configuration jobconf) {
- // nop
- }
-
protected HashMap<String, CredentialsProperties>
getActionCredentialsProperties(Context context,
WorkflowAction action) throws Exception {
HashMap<String, CredentialsProperties> props = new HashMap<String,
CredentialsProperties>();
@@ -1624,7 +1632,7 @@ public class JavaActionExecutor extends ActionExecutor {
}
if (credProp == null && credName != null) {
throw new
ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA021",
- "Could not load credentials with name [{0}]].",
credName);
+ "Could not load credentials with name [{0}].",
credName);
}
} else {
LOG.debug("credentials is null for the action");
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/TestCredentials.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestCredentials.java
index 7601628..a411d27 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestCredentials.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestCredentials.java
@@ -18,17 +18,36 @@
package org.apache.oozie.action.hadoop;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.oozie.action.hadoop.CredentialsProperties;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.oozie.WorkflowJobBean;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
/**
* Test Credentials
- *
*/
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-public class TestCredentials {
+public class TestCredentials extends ActionExecutorTestCase {
+
+ private Map<String, CredentialsProperties> credPropertiesMap;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ credPropertiesMap = new HashMap<>();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
@Test
public void testHbaseCredentials() {
@@ -36,8 +55,86 @@ public class TestCredentials {
prop.getProperties().put("hbase.zookeeper.quorum", "dummyHost");
HbaseCredentials hb = new HbaseCredentials();
JobConf jc = new JobConf(false);
- hb.copyHbaseConfToJobConf(jc,prop);
+ hb.copyHbaseConfToJobConf(jc, prop);
assertEquals("dummyHost", jc.get("hbase.zookeeper.quorum"));
}
+ @Test
+ public void testThrowErrorWhenMissingFileSystemPathProperty() throws
Exception {
+ CredentialsProperties props = new CredentialsProperties("filesystem",
"filesystem");
+ String exMsg = FileSystemCredentials.FILESYSTEM_PATH + " property is
required to get filesystem type credential";
+ FileSystemCredentials fs = new FileSystemCredentials();
+ try {
+ fs.updateCredentials(null, null, props, null);
+ Assert.fail("No exception was thrown!");
+ } catch (CredentialException ex) {
+ Assert.assertTrue(ex.getMessage().contains(exMsg));
+ }
+ }
+
+ /**
+ * Test adding credential for default NameNode defined in
mapreduce.job.hdfs-servers.
+ */
+ @Test
+ public void testAddNameNodeCredentials() {
+ Configuration actionConf = new Configuration();
+ String jobNameNodes = "hdfs://namenode1";
+ actionConf.set(MRJobConfig.JOB_NAMENODES, jobNameNodes);
+ new JavaActionExecutor().addNameNodeCredentials(actionConf,
credPropertiesMap);
+ verifyCredentialsMapping(jobNameNodes);
+ }
+
+ /**
+ * Test adding credentials for multiple NameNodes/resources in case of
cross cluster distcp action.
+ * Should override the value of mapreduce.job.hdfs-servers with
oozie.launcher.mapreduce.job.hdfs-servers, in case its defined.
+ */
+ @Test
+ public void testAddDistCpNameNodeCredentialsOverrideConf() {
+ Configuration actionConf = new Configuration();
+ String jobNameNodes =
"hdfs://namenode1,abfs://resource2,s3a://resource3";
+ String defaultNameNode = "hdfs://namenode1";
+
actionConf.set(DistcpActionExecutor.OOZIE_LAUNCHER_MAPREDUCE_JOB_HDFS_SERVERS,
jobNameNodes);
+ actionConf.set(MRJobConfig.JOB_NAMENODES, defaultNameNode);
+ new DistcpActionExecutor().addNameNodeCredentials(actionConf,
credPropertiesMap);
+ verifyCredentialsMapping(jobNameNodes);
+ Assert.assertEquals(actionConf.get(MRJobConfig.JOB_NAMENODES),
jobNameNodes);
+ Assert.assertNotEquals(actionConf.get(MRJobConfig.JOB_NAMENODES),
defaultNameNode);
+ }
+
+ /**
+ * Test adding credentials for multiple NameNodes in case of cross cluster
distcp action.
+ * Should NOT override the value of mapreduce.job.hdfs-servers, as
oozie.launcher.mapreduce.job.hdfs-servers is not defined.
+ */
+ @Test
+ public void testAddDistCpNameNodeCredentialsDefaultConf() {
+ Configuration actionConf = new Configuration();
+ String jobNameNodes =
"hdfs://namenode1,hdfs://namenode2,hdfs://namenode3";
+ actionConf.set(MRJobConfig.JOB_NAMENODES, jobNameNodes);
+ new DistcpActionExecutor().addNameNodeCredentials(actionConf,
credPropertiesMap);
+ verifyCredentialsMapping(jobNameNodes);
+ Assert.assertEquals(actionConf.get(MRJobConfig.JOB_NAMENODES),
jobNameNodes);
+ }
+
+ /**
+ * Test adding credentials for workflow application path.
+ */
+ @Test
+ public void testAddWorkflowAppFileSystemCredentials() {
+ String wfAppPath = "hdfs://namenode1/user/test_user/app/";
+ Context context = Mockito.mock(Context.class);
+ WorkflowJobBean wf = Mockito.mock(WorkflowJobBean.class);
+ Mockito.when(context.getWorkflow()).thenReturn(wf);
+ Mockito.when(wf.getAppPath()).thenReturn(wfAppPath);
+ new JavaActionExecutor().addWorkflowAppFileSystemCredentials(context,
credPropertiesMap);
+ CredentialsProperties props =
credPropertiesMap.get(CredentialsProviderFactory.WORKFLOW_APP_FS);
+
Assert.assertEquals(props.getProperties().get(FileSystemCredentials.FILESYSTEM_PATH),
wfAppPath);
+ }
+
+ private void verifyCredentialsMapping(String jobNameNodes) {
+ CredentialsProperties props =
credPropertiesMap.get(CredentialsProviderFactory.NAMENODE_FS);
+ Assert.assertNotNull(props);
+ Assert.assertEquals(CredentialsProviderFactory.FS, props.getType());
+ Assert.assertEquals(CredentialsProviderFactory.NAMENODE_FS,
props.getName());
+ Assert.assertEquals(jobNameNodes,
props.getProperties().get(FileSystemCredentials.FILESYSTEM_PATH));
+ }
}
diff --git a/docs/src/site/markdown/DG_ActionAuthentication.md
b/docs/src/site/markdown/DG_ActionAuthentication.md
index d280baf..e283b51 100644
--- a/docs/src/site/markdown/DG_ActionAuthentication.md
+++ b/docs/src/site/markdown/DG_ActionAuthentication.md
@@ -110,6 +110,8 @@ Oozie currently comes with the following Credentials
implementations:
1. HCatalog and Hive Metastore:
`org.apache.oozie.action.hadoop.HCatCredentials`
1. HBase: `org.apache.oozie.action.hadoop.HBaseCredentials`
1. Hive Server 2: `org.apache.oozie.action.hadoop.Hive2Credentials`
+ 1. File system (for workflows that require cross cluster or cloud storage
access):
+ `org.apache.oozie.action.hadoop.FileSystemCredentials`
HCatCredentials requires these two properties:
@@ -123,10 +125,13 @@ HBase does not require any additional properties since
the hbase-site.xml on the
to obtain a delegation token; though properties can be overwritten here if
desired.
Hive2Credentials requires these two properties:
-
1. `hive2.server.principal`
1. `hive2.jdbc.url`
+FileSystemCredentials requires the following property:
+
+ 1. `filesystem.path` - Cloud storage bucket or namenode path, where the
action will need access rights in runtime.
+
[::Go back to Oozie Documentation Index::](index.html)
diff --git a/release-log.txt b/release-log.txt
index 999371e..cf0f505 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3575 Add credential support for cloud file systems (matijhs via gezapeti)
OOZIE-3579 [docs] Fix typos in coordinator documentation (qsbao via asalamon74)
OOZIE-3066 Possibility to set retry-max globally (qsbao via asalamon74)
OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko,
gezapeti)
