Repository: ambari Updated Branches: refs/heads/trunk d5d40d154 -> 1073da759
AMBARI-8356. Push keberos keytabs from ambari server to appropriate service component host. (dilli via jaimin) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1073da75 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1073da75 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1073da75 Branch: refs/heads/trunk Commit: 1073da759316eb13b4290558575e1603048e23eb Parents: d5d40d1 Author: Jaimin Jetly <[email protected]> Authored: Thu Dec 18 11:33:41 2014 -0800 Committer: Jaimin Jetly <[email protected]> Committed: Thu Dec 18 11:33:41 2014 -0800 ---------------------------------------------------------------------- .../ambari/server/agent/ExecutionCommand.java | 24 ++- .../ambari/server/agent/HeartBeatHandler.java | 79 ++++++++++ .../kerberos/KerberosServerAction.java | 5 + .../KERBEROS/package/scripts/kerberos_common.py | 36 ++--- .../services/KERBEROS/package/scripts/params.py | 2 + .../agent/HeartBeatHandlerInjectKeytabTest.java | 145 +++++++++++++++++++ 6 files changed, 269 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index a1ad29e..3d2c622 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -17,10 +17,7 @@ */ package org.apache.ambari.server.agent; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.utils.StageUtils; @@ -91,6 +88,9 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("componentName") private String componentName; + @SerializedName("kerberosCommandParams") + private List<Map<String, String>> kerberosCommandParams = new ArrayList<Map<String, String>>(); + public String getCommandId() { return commandId; } @@ -255,6 +255,22 @@ public class ExecutionCommand extends AgentCommand { } /** + * Returns parameters for kerberos commands + * @return parameters for kerberos commands + */ + public List<Map<String, String>> getKerberosCommandParams() { + return kerberosCommandParams; + } + + /** + * Sets parameters for kerberos commands + * @params parameters for kerberos commands + */ + public void setKerberosCommandParams(List<Map<String, String>> params) { + this.kerberosCommandParams = params; + } + + /** * Contains key name strings. These strings are used inside maps * incapsulated inside command. */ http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java index eb7b308..da4d38b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java @@ -17,6 +17,10 @@ */ package org.apache.ambari.server.agent; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -45,6 +49,9 @@ import org.apache.ambari.server.events.AlertReceivedEvent; import org.apache.ambari.server.events.publishers.AlertEventPublisher; import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.metadata.ActionMetadata; +import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFile; +import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFileReader; +import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction; import org.apache.ambari.server.state.AgentVersion; import org.apache.ambari.server.state.Alert; import org.apache.ambari.server.state.Cluster; @@ -77,6 +84,10 @@ import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartedEve import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStoppedEvent; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.utils.VersionUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -629,6 +640,16 @@ public class HeartBeatHandler { switch (ac.getCommandType()) { case BACKGROUND_EXECUTION_COMMAND: case EXECUTION_COMMAND: { + ExecutionCommand ec = (ExecutionCommand)ac; + Map<String, String> hlp = ec.getHostLevelParams(); + if ((hlp != null) && "SET_KEYTAB".equals(hlp.get("custom_command"))) { + LOG.info("SET_KEYTAB called") ; + try { + injectKeytab(ec, hostname); + } catch (IOException e) { + throw new AmbariException("Could not inject keytab into command", e); + } + } response.addExecutionCommand((ExecutionCommand) ac); break; } @@ -866,4 +887,62 @@ public class HeartBeatHandler { return commands; } + + static void injectKeytab(ExecutionCommand ec, String targetHost) throws AmbariException { + Map<String, String> hlp = ec.getHostLevelParams(); + if ((hlp == null) || !"SET_KEYTAB".equals(hlp.get("custom_command"))) { + return; + } + List<Map<String, String>> kcp = ec.getKerberosCommandParams(); + String dataDir = ec.getCommandParams().get(KerberosServerAction.DATA_DIRECTORY); + File file = new File(dataDir + File.separator + "index.dat"); + CSVParser csvParser = null; + try { + KerberosActionDataFileReader reader = new KerberosActionDataFileReader(file); + Iterator<Map<String, String>> iterator = reader.iterator(); + while (iterator.hasNext()) { + Map<String, String> record = iterator.next(); + String hostName = record.get(KerberosActionDataFile.HOSTNAME); + if (!targetHost.equalsIgnoreCase(hostName)) { + continue; + } + Map<String, String> keytabMap = new HashMap<String, String>(); + keytabMap.put(KerberosActionDataFile.HOSTNAME, hostName); + keytabMap.put(KerberosActionDataFile.SERVICE, record.get(KerberosActionDataFile.SERVICE)); + keytabMap.put(KerberosActionDataFile.COMPONENT, record.get(KerberosActionDataFile.COMPONENT)); + keytabMap.put(KerberosActionDataFile.PRINCIPAL, record.get(KerberosActionDataFile.PRINCIPAL)); + keytabMap.put(KerberosActionDataFile.PRINCIPAL_CONFIGURATION, record.get(KerberosActionDataFile.PRINCIPAL_CONFIGURATION)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_PATH, record.get(KerberosActionDataFile.KEYTAB_FILE_PATH)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_OWNER_NAME, record.get(KerberosActionDataFile.KEYTAB_FILE_OWNER_NAME)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_OWNER_ACCESS, record.get(KerberosActionDataFile.KEYTAB_FILE_OWNER_ACCESS)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_GROUP_NAME, record.get(KerberosActionDataFile.KEYTAB_FILE_GROUP_NAME)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_GROUP_ACCESS, record.get(KerberosActionDataFile.KEYTAB_FILE_GROUP_ACCESS)); + keytabMap.put(KerberosActionDataFile.KEYTAB_FILE_CONFIGURATION, record.get(KerberosActionDataFile.KEYTAB_FILE_CONFIGURATION)); + + String sha1Keytab = DigestUtils.sha1Hex(record.get(KerberosActionDataFile.KEYTAB_FILE_PATH)); + + BufferedInputStream bufferedIn = new BufferedInputStream( + new FileInputStream(dataDir + File.separator + + hostName + File.separator + sha1Keytab)); + byte[] keytabContent = IOUtils.toByteArray(bufferedIn); + String keytabContentBase64 = Base64.encodeBase64String(keytabContent); + keytabMap.put(KerberosServerAction.KEYTAB_CONTENT_BASE64, keytabContentBase64); + kcp.add(keytabMap); + } + } catch (IOException e) { + throw new AmbariException("Could not inject keytabs to enable kerberos"); + } finally { + if (csvParser != null && !csvParser.isClosed()) { + try { + csvParser.close(); + } catch (Throwable t) { + // ignored + } + } + } + + ec.setKerberosCommandParams(kcp); + } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java index 2df1829..71a7659 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java @@ -79,6 +79,11 @@ public abstract class KerberosServerAction extends AbstractServerAction { */ private static final String PRINCIPAL_PASSWORD_MAP = "principal_password_map"; + /* + * Key used in kerberosCommandParams in ExecutionCommand for base64 encoded keytab content + */ + public static final String KEYTAB_CONTENT_BASE64 = "keytab_content_base64"; + private static final Logger LOG = LoggerFactory.getLogger(KerberosServerAction.class); /** http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/kerberos_common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/kerberos_common.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/kerberos_common.py index 269658b..ae50ef0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/kerberos_common.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/kerberos_common.py @@ -351,31 +351,31 @@ class KerberosScript(Script): def write_keytab_file(): import params - if params.keytab_details is not None: - data = get_property_value(params.keytab_details, 'data') - - if (data is not None) and (len(data) > 0): - file_path = get_property_value(params.keytab_details, 'file-path') - - if (file_path is not None) and (len(file_path) > 0): - with open(file_path, 'w') as f: - f.write(base64.b64decode(data)) - - KerberosScript._set_file_access(file_path, params.keytab_details, params.default_group) + if params.kerberos_command_params is not None: + for item in params.kerberos_command_params: + keytab_content_base64 = get_property_value(item, 'keytab_content_base64') + if (keytab_content_base64 is not None) and (len(keytab_content_base64) > 0): + keytab_file_path = get_property_value(item, 'keytab_file_path') + if (keytab_file_path is not None) and (len(keytab_file_path) > 0): + head, tail = os.path.split(keytab_file_path) + if head and not os.path.isdir(head): + os.makedirs(head) + with open(keytab_file_path, 'w') as f: + f.write(base64.b64decode(keytab_content_base64)) + owner = get_property_value(item, 'keytab_file_owner') + owner_access = get_property_value(item, 'keytab_file_owner_access') + group = get_property_value(item, 'keytab_file_group') + group_access = get_property_value(item, 'keytab_file_group_access') + KerberosScript._set_file_access(keytab_file_path, owner, owner_access, group, group_access) @staticmethod - def _set_file_access(file_path, access_details, default_group=None): - if (file_path is not None) and os.path.isfile(file_path) and (access_details is not None): + def _set_file_access(file_path, owner, owner_access='rw', group=None, group_access=''): + if (file_path is not None) and os.path.isfile(file_path) and (owner is not None): import stat import pwd import grp - owner = get_property_value(access_details, 'owner/name') - owner_access = get_property_value(access_details, 'owner/access', 'rw') - group = get_property_value(access_details, 'group/name', default_group) - group_access = get_property_value(access_details, 'group/access', '') - pwnam = pwd.getpwnam(owner) if (owner is not None) and (len(owner) > 0) else None uid = pwnam.pw_uid if pwnam is not None else os.geteuid() http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/params.py index fdc1ba0..58549bc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/KERBEROS/package/scripts/params.py @@ -65,6 +65,8 @@ if config is not None: if command_params is not None: keytab_details = get_unstructured_data(command_params, 'keytab') + kerberos_command_params = get_property_value(config, 'kerberosCommandParams') + configurations = get_property_value(config, 'configurations') if configurations is not None: cluster_env = get_property_value(configurations, 'cluster-env') http://git-wip-us.apache.org/repos/asf/ambari/blob/1073da75/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartBeatHandlerInjectKeytabTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartBeatHandlerInjectKeytabTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartBeatHandlerInjectKeytabTest.java new file mode 100644 index 0000000..e6f48a1 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartBeatHandlerInjectKeytabTest.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.ambari.server.agent; + + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFile; +import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFileBuilder; +import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests injectKeytab method of HeartBeatHandler + */ +public class HeartBeatHandlerInjectKeytabTest { + + String dataDir; + + @Before + public void setup() throws Exception { + File temporaryDirectory; + File indexFile; + KerberosActionDataFileBuilder kerberosActionDataFileBuilder = null; + + try { + temporaryDirectory = File.createTempFile(".ambari_", ".d"); + } catch (IOException e) { + throw new AmbariException("Unexpected error", e); + } + + // Convert the temporary file into a temporary directory... + if (!temporaryDirectory.delete() || !temporaryDirectory.mkdirs()) { + throw new AmbariException("Failed to create temporary directory"); + } + + dataDir = temporaryDirectory.getAbsolutePath(); + System.out.println("dataDir: " + dataDir); + + indexFile = new File(temporaryDirectory, KerberosActionDataFile.DATA_FILE_NAME); + kerberosActionDataFileBuilder = new KerberosActionDataFileBuilder(indexFile); + + kerberosActionDataFileBuilder.addRecord("c6403.ambari.apache.org", "HDFS", "DATANODE", + "dn/_HOST@_REALM", "hdfs-site/dfs.namenode.kerberos.principal", + "/etc/security/keytabs/dn.service.keytab", + "hdfs", "r", "hadoop", "", "hdfs-site/dfs.namenode.keytab.file"); + + kerberosActionDataFileBuilder.close(); + File hostDirectory = new File(dataDir, "c6403.ambari.apache.org"); + + // Ensure the host directory exists... + if (hostDirectory.exists() || hostDirectory.mkdirs()) { + File file = new File(hostDirectory, DigestUtils.sha1Hex("/etc/security/keytabs/dn.service.keytab")); + if (!file.exists()) { + file.createNewFile(); + } + + FileWriter fw = new FileWriter(file.getAbsoluteFile()); + BufferedWriter bw = new BufferedWriter(fw); + bw.write("hello"); + bw.close(); + } + } + + @Test + public void testInjectKeytabApplicableHost() throws Exception { + + ExecutionCommand executionCommand = new ExecutionCommand(); + + Map<String, String> hlp = new HashMap<String, String>(); + hlp.put("custom_command", "SET_KEYTAB"); + executionCommand.setHostLevelParams(hlp); + + Map<String, String> commandparams = new HashMap<String, String>(); + commandparams.put(KerberosServerAction.DATA_DIRECTORY, dataDir); + executionCommand.setCommandParams(commandparams); + + HeartBeatHandler.injectKeytab(executionCommand, "c6403.ambari.apache.org"); + + List<Map<String, String>> kcp = executionCommand.getKerberosCommandParams(); + + Assert.assertEquals("c6403.ambari.apache.org", kcp.get(0).get(KerberosActionDataFile.HOSTNAME)); + Assert.assertEquals("HDFS", kcp.get(0).get(KerberosActionDataFile.SERVICE)); + Assert.assertEquals("DATANODE", kcp.get(0).get(KerberosActionDataFile.COMPONENT)); + Assert.assertEquals("dn/_HOST@_REALM", kcp.get(0).get(KerberosActionDataFile.PRINCIPAL)); + Assert.assertEquals("hdfs-site/dfs.namenode.kerberos.principal", kcp.get(0).get(KerberosActionDataFile.PRINCIPAL_CONFIGURATION)); + Assert.assertEquals("/etc/security/keytabs/dn.service.keytab", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_PATH)); + Assert.assertEquals("hdfs", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_OWNER_NAME)); + Assert.assertEquals("r", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_OWNER_ACCESS)); + Assert.assertEquals("hadoop", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_GROUP_NAME)); + Assert.assertEquals("", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_GROUP_ACCESS)); + Assert.assertEquals("hdfs-site/dfs.namenode.keytab.file", kcp.get(0).get(KerberosActionDataFile.KEYTAB_FILE_CONFIGURATION)); + + Assert.assertEquals(Base64.encodeBase64String("hello".getBytes()), kcp.get(0).get(KerberosServerAction.KEYTAB_CONTENT_BASE64)); + + } + + @Test + public void testInjectKeytabNotApplicableHost() throws Exception { + ExecutionCommand executionCommand = new ExecutionCommand(); + + Map<String, String> hlp = new HashMap<String, String>(); + hlp.put("custom_command", "SET_KEYTAB"); + executionCommand.setHostLevelParams(hlp); + + Map<String, String> commandparams = new HashMap<String, String>(); + commandparams.put(KerberosServerAction.DATA_DIRECTORY, dataDir); + executionCommand.setCommandParams(commandparams); + + HeartBeatHandler.injectKeytab(executionCommand, "c6400.ambari.apache.org"); + + List<Map<String, String>> kcp = executionCommand.getKerberosCommandParams(); + Assert.assertTrue(kcp.isEmpty()); + + } + +}
