Repository: ambari
Updated Branches:
  refs/heads/trunk 7577ebb26 -> 615438b27


AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/615438b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/615438b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/615438b2

Branch: refs/heads/trunk
Commit: 615438b272e6bd8efd37481ef684ae7d68921e64
Parents: 7577ebb
Author: Nahappan Somasundaram <nsomasunda...@hortonworks.com>
Authored: Tue Nov 29 13:36:16 2016 -0800
Committer: Nahappan Somasundaram <nsomasunda...@hortonworks.com>
Committed: Thu Dec 1 13:58:54 2016 -0800

----------------------------------------------------------------------
 ambari-agent/conf/unix/ambari-agent.ini         |   3 +
 .../ambari_agent/CustomServiceOrchestrator.py   | 120 +++++++++++++++++++
 ambari-agent/src/packages/tarball/all.xml       |  30 +++++
 .../ambari/server/agent/ExecutionCommand.java   |  28 +++++
 .../ambari/server/agent/HeartBeatHandler.java   |  10 +-
 .../server/agent/TestHeartbeatHandler.java      |  28 +++--
 6 files changed, 210 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini 
b/ambari-agent/conf/unix/ambari-agent.ini
index 61948d4..d6fcf5f 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -46,6 +46,9 @@ keysdir=/var/lib/ambari-agent/keys
 server_crt=ca.crt
 passphrase_env_var_name=AMBARI_PASSPHRASE
 ssl_verify_cert=0
+credential_lib_dir=/var/lib/ambari-agent/cred/lib
+credential_conf_dir=/var/lib/ambari-agent/cred/conf
+credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell
 
 [services]
 pidLookupPath=/var/run/

http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git 
a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py 
b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 7d61611..1eab0a5 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,8 @@ from FileCache import FileCache
 from AgentException import AgentException
 from PythonExecutor import PythonExecutor
 from PythonReflectiveExecutor import PythonReflectiveExecutor
+from resource_management.core.utils import PasswordString
+import subprocess
 import Constants
 import hostname
 
@@ -65,6 +67,11 @@ class CustomServiceOrchestrator():
   REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot 
and often (this increases their speed)
   DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS
 
+  # Path where hadoop credential JARS will be available
+  DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib'
+  DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
+  DEFAULT_CREDENTIAL_SHELL_CMD = 
'org.apache.hadoop.security.alias.CredentialShell'
+
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
@@ -78,6 +85,14 @@ class CustomServiceOrchestrator():
     # cache reset will be called on every agent registration
     controller.registration_listeners.append(self.file_cache.reset)
 
+    # Construct the hadoop credential lib JARs path
+    self.credential_shell_lib_path = os.path.join(config.get('security', 
'credential_lib_dir',
+                                                             
self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
+
+    self.credential_conf_dir = config.get('security', 'credential_conf_dir', 
self.DEFAULT_CREDENTIAL_CONF_DIR)
+
+    self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', 
self.DEFAULT_CREDENTIAL_SHELL_CMD)
+
     # Clean up old status command files if any
     try:
       os.unlink(self.status_commands_stdout)
@@ -114,6 +129,102 @@ class CustomServiceOrchestrator():
     else:
       return PythonExecutor(self.tmp_dir, self.config)
 
+  def getProviderDirectory(self, service_name):
+    """
+    Gets the path to the service conf folder where the JCEKS file will be 
created.
+
+    :param service_name: Name of the service, for example, HIVE
+    :return: lower case path to the service conf folder
+    """
+
+    # The stack definition scripts of the service can move the
+    # JCEKS file around to where it wants, which is usually
+    # /etc/<service_name>/conf
+
+    conf_dir = os.path.join(self.credential_conf_dir, service_name.lower())
+    if not os.path.exists(conf_dir):
+      os.makedirs(conf_dir, 0644)
+
+    return conf_dir
+
+  def getAffectedConfigTypes(self, commandJson):
+    """
+    Gets the affected config types for the service in this command
+
+    :param commandJson:
+    :return:
+    """
+    return commandJson.get('configuration_attributes')
+
+  def getCredentialProviderPropertyName(self):
+    """
+    Gets the property name used by the hadoop credential provider
+    :return:
+    """
+    return 'hadoop.security.credential.provider.path'
+
+  def generateJceks(self, commandJson):
+    """
+    Generates the JCEKS file with passwords for the service specified in 
commandJson
+
+    :param commandJson: command JSON
+    :return: An exit value from the external process that generated the JCEKS 
file. None if
+    there are no passwords in the JSON.
+    """
+    cmd_result = None
+    roleCommand = None
+    if 'roleCommand' in commandJson:
+      roleCommand = commandJson['roleCommand']
+
+    logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
+
+    # Password properties for a config type, if present,
+    # are under 
configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, 
attributes2}
+    passwordProperties = {}
+    config_types = self.getAffectedConfigTypes(commandJson)
+    for config_type in config_types:
+      elem = config_types.get(config_type)
+      hidden = elem.get('hidden')
+      if hidden is not None:
+        passwordProperties[config_type] = hidden
+
+    # Set up the variables for the external command to generate a JCEKS file
+    java_home = commandJson['hostLevelParams']['java_home']
+    java_bin = '{java_home}/bin/java'.format(java_home=java_home)
+
+    cs_lib_path = self.credential_shell_lib_path
+    serviceName = commandJson['serviceName']
+
+    # Gather the password values and remove them from the configuration
+    configs = commandJson.get('configurations')
+    for key, value in passwordProperties.items():
+      config = configs.get(key)
+      if config is not None:
+        file_path = os.path.join(self.getProviderDirectory(serviceName), 
"{0}.jceks".format(key))
+        if os.path.exists(file_path):
+          os.remove(file_path)
+        provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
+        logger.info('provider_path={0}'.format(provider_path))
+        for alias in value:
+          pwd = config.get(alias)
+          if pwd is not None:
+            # Remove the clear text password
+            config.pop(alias, None)
+            # Add JCEKS provider path instead
+            config[self.getCredentialProviderPropertyName()] = provider_path
+            logger.debug("config={0}".format(config))
+            protected_pwd = PasswordString(pwd)
+            # Generate the JCEKS file
+            cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 
'create',
+                   alias, '-value', protected_pwd, '-provider', provider_path)
+            logger.info(cmd)
+            cmd_result = subprocess.call(cmd)
+            logger.info('cmd_result = {0}'.format(cmd_result))
+            os.chmod(file_path, 0644) # group and others should have read 
access so that the service user can read
+
+    return cmd_result
+
+
   def runCommand(self, command, tmpoutfile, tmperrfile, 
forced_command_name=None,
                  override_output_files=True, retry=False):
     """
@@ -179,6 +290,15 @@ class CustomServiceOrchestrator():
         handle.on_background_command_started = self.map_task_to_process
         del command['__handle']
 
+      # If command contains credentialStoreEnabled, then
+      # generate the JCEKS file for the configurations.
+      credentialStoreEnabled = False
+      if 'credentialStoreEnabled' in command:
+        credentialStoreEnabled = (command['credentialStoreEnabled'] == "true")
+
+      if credentialStoreEnabled == True:
+        self.generateJceks(command)
+
       json_path = self.dump_command_to_json(command, retry)
       pre_hook_tuple = self.resolve_hook_script_path(hook_dir,
           self.PRE_HOOK_PREFIX, command_name, script_type)

http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-agent/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/src/packages/tarball/all.xml 
b/ambari-agent/src/packages/tarball/all.xml
index c481208..f8a54e3 100644
--- a/ambari-agent/src/packages/tarball/all.xml
+++ b/ambari-agent/src/packages/tarball/all.xml
@@ -190,4 +190,34 @@
       <outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory>
     </file>
   </files>
+  <moduleSets>
+    <moduleSet>
+      <binaries>
+        <includeDependencies>false</includeDependencies>
+        
<outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+        <unpack>false</unpack>
+        <directoryMode>755</directoryMode>
+        <fileMode>644</fileMode>
+        <dependencySets>
+          <dependencySet>
+            
<outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+            <unpack>false</unpack>
+            <includes>
+              <include>commons-cli:commons-cli</include>
+              <include>commons-collections:commons-collections</include>
+              <include>commons-configuration:commons-configuration</include>
+              
<include>commons-io:commons-io:jar:${commons.io.version}</include>
+              <include>commons-lang:commons-lang</include>
+              <include>commons-logging:commons-logging</include>
+              <include>com.google.guava:guava</include>
+              <include>org.slf4j:slf4j-api</include>
+              <include>org.apache.hadoop:hadoop-common</include>
+              <include>org.apache.hadoop:hadoop-auth</include>
+              <include>org.apache.htrace:htrace-core</include>
+            </includes>
+          </dependencySet>
+        </dependencySets>
+      </binaries>
+    </moduleSet>
+  </moduleSets>
 </assembly>

http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 29737ee..7353366 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -112,6 +112,13 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("availableServices")
   private Map<String, String> availableServices = new HashMap<>();
 
+  /**
+   * "true" or "false" indicating whether this
+   * service is enabled for credential store use.
+   */
+  @SerializedName("credentialStoreEnabled")
+  private String credentialStoreEnabled;
+
   public String getCommandId() {
     return commandId;
   }
@@ -295,6 +302,27 @@ public class ExecutionCommand extends AgentCommand {
        this.serviceType = serviceType;
   }
 
+  /**
+   * Get a value indicating whether this service is enabled
+   * for credential store use.
+   *
+   * @return "true" or "false", any other value is
+   * considered as "false"
+   */
+  public String getCredentialStoreEnabled() {
+    return credentialStoreEnabled;
+  }
+
+  /**
+   * Set a value indicating whether this service is enabled
+   * for credential store use.
+   *
+   * @param credentialStoreEnabled
+   */
+  public void setCredentialStoreEnabled(String credentialStoreEnabled) {
+    this.credentialStoreEnabled = credentialStoreEnabled;
+  }
+
   public String getComponentName() {
     return componentName;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
index 75bef30..48d11c4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
@@ -329,7 +329,15 @@ public class HeartBeatHandler {
           case BACKGROUND_EXECUTION_COMMAND:
           case EXECUTION_COMMAND: {
             ExecutionCommand ec = (ExecutionCommand)ac;
-            LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand 
for host {}, role {}, roleCommand {}, and command ID {}, taskId {}",
+            /*
+             * Set the value of credentialStore enabled before sending the 
command to agent.
+             */
+            Cluster cluster = clusterFsm.getCluster(ec.getClusterName());
+            Service service = cluster.getService(ec.getServiceName());
+            if (service != null) {
+              
ec.setCredentialStoreEnabled(String.valueOf(service.isCredentialStoreEnabled()));
+            }
+            LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand 
for host {}, role {}, roleCommand {}, and command ID {}, task ID {}",
                      ec.getHostname(), ec.getRole(), ec.getRoleCommand(), 
ec.getCommandId(), ec.getTaskId());
             Map<String, String> hlp = ec.getHostLevelParams();
             if (hlp != null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/615438b2/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index a50a116..ac58f64 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -48,6 +48,7 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -171,22 +172,31 @@ public class TestHeartbeatHandler {
     ActionManager am = actionManagerTestHelper.getMockActionManager();
     expect(am.getTasks(anyObject(List.class))).andReturn(new 
ArrayList<HostRoleCommand>());
     replay(am);
+
+    Cluster cluster = heartbeatTestHelper.getDummyCluster();
+    Service hdfs = cluster.addService(HDFS);
+    hdfs.addServiceComponent(DATANODE);
+    hdfs.addServiceComponent(NAMENODE);
+    hdfs.addServiceComponent(SECONDARY_NAMENODE);
+    Collection<Host> hosts = cluster.getHosts();
+    assertEquals(hosts.size(), 1);
+
     Clusters fsm = clusters;
-    fsm.addHost(DummyHostname1);
-    Host hostObject = clusters.getHost(DummyHostname1);
+    Host hostObject = hosts.iterator().next();
     hostObject.setIPv4("ipv4");
     hostObject.setIPv6("ipv6");
     hostObject.setOsType(DummyOsType);
 
+    String hostname = hostObject.getHostName();
     ActionQueue aq = new ActionQueue();
 
     HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
     Register reg = new Register();
     HostInfo hi = new HostInfo();
-    hi.setHostName(DummyHostname1);
+    hi.setHostName(hostname);
     hi.setOS(DummyOs);
     hi.setOSRelease(DummyOSRelease);
-    reg.setHostname(DummyHostname1);
+    reg.setHostname(hostname);
     reg.setHardwareProfile(hi);
     reg.setAgentVersion(metaInfo.getServerVersion());
     handler.handleRegistration(reg);
@@ -195,19 +205,21 @@ public class TestHeartbeatHandler {
 
     ExecutionCommand execCmd = new ExecutionCommand();
     execCmd.setRequestAndStage(2, 34);
-    execCmd.setHostname(DummyHostname1);
-    aq.enqueue(DummyHostname1, new ExecutionCommand());
+    execCmd.setHostname(hostname);
+    execCmd.setClusterName(cluster.getClusterName());
+    execCmd.setServiceName(HDFS);
+    aq.enqueue(hostname, execCmd);
     HeartBeat hb = new HeartBeat();
     hb.setResponseId(0);
     HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
     List<Alert> al = new ArrayList<Alert>();
     al.add(new Alert());
     hb.setNodeStatus(hs);
-    hb.setHostname(DummyHostname1);
+    hb.setHostname(hostname);
 
     handler.handleHeartBeat(hb);
     assertEquals(HostState.HEALTHY, hostObject.getState());
-    assertEquals(0, aq.dequeueAll(DummyHostname1).size());
+    assertEquals(0, aq.dequeueAll(hostname).size());
   }
 
 

Reply via email to