Revert "AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information"
This reverts commit 615438b272e6bd8efd37481ef684ae7d68921e64. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60a6bd45 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60a6bd45 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60a6bd45 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 60a6bd4575fb87fc26c4a277cbabf850ef2089e1 Parents: 32840c1 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Dec 2 10:50:50 2016 -0500 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Dec 2 10:50:50 2016 -0500 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-agent.ini | 3 - .../ambari_agent/CustomServiceOrchestrator.py | 120 ------------------- ambari-agent/src/packages/tarball/all.xml | 30 ----- .../ambari/server/agent/ExecutionCommand.java | 28 ----- .../ambari/server/agent/HeartBeatHandler.java | 10 +- .../server/agent/TestHeartbeatHandler.java | 28 ++--- 6 files changed, 9 insertions(+), 210 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/conf/unix/ambari-agent.ini ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index d6fcf5f..61948d4 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -46,9 +46,6 @@ keysdir=/var/lib/ambari-agent/keys server_crt=ca.crt passphrase_env_var_name=AMBARI_PASSPHRASE ssl_verify_cert=0 -credential_lib_dir=/var/lib/ambari-agent/cred/lib -credential_conf_dir=/var/lib/ambari-agent/cred/conf -credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell [services] pidLookupPath=/var/run/ http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 1eab0a5..7d61611 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -29,8 +29,6 @@ from FileCache import FileCache from AgentException import AgentException from PythonExecutor import PythonExecutor from PythonReflectiveExecutor import PythonReflectiveExecutor -from resource_management.core.utils import PasswordString -import subprocess import Constants import hostname @@ -67,11 +65,6 @@ class CustomServiceOrchestrator(): REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot and often (this increases their speed) DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS - # Path where hadoop credential JARS will be available - DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib' - DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf' - DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell' - def __init__(self, config, controller): self.config = config self.tmp_dir = config.get('agent', 'prefix') @@ -85,14 +78,6 @@ class CustomServiceOrchestrator(): # cache reset will be called on every agent registration controller.registration_listeners.append(self.file_cache.reset) - # Construct the hadoop credential lib JARs path - self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir', - self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*') - - self.credential_conf_dir = config.get('security', 'credential_conf_dir', self.DEFAULT_CREDENTIAL_CONF_DIR) - - self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', self.DEFAULT_CREDENTIAL_SHELL_CMD) - # Clean up old status command files if any try: os.unlink(self.status_commands_stdout) @@ -129,102 +114,6 @@ class CustomServiceOrchestrator(): else: return PythonExecutor(self.tmp_dir, self.config) - def getProviderDirectory(self, service_name): - """ - Gets the path to the service conf folder where the JCEKS file will be created. - - :param service_name: Name of the service, for example, HIVE - :return: lower case path to the service conf folder - """ - - # The stack definition scripts of the service can move the - # JCEKS file around to where it wants, which is usually - # /etc/<service_name>/conf - - conf_dir = os.path.join(self.credential_conf_dir, service_name.lower()) - if not os.path.exists(conf_dir): - os.makedirs(conf_dir, 0644) - - return conf_dir - - def getAffectedConfigTypes(self, commandJson): - """ - Gets the affected config types for the service in this command - - :param commandJson: - :return: - """ - return commandJson.get('configuration_attributes') - - def getCredentialProviderPropertyName(self): - """ - Gets the property name used by the hadoop credential provider - :return: - """ - return 'hadoop.security.credential.provider.path' - - def generateJceks(self, commandJson): - """ - Generates the JCEKS file with passwords for the service specified in commandJson - - :param commandJson: command JSON - :return: An exit value from the external process that generated the JCEKS file. None if - there are no passwords in the JSON. - """ - cmd_result = None - roleCommand = None - if 'roleCommand' in commandJson: - roleCommand = commandJson['roleCommand'] - - logger.info('generateJceks: roleCommand={0}'.format(roleCommand)) - - # Password properties for a config type, if present, - # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2} - passwordProperties = {} - config_types = self.getAffectedConfigTypes(commandJson) - for config_type in config_types: - elem = config_types.get(config_type) - hidden = elem.get('hidden') - if hidden is not None: - passwordProperties[config_type] = hidden - - # Set up the variables for the external command to generate a JCEKS file - java_home = commandJson['hostLevelParams']['java_home'] - java_bin = '{java_home}/bin/java'.format(java_home=java_home) - - cs_lib_path = self.credential_shell_lib_path - serviceName = commandJson['serviceName'] - - # Gather the password values and remove them from the configuration - configs = commandJson.get('configurations') - for key, value in passwordProperties.items(): - config = configs.get(key) - if config is not None: - file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key)) - if os.path.exists(file_path): - os.remove(file_path) - provider_path = 'jceks://file{file_path}'.format(file_path=file_path) - logger.info('provider_path={0}'.format(provider_path)) - for alias in value: - pwd = config.get(alias) - if pwd is not None: - # Remove the clear text password - config.pop(alias, None) - # Add JCEKS provider path instead - config[self.getCredentialProviderPropertyName()] = provider_path - logger.debug("config={0}".format(config)) - protected_pwd = PasswordString(pwd) - # Generate the JCEKS file - cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create', - alias, '-value', protected_pwd, '-provider', provider_path) - logger.info(cmd) - cmd_result = subprocess.call(cmd) - logger.info('cmd_result = {0}'.format(cmd_result)) - os.chmod(file_path, 0644) # group and others should have read access so that the service user can read - - return cmd_result - - def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None, override_output_files=True, retry=False): """ @@ -290,15 +179,6 @@ class CustomServiceOrchestrator(): handle.on_background_command_started = self.map_task_to_process del command['__handle'] - # If command contains credentialStoreEnabled, then - # generate the JCEKS file for the configurations. - credentialStoreEnabled = False - if 'credentialStoreEnabled' in command: - credentialStoreEnabled = (command['credentialStoreEnabled'] == "true") - - if credentialStoreEnabled == True: - self.generateJceks(command) - json_path = self.dump_command_to_json(command, retry) pre_hook_tuple = self.resolve_hook_script_path(hook_dir, self.PRE_HOOK_PREFIX, command_name, script_type) http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/src/packages/tarball/all.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/src/packages/tarball/all.xml b/ambari-agent/src/packages/tarball/all.xml index f8a54e3..c481208 100644 --- a/ambari-agent/src/packages/tarball/all.xml +++ b/ambari-agent/src/packages/tarball/all.xml @@ -190,34 +190,4 @@ <outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory> </file> </files> - <moduleSets> - <moduleSet> - <binaries> - <includeDependencies>false</includeDependencies> - <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory> - <unpack>false</unpack> - <directoryMode>755</directoryMode> - <fileMode>644</fileMode> - <dependencySets> - <dependencySet> - <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory> - <unpack>false</unpack> - <includes> - <include>commons-cli:commons-cli</include> - <include>commons-collections:commons-collections</include> - <include>commons-configuration:commons-configuration</include> - <include>commons-io:commons-io:jar:${commons.io.version}</include> - <include>commons-lang:commons-lang</include> - <include>commons-logging:commons-logging</include> - <include>com.google.guava:guava</include> - <include>org.slf4j:slf4j-api</include> - <include>org.apache.hadoop:hadoop-common</include> - <include>org.apache.hadoop:hadoop-auth</include> - <include>org.apache.htrace:htrace-core</include> - </includes> - </dependencySet> - </dependencySets> - </binaries> - </moduleSet> - </moduleSets> </assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 7353366..29737ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -112,13 +112,6 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("availableServices") private Map<String, String> availableServices = new HashMap<>(); - /** - * "true" or "false" indicating whether this - * service is enabled for credential store use. - */ - @SerializedName("credentialStoreEnabled") - private String credentialStoreEnabled; - public String getCommandId() { return commandId; } @@ -302,27 +295,6 @@ public class ExecutionCommand extends AgentCommand { this.serviceType = serviceType; } - /** - * Get a value indicating whether this service is enabled - * for credential store use. - * - * @return "true" or "false", any other value is - * considered as "false" - */ - public String getCredentialStoreEnabled() { - return credentialStoreEnabled; - } - - /** - * Set a value indicating whether this service is enabled - * for credential store use. - * - * @param credentialStoreEnabled - */ - public void setCredentialStoreEnabled(String credentialStoreEnabled) { - this.credentialStoreEnabled = credentialStoreEnabled; - } - public String getComponentName() { return componentName; } http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java index 48d11c4..75bef30 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java @@ -329,15 +329,7 @@ public class HeartBeatHandler { case BACKGROUND_EXECUTION_COMMAND: case EXECUTION_COMMAND: { ExecutionCommand ec = (ExecutionCommand)ac; - /* - * Set the value of credentialStore enabled before sending the command to agent. - */ - Cluster cluster = clusterFsm.getCluster(ec.getClusterName()); - Service service = cluster.getService(ec.getServiceName()); - if (service != null) { - ec.setCredentialStoreEnabled(String.valueOf(service.isCredentialStoreEnabled())); - } - LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, task ID {}", + LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, taskId {}", ec.getHostname(), ec.getRole(), ec.getRoleCommand(), ec.getCommandId(), ec.getTaskId()); Map<String, String> hlp = ec.getHostLevelParams(); if (hlp != null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java index ac58f64..a50a116 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java @@ -48,7 +48,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -172,31 +171,22 @@ public class TestHeartbeatHandler { ActionManager am = actionManagerTestHelper.getMockActionManager(); expect(am.getTasks(anyObject(List.class))).andReturn(new ArrayList<HostRoleCommand>()); replay(am); - - Cluster cluster = heartbeatTestHelper.getDummyCluster(); - Service hdfs = cluster.addService(HDFS); - hdfs.addServiceComponent(DATANODE); - hdfs.addServiceComponent(NAMENODE); - hdfs.addServiceComponent(SECONDARY_NAMENODE); - Collection<Host> hosts = cluster.getHosts(); - assertEquals(hosts.size(), 1); - Clusters fsm = clusters; - Host hostObject = hosts.iterator().next(); + fsm.addHost(DummyHostname1); + Host hostObject = clusters.getHost(DummyHostname1); hostObject.setIPv4("ipv4"); hostObject.setIPv6("ipv6"); hostObject.setOsType(DummyOsType); - String hostname = hostObject.getHostName(); ActionQueue aq = new ActionQueue(); HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector); Register reg = new Register(); HostInfo hi = new HostInfo(); - hi.setHostName(hostname); + hi.setHostName(DummyHostname1); hi.setOS(DummyOs); hi.setOSRelease(DummyOSRelease); - reg.setHostname(hostname); + reg.setHostname(DummyHostname1); reg.setHardwareProfile(hi); reg.setAgentVersion(metaInfo.getServerVersion()); handler.handleRegistration(reg); @@ -205,21 +195,19 @@ public class TestHeartbeatHandler { ExecutionCommand execCmd = new ExecutionCommand(); execCmd.setRequestAndStage(2, 34); - execCmd.setHostname(hostname); - execCmd.setClusterName(cluster.getClusterName()); - execCmd.setServiceName(HDFS); - aq.enqueue(hostname, execCmd); + execCmd.setHostname(DummyHostname1); + aq.enqueue(DummyHostname1, new ExecutionCommand()); HeartBeat hb = new HeartBeat(); hb.setResponseId(0); HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus); List<Alert> al = new ArrayList<Alert>(); al.add(new Alert()); hb.setNodeStatus(hs); - hb.setHostname(hostname); + hb.setHostname(DummyHostname1); handler.handleHeartBeat(hb); assertEquals(HostState.HEALTHY, hostObject.getState()); - assertEquals(0, aq.dequeueAll(hostname).size()); + assertEquals(0, aq.dequeueAll(DummyHostname1).size()); }