Repository: ambari Updated Branches: refs/heads/trunk 6703610ec -> c5b9cc3d0
AMBARI-8358. update service configs for kerberos related configs when kerberos is enabled. (Dilli Arumugam via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c5b9cc3d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c5b9cc3d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c5b9cc3d Branch: refs/heads/trunk Commit: c5b9cc3d0c9f53b0e5af6479aaaacff50e2ec4b8 Parents: 6703610 Author: Yusaku Sako <[email protected]> Authored: Thu Dec 18 11:09:21 2014 -0800 Committer: Yusaku Sako <[email protected]> Committed: Thu Dec 18 11:09:21 2014 -0800 ---------------------------------------------------------------------- .../UpdateKerberosConfigsServerAction.java | 282 +++++++++++++++++++ .../UpdateKerberosConfigsServerActionTest.java | 151 ++++++++++ 2 files changed, 433 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c5b9cc3d/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java new file mode 100644 index 0000000..a9dbbbd --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.kerberos; + +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * UpdateKerberosConfigServerAction is implementation of ServerAction that updates service configs + * while enabling Kerberos + */ +public class UpdateKerberosConfigsServerAction extends AbstractServerAction { + + private final static Logger LOG = + LoggerFactory.getLogger(UpdateKerberosConfigsServerAction.class); + + private HashMap<String, Map<String, String>> configtypesPropsVal = new HashMap(); + + @Inject + private AmbariManagementController controller; + + /** + * Executes this ServerAction + * <p/> + * This is typically called by the ServerActionExecutor in it's own thread, but there is no + * guarantee that this is the case. It is expected that the ExecutionCommand and HostRoleCommand + * properties are set before calling this method. + * + * @param requestSharedDataContext a Map to be used a shared data among all ServerActions related + * to a given request + * @return a CommandReport declaring the status of the task + * @throws org.apache.ambari.server.AmbariException + * + * @throws InterruptedException + */ + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + + CommandReport commandReport = null; + + String clusterName = getExecutionCommand().getClusterName(); + Clusters clusters = controller.getClusters(); + Cluster cluster = clusters.getCluster(clusterName); + + String dataDir = getCommandParameterValue(getCommandParameters(), KerberosServerAction.DATA_DIRECTORY); + File indexFile = new File(dataDir + File.separator + KerberosActionDataFile.DATA_FILE_NAME); + File configFile = new File(dataDir + File.separator + KerberosConfigDataFile.DATA_FILE_NAME); + + KerberosActionDataFileReader indexReader = null; + KerberosConfigDataFileReader configReader = null; + + try { + indexReader = new KerberosActionDataFileReader(indexFile); + Iterator<Map<String, String>> indexRecords = indexReader.iterator(); + while (indexRecords.hasNext()) { + Map<String, String> record = indexRecords.next(); + String hostName = record.get(KerberosActionDataFile.HOSTNAME); + String principal = record.get(KerberosActionDataFile.PRINCIPAL); + String principalConfig = record.get(KerberosActionDataFile.PRINCIPAL_CONFIGURATION); + String[] principalTokens = principalConfig.split("/"); + if (principalTokens.length == 2) { + String principalConfigType = principalTokens[0]; + String principalConfigProp = principalTokens[1]; + addConfigTypePropVal(principalConfigType, principalConfigProp, principal); + } + + String keytabPath = record.get(KerberosActionDataFile.KEYTAB_FILE_PATH); + String keytabConfig = record.get(KerberosActionDataFile.KEYTAB_FILE_CONFIGURATION); + String[] keytabTokens = keytabConfig.split("/"); + if (keytabTokens.length == 2) { + String keytabConfigType = keytabTokens[0]; + String keytabConfigProp = keytabTokens[1]; + addConfigTypePropVal(keytabConfigType, keytabConfigProp, keytabPath); + } + } + + configReader = new KerberosConfigDataFileReader(configFile); + Iterator<Map<String, String>> configRecords = configReader.iterator(); + while (configRecords.hasNext()) { + Map<String, String> record = configRecords.next(); + String configType = record.get(KerberosConfigDataFile.CONFIGURATION_TYPE); + String configKey = record.get(KerberosConfigDataFile.KEY); + String configVal = record.get(KerberosConfigDataFile.VALUE); + addConfigTypePropVal(configType, configKey, configVal); + } + + for(Map.Entry<String, Map<String,String>> entry : configtypesPropsVal.entrySet()) { + Map<String, String> properties = entry.getValue(); + updateConfigurationPropertiesForCluster( + cluster, + entry.getKey(), // configType + properties, + true, // updateIfExists + true, // createNew + "update services configs to enable kerberos"); + } + + } catch (IOException e) { + String message = "Could not update services configs to enable kerberos"; + LOG.error(message, e); + commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", "", message); + } finally { + if (indexReader != null && !indexReader.isClosed()) { + try { + indexReader.close(); + } catch (Throwable t) { + // ignored + } + } + if (configReader != null && !configReader.isClosed()) { + try { + configReader.close(); + } catch (Throwable t) { + // ignored + } + } + } + return (commandReport == null) + ? createCommandReport(0, HostRoleStatus.COMPLETED, "{}", null, null) + : commandReport; + } + + + /** + * + * Updates service config properties of a cluster + * @param cluster the cluster for which to update service configs + * @param configType service config type to be updated + * @param properties map of service config properties + * @param updateIfExists flag indicating whether to update if a property already exists + * @param createNewConfigType flag indicating whether to create new service config + * if the config type does not exist + * @param note a short note on change + * @throws AmbariException if the operation fails + */ + private void updateConfigurationPropertiesForCluster( + Cluster cluster, + String configType, + Map<String, String> properties, + boolean updateIfExists, + boolean createNewConfigType, + String note) + throws AmbariException { + + String newTag = "version" + System.currentTimeMillis(); + + if ((properties != null) && (properties.size() > 0)) { + Map<String, Config> all = cluster.getConfigsByType(configType); + if (all == null || !all.containsKey(newTag)) { + Map<String, String> oldConfigProperties; + Config oldConfig = cluster.getDesiredConfigByType(configType); + + if (oldConfig == null && !createNewConfigType) { + LOG.info("Config " + configType + " not found. Assuming service not installed. " + + "Skipping configuration properties update"); + return; + } else if (oldConfig == null) { + oldConfigProperties = new HashMap<String, String>(); + newTag = "version1"; + } else { + oldConfigProperties = oldConfig.getProperties(); + } + + Map<String, String> mergedProperties = + mergeProperties(oldConfigProperties, properties, updateIfExists); + + if (!Maps.difference(oldConfigProperties, mergedProperties).areEqual()) { + LOG.info("Applying configuration with tag '{}' to " + + "cluster '{}'", newTag, cluster.getClusterName()); + + ConfigurationRequest cr = new ConfigurationRequest(); + cr.setClusterName(cluster.getClusterName()); + cr.setVersionTag(newTag); + cr.setType(configType); + cr.setProperties(mergedProperties); + cr.setServiceConfigVersionNote(note); + controller.createConfiguration(cr); + + Config baseConfig = cluster.getConfig(cr.getType(), cr.getVersionTag()); + if (baseConfig != null) { + String authName = "kerberization"; + + if (cluster.addDesiredConfig(authName, Collections.singleton(baseConfig)) != null) { + String oldConfigString = (oldConfig != null) ? " from='" + oldConfig.getTag() + "'" : ""; + LOG.info("cluster '" + cluster.getClusterName() + "' " + + "changed by: '" + authName + "'; " + + "type='" + baseConfig.getType() + "' " + + "tag='" + baseConfig.getTag() + "'" + + oldConfigString); + } + } + } else { + LOG.info("No changes detected to config " + configType + ". Skipping configuration properties update"); + } + } + } + } + + /** + * Merges current properties and new properties + * @param originalProperties current properties + * @param newProperties new properties + * @param updateIfExists flag indicating whether to update if a property already exists + * @return merged properties + */ + private static Map<String, String> mergeProperties(Map<String, String> originalProperties, + Map<String, String> newProperties, + boolean updateIfExists) { + + Map<String, String> properties = new HashMap<String, String>(originalProperties); + for (Map.Entry<String, String> entry : newProperties.entrySet()) { + if (!properties.containsKey(entry.getKey()) || updateIfExists) { + properties.put(entry.getKey(), entry.getValue()); + } + } + return properties; + } + + /** + * Gets a property from the given commandParameters + * @param commandParameters map of command parameters + * @param propertyName property name to find value for + * @return value of given proeprty name, would return <code>null</code> + * if the provided commandParameters is null or if the requested property is not found + * in commandParams + */ + private static String getCommandParameterValue(Map<String, String> commandParameters, String propertyName) { + return ((commandParameters == null) || (propertyName == null)) ? null : commandParameters.get(propertyName); + } + + /** + * Adds a property to properties of a given service config type + * @param configtype service config type + * @param prop property to be added + * @param val value for the proeprty + */ + private void addConfigTypePropVal(String configtype, String prop, String val) { + Map<String, String> configtypePropsVal = configtypesPropsVal.get(configtype); + if (configtypePropsVal == null) { + configtypePropsVal = new HashMap<String, String>(); + configtypesPropsVal.put(configtype, configtypePropsVal); + } + configtypePropsVal.put(prop, val); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c5b9cc3d/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java new file mode 100644 index 0000000..888a919 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.kerberos; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.commons.codec.digest.DigestUtils; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import static org.easymock.EasyMock.*; + +public class UpdateKerberosConfigsServerActionTest { + + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + String dataDir; + private Injector injector; + private UpdateKerberosConfigsServerAction action; + final AmbariManagementController controller = EasyMock.createNiceMock(AmbariManagementController.class); + Clusters clusters = EasyMock.createNiceMock(Clusters.class); + Cluster cluster = EasyMock.createNiceMock(Cluster.class); + + @Before + public void setup() throws Exception { + setupIndexDat(); + setupConfigDat(); + + expect(controller.getClusters()).andReturn(clusters).once(); + + expect(controller.createConfiguration(anyObject(ConfigurationRequest.class))).andReturn( + null).once(); + + replay(controller); + + expect(cluster.getConfigsByType("hdfs-site")).andReturn(null).once(); + expect(cluster.getDesiredConfigByType("hdfs-site")).andReturn(null).once(); + replay(cluster); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster).once(); + replay(clusters); + + injector = Guice.createInjector(new AbstractModule() { + + @Override + protected void configure() { + bind(AmbariManagementController.class).toInstance(controller); + } + }); + action = injector.getInstance(UpdateKerberosConfigsServerAction.class); + } + + @After + public void verifyCalls() throws Exception { + verify(controller); + verify(clusters); + verify(cluster); + } + + private void setupIndexDat() throws Exception { + + File indexFile; + KerberosActionDataFileBuilder kerberosActionDataFileBuilder = null; + + dataDir = testFolder.getRoot().getAbsolutePath(); + System.out.println("dataDir: " + dataDir); + + indexFile = new File(dataDir, KerberosActionDataFile.DATA_FILE_NAME); + kerberosActionDataFileBuilder = new KerberosActionDataFileBuilder(indexFile); + + kerberosActionDataFileBuilder.addRecord("c6403.ambari.apache.org", "HDFS", "DATANODE", + "dn/_HOST@_REALM", "hdfs-site/dfs.namenode.kerberos.principal", + "/etc/security/keytabs/dn.service.keytab", + "hdfs", "r", "hadoop", "", "hdfs-site/dfs.namenode.keytab.file"); + + kerberosActionDataFileBuilder.close(); + File hostDirectory = new File(dataDir, "c6403.ambari.apache.org"); + + // Ensure the host directory exists... + if (hostDirectory.exists() || hostDirectory.mkdirs()) { + File file = new File(hostDirectory, DigestUtils.sha1Hex("/etc/security/keytabs/dn.service.keytab")); + if (!file.exists()) { + file.createNewFile(); + } + + FileWriter fw = new FileWriter(file.getAbsoluteFile()); + BufferedWriter bw = new BufferedWriter(fw); + bw.write("hello"); + bw.close(); + } + } + + private void setupConfigDat() throws Exception { + File configFile = new File(dataDir, KerberosConfigDataFile.DATA_FILE_NAME); + FileWriter fw = new FileWriter(configFile.getAbsoluteFile()); + BufferedWriter bw = new BufferedWriter(fw); + bw.write("config,key,value\n"); + bw.write("hdfs-site,hadoop.security.authentication,kerberos"); + bw.close(); + } + + @Test + public void testUpdateConfig() throws Exception { + ExecutionCommand executionCommand = new ExecutionCommand(); + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put(KerberosServerAction.DATA_DIRECTORY, dataDir); + executionCommand.setCommandParams(commandParams); + + action.setExecutionCommand(executionCommand); + + ConcurrentMap<String, Object> requestSharedDataContext = null; + + action.execute(requestSharedDataContext); + + } + + +}
