Repository: ambari Updated Branches: refs/heads/branch-2.5 9ab01cbd3 -> b5e917930
AMBARI-19133. hadoop.proxyuser.HTTP.hosts should not be updated when Hive is installed unless WebHcat is installed (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b5e91793 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b5e91793 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b5e91793 Branch: refs/heads/branch-2.5 Commit: b5e91793094048f218a4b74a303fc6bf6191b6b4 Parents: 9ab01cb Author: Robert Levas <[email protected]> Authored: Tue Dec 13 06:21:25 2016 -0500 Committer: Robert Levas <[email protected]> Committed: Tue Dec 13 06:22:31 2016 -0500 ---------------------------------------------------------------------- .../kerberos/VariableReplacementHelper.java | 159 +++++++++++++++++-- .../HIVE/0.12.0.2.0/kerberos.json | 28 ++-- .../stacks/HDP/2.5/services/HIVE/kerberos.json | 26 +-- .../kerberos/VariableReplacementHelperTest.java | 50 +++++- 4 files changed, 226 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b5e91793/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java index 66be3bf..77333b8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java @@ -20,7 +20,12 @@ package org.apache.ambari.server.state.kerberos; import com.google.inject.Singleton; import org.apache.ambari.server.AmbariException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -32,6 +37,8 @@ import java.util.regex.Pattern; @Singleton public class VariableReplacementHelper { + private static final Logger LOG = LoggerFactory.getLogger(VariableReplacementHelper.class); + /** * a regular expression Pattern used to find "variable" placeholders in strings */ @@ -49,6 +56,7 @@ public class VariableReplacementHelper { { put("each", new EachFunction()); put("toLower", new ToLowerFunction()); + put("append", new AppendFunction()); } }; @@ -108,7 +116,7 @@ public class VariableReplacementHelper { if (replacement != null) { if (function != null) { - replacement = applyReplacementFunction(function, replacement); + replacement = applyReplacementFunction(function, replacement, replacementsMap); } // Escape '$' and '\' so they don't cause any issues. @@ -138,11 +146,12 @@ public class VariableReplacementHelper { * <p/> * Commas in arguments should be escaped with a '/'. * - * @param function the name and arguments of the function - * @param replacement the data to use in the function - * @return a new string generated by appling the function + * @param function the name and arguments of the function + * @param replacement the data to use in the function + * @param replacementsMap a Map of data used to perform variable replacements, if needed + * @return a new string generated by applying the function */ - private String applyReplacementFunction(String function, String replacement) { + private String applyReplacementFunction(String function, String replacement, Map<String, Map<String, String>> replacementsMap) { if (function != null) { Matcher matcher = PATTERN_FUNCTION.matcher(function); @@ -161,7 +170,7 @@ public class VariableReplacementHelper { argsList[i] = argsList[i].trim().replace("\\,", ","); } - return f.perform(argsList, replacement); + return f.perform(argsList, replacement, replacementsMap); } } } @@ -178,11 +187,12 @@ public class VariableReplacementHelper { * Perform the function to generate a new string by applying the logic of this function to the * supplied data. * - * @param args an array of arguments, specific to the function - * @param data the data to apply the function logic to + * @param args an array of arguments, specific to the function + * @param data the data to apply the function logic to + * @param replacementsMap a Map of data used to perform variable replacements, if needed * @return the resulting string */ - String perform(String[] args, String data); + String perform(String[] args, String data, Map<String, Map<String, String>> replacementsMap); } /** @@ -198,7 +208,7 @@ public class VariableReplacementHelper { */ private static class EachFunction implements Function { @Override - public String perform(String[] args, String data) { + public String perform(String[] args, String data, Map<String, Map<String, String>> replacementsMap) { if ((args == null) || (args.length != 3)) { throw new IllegalArgumentException("Invalid number of arguments encountered"); } @@ -232,7 +242,7 @@ public class VariableReplacementHelper { */ private static class ToLowerFunction implements Function { @Override - public String perform(String[] args, String data) { + public String perform(String[] args, String data, Map<String, Map<String, String>> replacementsMap) { if (data != null) { return data.toLowerCase(); } @@ -240,4 +250,131 @@ public class VariableReplacementHelper { return ""; } } + + /** + * AppendFunction is a Function implementation that appends the current value to the value of some + * configuration property using a specified delimited value. + * <p/> + * This function expects the following arguments (in order) within the args array: + * <ol> + * <li>configuration specification to use to find the value to append</li> + * <li>delimiter to use when concatenating the value</li> + * <li>boolean value used to determine of the value should be appended only it is unique (true) or unconditionally appended (false)</li> + * </ol> + */ + private static class AppendFunction implements Function { + @Override + public String perform(String[] args, String data, Map<String, Map<String, String>> replacementsMap) { + if ((args == null) || (args.length != 3)) { + String message = "Invalid number of arguments encountered while processing the 'append' variable replacement function. The following arguments are expected:" + + "\n\t- Configuration specification used to get the initial value" + + "\n\t- Delimiter used for parsing the initial value and appending new values" + + "\n\t- A flag to indicate whether values should be unique ('true') or not ('false')"; + + LOG.error(message); + throw new IllegalArgumentException(message); + } + + String configurationSpec = args[0]; + String concatDelimiter = args[1]; + boolean uniqueOnly = Boolean.parseBoolean(args[2]); + String sourceData = getSourceData(replacementsMap, configurationSpec); + + Collection<String> sourceItems = parseItems(sourceData, concatDelimiter); + Collection<String> dataItems = parseItems(data, concatDelimiter); + Collection<String> items = new ArrayList<String>(); + + if (uniqueOnly) { + for (String item : sourceItems) { + if (!items.contains(item)) { + items.add(item); + } + } + + for (String item : dataItems) { + if (!items.contains(item)) { + items.add(item); + } + } + } else { + items.addAll(sourceItems); + items.addAll(dataItems); + } + + return StringUtils.join(items, concatDelimiter); + } + + /** + * Parse the string using the specified delimiter to create a collection of parsed (and trimmed) Strings. + * + * @param delimitedString the String to parse + * @param concatDelimiter the delimiter used to split the String + * @return a Collection of Strings split from the original string + */ + private Collection<String> parseItems(String delimitedString, String concatDelimiter) { + Collection<String> items = new ArrayList<String>(); + + if (!StringUtils.isEmpty(delimitedString)) { + for (String item : delimitedString.split(concatDelimiter)) { + item = item.trim(); + if (!item.isEmpty()) { + items.add(item); + } + } + } + + return items; + } + + /** + * Retrieves the source data given a configuration specification and a Map of configurations, grouped by configuration types. + * <p> + * The configuration specification is expected to be in one of the following forms: + * <ul> + * <li>config-type/property-name</li> + * <li>property-name</li> + * </ul> + * <p> + * The replacementsMap is expected to be a Map of config-types to their name/value pairs. + * + * @param replacementsMap a Map of data used to perform variable replacements, if needed + * @param configurationSpec a configuration specification declaring the config-type (optional) and the relevant + * property name + * @return the found value + */ + private String getSourceData(Map<String, Map<String, String>> replacementsMap, String configurationSpec) { + String sourceData = null; + if ((replacementsMap != null) && !replacementsMap.isEmpty() && !StringUtils.isEmpty(configurationSpec)) { + // Parse the configuration specification to get the config-type and property-name. + // If only one "part" is found when splitting the String, assume that is it the property name + // where the config-type will be assumed to be "", which contains properties not set in service + // configurations + String[] parts = configurationSpec.split("/"); + String type = null; + String name = null; + + if (parts.length == 2) { + type = parts[0]; + name = parts[1]; + } else if (parts.length == 1) { + name = parts[0]; + } + + if (!StringUtils.isEmpty(name)) { + Map<String, String> replacements; + if (type == null) { + replacements = replacementsMap.get(""); + } else { + replacements = replacementsMap.get(type); + } + + if (replacements != null) { + sourceData = replacements.get(name); + } + } + } + + return sourceData; + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b5e91793/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json index 872bfac..c34026a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json @@ -16,17 +16,6 @@ "hive.metastore.sasl.enabled": "true", "hive.server2.authentication": "KERBEROS" } - }, - { - "webhcat-site": { - "templeton.kerberos.secret": "secret", - "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" - } - }, - { - "core-site": { - "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}" - } } ], "components": [ @@ -37,7 +26,7 @@ "name": "hive_metastore_hive", "principal": { "value": "hive/_HOST@${realm}", - "type" : "service", + "type": "service", "configuration": "hive-site/hive.metastore.kerberos.principal", "local_username": "${hive-env/hive_user}" }, @@ -66,7 +55,7 @@ "name": "hive_server_hive", "principal": { "value": "hive/_HOST@${realm}", - "type" : "service", + "type": "service", "configuration": "hive-site/hive.server2.authentication.kerberos.principal", "local_username": "${hive-env/hive_user}" }, @@ -106,6 +95,19 @@ "configuration": "webhcat-site/templeton.kerberos.keytab" } } + ], + "configurations": [ + { + "core-site": { + "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host|append(core-site/hadoop.proxyuser.HTTP.hosts, \\\\,, true)}" + } + }, + { + "webhcat-site": { + "templeton.kerberos.secret": "secret", + "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" + } + } ] } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/b5e91793/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json index 34bda73..044bd65 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -18,17 +18,6 @@ } }, { - "webhcat-site": { - "templeton.kerberos.secret": "secret", - "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" - } - }, - { - "core-site": { - "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}" - } - }, - { "ranger-hive-audit": { "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", @@ -64,7 +53,7 @@ "name": "hive_server_hive", "principal": { "value": "hive/_HOST@${realm}", - "type" : "service", + "type": "service", "configuration": "hive-site/hive.server2.authentication.kerberos.principal", "local_username": "${hive-env/hive_user}" }, @@ -141,6 +130,19 @@ "configuration": "webhcat-site/templeton.kerberos.keytab" } } + ], + "configurations": [ + { + "core-site": { + "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host|append(core-site/hadoop.proxyuser.HTTP.hosts, \\\\,, true)}" + } + }, + { + "webhcat-site": { + "templeton.kerberos.secret": "secret", + "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" + } + } ] } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/b5e91793/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java index e3048e0..857047b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java @@ -19,14 +19,19 @@ package org.apache.ambari.server.state.kerberos; import junit.framework.Assert; + import org.apache.ambari.server.AmbariException; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; -@Category({ category.KerberosTest.class}) +@Category({category.KerberosTest.class}) public class VariableReplacementHelperTest { VariableReplacementHelper helper = new VariableReplacementHelper(); @@ -163,6 +168,13 @@ public class VariableReplacementHelperTest { put("clusterHostInfo", new HashMap<String, String>() {{ put("hive_metastore_host", "host1.unit.test, host2.unit.test , host3.unit.test"); // spaces are there on purpose. }}); + + put("foobar-site", new HashMap<String, String>() {{ + put("data", "one, two, three, four"); // spaces are there on purpose. + put("hello", "hello"); + put("hello_there", "hello, there"); + put("hello_there_one", "hello, there, one"); + }}); } }; @@ -172,6 +184,42 @@ public class VariableReplacementHelperTest { Assert.assertEquals("hive.metastore.local=false,hive.metastore.uris=thrift://host1.unit.test:9083\\,thrift://host2.unit.test:9083\\,thrift://host3.unit.test:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/[email protected]", helper.replaceVariables("hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host | each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}", configurations)); + List<String> expected; + List<String> actual; + + expected = new LinkedList<String>(Arrays.asList("four", "hello", "one", "three", "two")); + actual = new LinkedList<String>(Arrays.asList(helper.replaceVariables("${foobar-site/hello | append(foobar-site/data, \\,, true)}", configurations).split(","))); + Collections.sort(expected); + Collections.sort(actual); + Assert.assertEquals(expected, actual); + + expected = new LinkedList<String>(Arrays.asList("four", "hello", "one", "there", "three", "two")); + actual = new LinkedList<String>(Arrays.asList(helper.replaceVariables("${foobar-site/hello_there | append(foobar-site/data, \\,, true)}", configurations).split(","))); + Collections.sort(expected); + Collections.sort(actual); + Assert.assertEquals(expected, actual); + + expected = new LinkedList<String>(Arrays.asList("four", "hello", "one", "there", "three", "two")); + actual = new LinkedList<String>(Arrays.asList(helper.replaceVariables("${foobar-site/hello_there_one | append(foobar-site/data, \\,, true)}", configurations).split(","))); + Collections.sort(expected); + Collections.sort(actual); + Assert.assertEquals(expected, actual); + + expected = new LinkedList<String>(Arrays.asList("four", "hello", "one", "one", "there", "three", "two")); + actual = new LinkedList<String>(Arrays.asList(helper.replaceVariables("${foobar-site/hello_there_one | append(foobar-site/data, \\,, false)}", configurations).split(","))); + Collections.sort(expected); + Collections.sort(actual); + Assert.assertEquals(expected, actual); + + // Test invalid number of arguments. + try { + helper.replaceVariables("${foobar-site/hello_there_one | append(foobar-site/data, \\,)}", configurations); + Assert.fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) { + // Ignore this is expected. + } + Assert.assertEquals("test=unit.test", helper.replaceVariables("test=${realm|toLower()}", configurations)); }
