Repository: ambari Updated Branches: refs/heads/trunk 094c56ef3 -> c23de40d3
AMBARI-11176. Hive Upgrade Pack For HDP-2.2 to HDP-2.3 (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c23de40d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c23de40d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c23de40d Branch: refs/heads/trunk Commit: c23de40d31d807ed9bde8e6c8d53a68b83a6f118 Parents: 094c56e Author: Nate Cole <[email protected]> Authored: Fri May 15 12:39:01 2015 -0400 Committer: Nate Cole <[email protected]> Committed: Sat May 16 20:18:35 2015 -0400 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 4 +- .../serveraction/upgrades/ConfigureAction.java | 37 ++++- .../state/stack/upgrade/ConfigureTask.java | 55 +++++++ .../0.12.0.2.0/configuration/webhcat-env.xml | 2 +- .../0.12.0.2.0/package/files/startMetastore.sh | 4 +- .../0.12.0.2.0/package/files/templetonSmoke.sh | 9 +- .../package/scripts/hcat_service_check.py | 16 +- .../package/scripts/hive_metastore.py | 28 +++- .../0.12.0.2.0/package/scripts/hive_server.py | 24 +-- .../0.12.0.2.0/package/scripts/hive_service.py | 20 ++- .../package/scripts/webhcat_server.py | 2 +- .../package/scripts/webhcat_service.py | 27 ++-- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 6 + .../upgrades/ConfigureActionTest.java | 76 ++++++++++ .../stacks/2.0.6/HIVE/test_hive_metastore.py | 150 ++++++++++++++++++- .../stacks/2.0.6/HIVE/test_hive_server.py | 10 +- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 16 +- .../stacks/2.1/HIVE/test_hive_metastore.py | 6 +- .../src/test/python/stacks/utils/RMFTestCase.py | 8 +- 19 files changed, 437 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index ca72934..81a15ec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1019,15 +1019,17 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider ConfigureTask ct = (ConfigureTask) task; Map<String, String> configProperties = ct.getConfigurationProperties(cluster); List<ConfigureTask.Transfer> transfers = ct.getTransfers(); + List<ConfigureTask.Replace> replacements = ct.getReplacements(); // if the properties are empty it means that the conditions in the // task did not pass; - if (configProperties.isEmpty() && transfers.isEmpty()) { + if (configProperties.isEmpty() && transfers.isEmpty() && replacements.isEmpty()) { stageText = "No conditions were met for this configuration task."; itemDetail = stageText; } else { commandParams.putAll(configProperties); commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, s_gson.toJson(transfers)); + commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, s_gson.toJson(replacements)); // extract the config type, key and value to use to build the // summary and detail http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java index 729c36a..03b9fa3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java @@ -164,18 +164,26 @@ public class ConfigureAction extends AbstractServerAction { transferJson, new TypeToken<List<ConfigureTask.Transfer>>(){}.getType()); } + List<ConfigureTask.Replace> replacements = Collections.emptyList(); + String replaceJson = commandParameters.get(ConfigureTask.PARAMETER_REPLACEMENTS); + if (null != replaceJson) { + replacements = m_gson.fromJson( + replaceJson, new TypeToken<List<ConfigureTask.Replace>>(){}.getType()); + } + // if the two required properties are null and no transfer properties, then // assume that no conditions were met and let the action complete - if (null == configType && null == key && transfers.isEmpty()) { + if (null == configType && null == key && transfers.isEmpty() && replacements.isEmpty()) { return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", "", "Skipping configuration task"); } // if only 1 of the required properties was null and no transfer properties, // then something went wrong - if (null == clusterName || null == configType || (null == key && transfers.isEmpty())) { - String message = "cluster={0}, type={1}, key={2}, transfers={3}"; - message = MessageFormat.format(message, clusterName, configType, key, transfers); + if (null == clusterName || null == configType || + (null == key && transfers.isEmpty() && replacements.isEmpty())) { + String message = "cluster={0}, type={1}, key={2}, transfers={3}, replacements={4}"; + message = MessageFormat.format(message, clusterName, configType, key, transfers, replacements); return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", message); } @@ -337,6 +345,27 @@ public class ConfigureAction extends AbstractServerAction { outputBuffer.append(MessageFormat.format("{0}/{1} changed to {2}\n", configType, key, value)); } + + // !!! string replacements happen only on the new values. + for (ConfigureTask.Replace replacement : replacements) { + if (newValues.containsKey(replacement.key)) { + String toReplace = newValues.get(replacement.key); + + if (!toReplace.contains(replacement.find)) { + outputBuffer.append(MessageFormat.format("String {0} was not found in {1}/{2}\n", + replacement.find, configType, replacement.key)); + } else { + String replaced = StringUtils.replace(toReplace, replacement.find, replacement.replaceWith); + + newValues.put(replacement.key, replaced); + + outputBuffer.append(MessageFormat.format("Replaced {0}/{1} containing \"{2}\" with \"{3}\"\n", + configType, replacement.key, replacement.find, replacement.replaceWith)); + } + } + } + + // !!! check to see if we're going to a new stack and double check the // configs are for the target. Then simply update the new properties instead // of creating a whole new history record since it was already done http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index 06d3108..5b47ee6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -102,6 +102,12 @@ public class ConfigureTask extends ServerSideActionTask { public static final String PARAMETER_TRANSFERS = "configure-task-transfers"; /** + * Replacements can be several per task, so they're passed in as a json-ified list of + * objects. + */ + public static final String PARAMETER_REPLACEMENTS = "configure-task-replacements"; + + /** * Constructor. * */ @@ -130,6 +136,9 @@ public class ConfigureTask extends ServerSideActionTask { @XmlElement(name = "transfer") private List<Transfer> transfers; + @XmlElement(name="replace") + private List<Replace> replacements; + @Override public Type getType() { return type; @@ -263,6 +272,51 @@ public class ConfigureTask extends ServerSideActionTask { } /** + * Used to replace strings in a key with other strings. More complex + * scenarios will be possible with regex (when needed) + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "replace") + public static class Replace { + /** + * The key name + */ + @XmlAttribute(name="key") + public String key; + + /** + * The string to find + */ + @XmlAttribute(name="find") + public String find; + + /** + * The string to replace + */ + @XmlAttribute(name="replace-with") + public String replaceWith; + } + + /** + * @return the replacement tokens, never {@code null} + */ + public List<Replace> getReplacements() { + if (null == replacements) { + return Collections.emptyList(); + } + + List<Replace> list = new ArrayList<Replace>(); + for (Replace r : replacements) { + if (null == r.key || null == r.find || null == r.replaceWith) { + continue; + } + list.add(r); + } + + return list; + } + + /** * Gets a map containing the following properties pertaining to the * configuration value to change: * <ul> @@ -341,4 +395,5 @@ public class ConfigureTask extends ServerSideActionTask { return config.getProperties().get(propertyKey); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-env.xml index 14a473f..bd00f6d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-env.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-env.xml @@ -47,7 +47,7 @@ CONSOLE_LOG={{templeton_log_dir}}/webhcat-console.log #HCAT_PREFIX=hive_prefix # Set HADOOP_HOME to point to a specific hadoop install directory -export HADOOP_HOME={{hadoop_home}} +export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/startMetastore.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/startMetastore.sh b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/startMetastore.sh index da0f60b..86541f0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/startMetastore.sh +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/startMetastore.sh @@ -19,5 +19,7 @@ # under the License. # # -HIVE_CONF_DIR=$4 hive --service metastore -hiveconf hive.log.file=hivemetastore.log -hiveconf hive.log.dir=$5 > $1 2> $2 & +HIVE_BIN=${HIVE_BIN:-"hive"} + +HIVE_CONF_DIR=$4 $HIVE_BIN --service metastore -hiveconf hive.log.file=hivemetastore.log -hiveconf hive.log.dir=$5 > $1 2> $2 & echo $!|cat>$3 http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh index 2083312..dd00e02 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh @@ -37,10 +37,17 @@ else fi export no_proxy=$ttonhost -cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status 2>&1" +cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status 2>&1" retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` +# try again for 2.3 username requirement +if [[ "$httpExitCode" == "500" ]] ; then + cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status?user.name=$smoke_test_user 2>&1" + retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` + httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` +fi + if [[ "$httpExitCode" -ne "200" ]] ; then echo "Templeton Smoke Test (status cmd): Failed. : $retVal" export TEMPLETON_EXIT_CODE=1 http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_service_check.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_service_check.py index e8bad81..d35e0b9 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_service_check.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_service_check.py @@ -18,6 +18,7 @@ limitations under the License. """ +import os from resource_management import * from resource_management.libraries.functions import get_unique_id_and_date from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -51,11 +52,16 @@ def hcat_service_check(): prepare_cmd = format("{kinit_cmd}env JAVA_HOME={java64_home} {tmp_dir}/hcatSmoke.sh hcatsmoke{unique} prepare") + exec_path = params.execute_path + if params.version and params.stack_name: + upgrade_hive_bin = format("/usr/hdp/{version}/hive/bin") + exec_path = os.environ['PATH'] + os.pathsep + params.hadoop_bin_dir + os.pathsep + upgrade_hive_bin + Execute(prepare_cmd, tries=3, user=params.smokeuser, try_sleep=5, - path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', params.execute_path], + path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', exec_path], logoutput=True) if params.security_enabled: @@ -67,8 +73,7 @@ def hcat_service_check(): kinit_path_local=params.kinit_path_local, keytab=params.hdfs_user_keytab, principal=params.hdfs_principal_name, - bin_dir=params.execute_path - ) + bin_dir=params.execute_path) else: ExecuteHadoop(test_cmd, user=params.hdfs_user, @@ -86,6 +91,5 @@ def hcat_service_check(): tries=3, user=params.smokeuser, try_sleep=5, - path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', params.execute_path], - logoutput=True - ) + path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', exec_path], + logoutput=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index c5d30a1..3c7900d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -40,7 +40,7 @@ class HiveMetastore(Script): import params env.set_params(params) self.configure(env) # FOR SECURITY - hive_service('metastore', action='start') + hive_service('metastore', action='start', rolling_restart=rolling_restart) def stop(self, env, rolling_restart=False): import params @@ -77,6 +77,9 @@ class HiveMetastoreDefault(HiveMetastore): import params env.set_params(params) + if Script.is_hdp_stack_greater_or_equal("2.3"): + self.upgrade_schema(env) + if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-metastore", params.version) @@ -131,5 +134,28 @@ class HiveMetastoreDefault(HiveMetastore): else: self.put_structured_out({"securityState": "UNSECURED"}) + def upgrade_schema(self, env): + """ + Executes the schema upgrade binary. This is its own function because it could + be called as a standalone task from the upgrade pack, but is safe to run it for each + metastore instance. + """ + Logger.info("Upgrading Hive Metastore") + import params + env.set_params(params) + + if params.security_enabled: + kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ") + Execute(kinit_command,user=params.smokeuser) + + binary = format("/usr/hdp/{version}/hive/bin/schematool") + + env_dict = { + 'HIVE_CONF_DIR': params.hive_server_conf_dir + } + + command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema") + Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True) + if __name__ == "__main__": HiveMetastore().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index a41632e..aeac939 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -24,6 +24,7 @@ from hive import hive from hive_service import hive_service from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions.get_hdp_version import get_hdp_version from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML @@ -72,8 +73,7 @@ class HiveServerDefault(HiveServer): self.configure(env) # FOR SECURITY setup_ranger_hive() - hive_service( 'hiveserver2', action = 'start', - rolling_restart=rolling_restart ) + hive_service( 'hiveserver2', action = 'start', rolling_restart=rolling_restart) def stop(self, env, rolling_restart=False): import params @@ -100,14 +100,18 @@ class HiveServerDefault(HiveServer): if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-server2", params.version) - params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.mapreduce_tar_source, - group=params.user_group, - mode=params.tarballs_mode - ) - params.HdfsResource(None, action="execute") + old = params.hdp_stack_version + try: + params.hdp_stack_version = get_hdp_version('hive-server2') + params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), + type="file", + action="create_on_execute", + source=params.mapreduce_tar_source, + group=params.user_group, + mode=params.tarballs_mode) + params.HdfsResource(None, action="execute") + finally: + params.hdp_stack_version = old def security_status(self, env): import status_params http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index af9d5d3..86312d2 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -57,7 +57,9 @@ def hive_service(name, action='start', rolling_restart=False): if name == 'hiveserver2': check_fs_root() - demon_cmd = cmd + daemon_cmd = cmd + hadoop_home = params.hadoop_home + hive_bin = "hive" # upgrading hiveserver2 (rolling_restart) means that there is an existing, # de-registering hiveserver2; the pid will still exist, but the new @@ -65,16 +67,20 @@ def hive_service(name, action='start', rolling_restart=False): if rolling_restart: process_id_exists_command = None + if (params.version): + import os + hadoop_home = format("/usr/hdp/{version}/hadoop") + hive_bin = os.path.join(params.hive_bin, hive_bin) + if params.security_enabled: hive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; ") Execute(hive_kinit_cmd, user=params.hive_user) - Execute(demon_cmd, - user=params.hive_user, - environment={'HADOOP_HOME': params.hadoop_home, 'JAVA_HOME': params.java64_home}, - path=params.execute_path, - not_if=process_id_exists_command - ) + Execute(daemon_cmd, + user = params.hive_user, + environment = { 'HADOOP_HOME': hadoop_home, 'JAVA_HOME': params.java64_home, 'HIVE_BIN': hive_bin }, + path = params.execute_path, + not_if = process_id_exists_command) if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \ params.hive_jdbc_driver == "org.postgresql.Driver" or \ http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py index 735590b..012d1e2 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py @@ -39,7 +39,7 @@ class WebHCatServer(Script): import params env.set_params(params) self.configure(env) # FOR SECURITY - webhcat_service(action='start') + webhcat_service(action='start', rolling_restart=rolling_restart) def stop(self, env, rolling_restart=False): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py index 7fb82da..3e6ec9e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py @@ -30,23 +30,30 @@ def webhcat_service(action='start'): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) -def webhcat_service(action='start'): +def webhcat_service(action='start', rolling_restart=False): import params - cmd = format('env HADOOP_HOME={hadoop_home} {webhcat_bin_dir}/webhcat_server.sh') + environ = { + 'HADOOP_HOME': params.hadoop_home + } + + cmd = format('{webhcat_bin_dir}/webhcat_server.sh') if action == 'start': - demon_cmd = format('cd {hcat_pid_dir} ; {cmd} start') + if rolling_restart and params.version: + environ['HADOOP_HOME'] = format("/usr/hdp/{version}/hadoop") + + daemon_cmd = format('cd {hcat_pid_dir} ; {cmd} start') no_op_test = format('ls {webhcat_pid_file} >/dev/null 2>&1 && ps -p `cat {webhcat_pid_file}` >/dev/null 2>&1') - Execute(demon_cmd, + Execute(daemon_cmd, user=params.webhcat_user, - not_if=no_op_test - ) + not_if=no_op_test, + environment = environ) elif action == 'stop': - demon_cmd = format('{cmd} stop') - Execute(demon_cmd, - user=params.webhcat_user - ) + daemon_cmd = format('{cmd} stop') + Execute(daemon_cmd, + user = params.webhcat_user, + environment = environ) File(params.webhcat_pid_file, action="delete", ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 8905561..69f7b93 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -524,6 +524,12 @@ </component> <component name="WEBHCAT_SERVER"> + <pre-upgrade> + <task xsi:type="configure"> + <type>webhcat-env</type> + <replace key="content" find="export HADOOP_HOME={{hadoop_home}}" replace-with="export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}" /> + </task> + </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java index 50e0a17..6ca677a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java @@ -431,6 +431,82 @@ public class ConfigureActionTest { assertEquals("['c6401','c6402','c6403']", map.get("zoo.server.array")); } + + + @Test + public void testValueReplacement() throws Exception { + makeUpgradeCluster(); + + Cluster c = m_injector.getInstance(Clusters.class).getCluster("c1"); + assertEquals(1, c.getConfigsByType("zoo.cfg").size()); + + c.setDesiredStackVersion(HDP_21_STACK); + ConfigFactory cf = m_injector.getInstance(ConfigFactory.class); + Config config = cf.createNew(c, "zoo.cfg", new HashMap<String, String>() { + { + put("key_to_replace", "My New Cat"); + put("key_with_no_match", "WxyAndZ"); + } + }, new HashMap<String, Map<String, String>>()); + config.setTag("version2"); + config.persist(); + + c.addConfig(config); + c.addDesiredConfig("user", Collections.singleton(config)); + assertEquals(2, c.getConfigsByType("zoo.cfg").size()); + + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put("upgrade_direction", "upgrade"); + commandParams.put("version", HDP_2_2_1_0); + commandParams.put("clusterName", "c1"); + commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); + + // Replacement task + List<ConfigureTask.Replace> replacements = new ArrayList<ConfigureTask.Replace>(); + ConfigureTask.Replace replace = new ConfigureTask.Replace(); + replace.key = "key_to_replace"; + replace.find = "New Cat"; + replace.replaceWith = "Wet Dog"; + replacements.add(replace); + + replace = new ConfigureTask.Replace(); + replace.key = "key_with_no_match"; + replace.find = "abc"; + replace.replaceWith = "def"; + replacements.add(replace); + + commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, new Gson().toJson(replacements)); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + executionCommand.setRoleParams(new HashMap<String, String>()); + executionCommand.getRoleParams().put(ServerAction.ACTION_USER_NAME, "username"); + + HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); + + hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); + + ConfigureAction action = m_injector.getInstance(ConfigureAction.class); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hostRoleCommand); + + CommandReport report = action.execute(null); + assertNotNull(report); + + assertEquals(3, c.getConfigsByType("zoo.cfg").size()); + + config = c.getDesiredConfigByType("zoo.cfg"); + assertNotNull(config); + assertFalse("version2".equals(config.getTag())); + + assertEquals("My Wet Dog", config.getProperties().get("key_to_replace")); + assertEquals("WxyAndZ", config.getProperties().get("key_with_no_match")); + + } + + + private void makeUpgradeCluster() throws Exception { String clusterName = "c1"; String hostName = "h1"; http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 6bc9797..e46311c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' +import json import os from mock.mock import MagicMock, call, patch from stacks.utils.RMFTestCase import * @@ -46,7 +47,7 @@ class TestHiveMetastore(RMFTestCase): self.assert_configure_default() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive.pid` >/dev/null 2>&1', user = 'hive', path = ['/bin:/usr/lib/hive/bin:/usr/bin'], @@ -106,7 +107,7 @@ class TestHiveMetastore(RMFTestCase): user = 'hive', ) self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive.pid` >/dev/null 2>&1', user = 'hive', path = ['/bin:/usr/lib/hive/bin:/usr/bin'], @@ -348,3 +349,148 @@ class TestHiveMetastore(RMFTestCase): recursive = True, cd_access = 'a', ) + + @patch("resource_management.core.shell.call") + @patch("resource_management.libraries.functions.get_hdp_version") + def test_start_ru(self, call_mock, get_hdp_version_mock): + get_hdp_version_mock.return_value = '2.3.0.0-1234' + + config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" + with open(config_file, "r") as f: + json_content = json.load(f) + + version = "2.3.0.0-1234" + json_content['commandParams']['version'] = version + json_content['hostLevelParams']['stack_name'] = "HDP" + json_content['hostLevelParams']['stack_version'] = "2.3" + json_content['role'] = "HIVE_SERVER" + json_content['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] = "aaa" + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "start", + command_args = [True], + config_dict = json_content, + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES) + + self.assertResourceCalled('Directory', '/etc/hive', + mode = 0755) + + self.assertResourceCalled('Directory', '/usr/hdp/current/hive-server2/conf', + owner = 'hive', + group = 'hadoop', + recursive = True) + + self.assertResourceCalled('XmlConfig', 'mapred-site.xml', + group = 'hadoop', + conf_dir = '/usr/hdp/current/hive-server2/conf', + mode = 0644, + configuration_attributes = {u'final': {u'mapred.healthChecker.script.path': u'true', + u'mapreduce.jobtracker.staging.root.dir': u'true'}}, + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site']) + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop') + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + mode = 420, + group = 'hadoop', + owner = 'hive') + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + mode = 420, + group = 'hadoop', + owner = 'hive') + + self.assertResourceCalled('XmlConfig', 'hive-site.xml', + group = 'hadoop', + conf_dir = '/usr/hdp/current/hive-server2/conf/conf.server', + mode = 0644, + configuration_attributes = {u'final': {u'hive.optimize.bucketmapjoin.sortedmerge': u'true', + u'javax.jdo.option.ConnectionDriverName': u'true', + u'javax.jdo.option.ConnectionPassword': u'true'}}, + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site']) + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop') + + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True) + + self.assertResourceCalled('File', '/etc/security/limits.d/hive.conf', + content = Template('hive.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644) + + self.assertResourceCalled('Execute', ('cp', + '--remove-destination', + '/usr/share/java/mysql-connector-java.jar', + '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar'), + path = ['/bin', '/usr/bin/'], + sudo = True) + + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar', + mode = 0644) + + self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', + content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar')) + + self.assertResourceCalled('File', '/tmp/start_metastore_script', + content = StaticFile('startMetastore.sh'), + mode = 0755) + + self.maxDiff = None + + self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa', + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa'", + user = 'hive') + + self.assertResourceCalled('Directory', '/var/run/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + cd_access = 'a') + + self.assertResourceCalled('Directory', '/var/log/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + cd_access = 'a') + + self.assertResourceCalled('Directory', '/var/lib/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + cd_access = 'a') + + self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /usr/hdp/current/hive-server2/conf/conf.server /var/log/hive', + environment = {'HADOOP_HOME': '/usr/hdp/2.3.0.0-1234/hadoop', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': '/usr/hdp/current/hive-server2/bin/hive'}, + not_if = None, + user = 'hive', + path = ['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin']) + + self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive aaa com.mysql.jdbc.Driver', + path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], + tries = 5, + try_sleep = 10) + + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 04db874..90f3e06 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -69,7 +69,7 @@ class TestHiveServer(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', - environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if='ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', user='hive', path=['/bin:/usr/lib/hive/bin:/usr/bin'] @@ -106,7 +106,7 @@ class TestHiveServer(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', - environment = {'HADOOP_HOME' : '/usr', 'JAVA_HOME':'/usr/jdk64/jdk1.7.0_45'}, + environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, path = ["/bin:/usr/lib/hive/bin:/usr/bin"], user = 'hive' ) @@ -139,7 +139,7 @@ class TestHiveServer(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', - environment = {'HADOOP_HOME' : '/usr', 'JAVA_HOME':'/usr/jdk64/jdk1.7.0_45'}, + environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, path = ["/bin:/usr/lib/hive/bin:/usr/bin"], user = 'hive' ) @@ -173,7 +173,7 @@ class TestHiveServer(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', - environment = {'HADOOP_HOME' : '/usr', 'JAVA_HOME':'/usr/jdk64/jdk1.7.0_45'}, + environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, path = ["/bin:/usr/lib/hive/bin:/usr/bin"], user = 'hive' ) @@ -247,7 +247,7 @@ class TestHiveServer(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', - environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment={'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if='ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', user='hive', path=['/bin:/usr/lib/hive/bin:/usr/bin'], http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py index f2d7edb..c377715 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py @@ -48,9 +48,10 @@ class TestWebHCatServer(RMFTestCase): ) self.assert_configure_default() - self.assertResourceCalled('Execute', 'cd /var/run/webhcat ; env HADOOP_HOME=/usr /usr/lib/hcatalog/sbin/webhcat_server.sh start', + self.assertResourceCalled('Execute', 'cd /var/run/webhcat ; /usr/lib/hcatalog/sbin/webhcat_server.sh start', not_if = 'ls /var/run/webhcat/webhcat.pid >/dev/null 2>&1 && ps -p `cat /var/run/webhcat/webhcat.pid` >/dev/null 2>&1', - user = 'hcat' + user = 'hcat', + environment = {'HADOOP_HOME': '/usr' } ) self.assertNoMoreResources() @@ -63,8 +64,9 @@ class TestWebHCatServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr /usr/lib/hcatalog/sbin/webhcat_server.sh stop', + self.assertResourceCalled('Execute', '/usr/lib/hcatalog/sbin/webhcat_server.sh stop', user = 'hcat', + environment = {'HADOOP_HOME': '/usr' } ) self.assertResourceCalled('File', '/var/run/webhcat/webhcat.pid', action = ['delete'], @@ -93,9 +95,10 @@ class TestWebHCatServer(RMFTestCase): ) self.assert_configure_secured() - self.assertResourceCalled('Execute', 'cd /var/run/webhcat ; env HADOOP_HOME=/usr /usr/lib/hcatalog/sbin/webhcat_server.sh start', + self.assertResourceCalled('Execute', 'cd /var/run/webhcat ; /usr/lib/hcatalog/sbin/webhcat_server.sh start', not_if = 'ls /var/run/webhcat/webhcat.pid >/dev/null 2>&1 && ps -p `cat /var/run/webhcat/webhcat.pid` >/dev/null 2>&1', - user = 'hcat' + user = 'hcat', + environment = {'HADOOP_HOME': '/usr' } ) self.assertNoMoreResources() @@ -108,8 +111,9 @@ class TestWebHCatServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr /usr/lib/hcatalog/sbin/webhcat_server.sh stop', + self.assertResourceCalled('Execute', '/usr/lib/hcatalog/sbin/webhcat_server.sh stop', user = 'hcat', + environment = {'HADOOP_HOME': '/usr' } ) self.assertResourceCalled('File', '/var/run/webhcat/webhcat.pid', action = ['delete'], http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 2ed5f14..b97fa34 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -48,7 +48,7 @@ class TestHiveMetastore(RMFTestCase): self.assert_configure_default() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive.pid` >/dev/null 2>&1', user = 'hive', path = ['/bin:/usr/lib/hive/bin:/usr/bin'], @@ -76,7 +76,7 @@ class TestHiveMetastore(RMFTestCase): self.assert_configure_default() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive.pid` >/dev/null 2>&1', user = 'hive', path = ['/bin:/usr/lib/hive/bin:/usr/bin'], @@ -139,7 +139,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, + environment = {'HADOOP_HOME': '/usr', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': 'hive'}, not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive.pid` >/dev/null 2>&1', user = 'hive', path = ['/bin:/usr/lib/hive/bin:/usr/bin'], http://git-wip-us.apache.org/repos/asf/ambari/blob/c23de40d/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index 0bbde77..a186527 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -69,8 +69,8 @@ class RMFTestCase(TestCase): os_env={'PATH':'/bin'}, target=TARGET_STACKS, mocks_dict={}, - try_install=False - ): + try_install=False, + command_args=[]): norm_path = os.path.normpath(path) src_dir = RMFTestCase.get_src_folder() if target == self.TARGET_STACKS: @@ -143,9 +143,9 @@ class RMFTestCase(TestCase): with patch.object(os, "environ", new=os_env) as mocks_dict['environ']: if not try_install: with patch.object(Script, 'install_packages') as install_mock_value: - method(RMFTestCase.env) + method(RMFTestCase.env, *command_args) else: - method(RMFTestCase.env) + method(RMFTestCase.env, *command_args) sys.path.remove(scriptsdir)
