This is an automated email from the ASF dual-hosted git repository. mpapirkovskyy pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push: new 4b6541b AMBARI-24263. Restart services just before stack upgrade fails due to AMS package incompatibility errors, causing EU not to be started. (mpapirkovskyy) (#1707) (#1726) 4b6541b is described below commit 4b6541b50f0161c6a4b823fb429a4db8aa8911e0 Author: Myroslav Papirkovskyi <mpapirkovs...@apache.org> AuthorDate: Mon Jul 9 20:43:51 2018 +0300 AMBARI-24263. Restart services just before stack upgrade fails due to AMS package incompatibility errors, causing EU not to be started. (mpapirkovskyy) (#1707) (#1726) * AMBARI-24263. Restart services just before stack upgrade fails due to AMS package incompatibility errors, causing EU not to be started. (mpapirkovskyy) * AMBARI-24263. Scriptlets logic update. Nullable metrics template. (mpapirkovskyy) * AMBARI-24263. Conditional metrics templates. (mpapirkovskyy) * AMBARI-24263. Templates typo fix. (mpapirkovskyy) * AMBARI-24263. Fix template in config. (mpapirkovskyy) * AMBARI-24263. Fix IFS usage side effects. (mpapirkovskyy) * AMBARI-24263. Restart services just before stack upgrade fails due to AMS package incompatibility errors, causing EU not to be started. (mpapirkovskyy) --- .../libraries/functions/constants.py | 1 + ambari-metrics/ambari-metrics-assembly/pom.xml | 5 +- .../src/main/package/deb/control/preinst | 45 +++++++++++++++++ .../src/main/package/rpm/sink/preinstall.sh | 45 +++++++++++++++++ .../ambari/server/upgrade/UpgradeCatalog270.java | 27 ++++++++++ .../ACCUMULO/1.6.1.2.2.0/package/scripts/params.py | 1 + .../hadoop-metrics2-accumulo.properties.j2 | 4 ++ .../0.1.0/package/scripts/hbase_service.py | 3 -- .../templates/hadoop-metrics2-hbase.properties.j2 | 2 +- .../0.96.0.2.0/package/scripts/hbase_service.py | 3 -- .../0.96.0.2.0/package/scripts/params_linux.py | 1 + ...oop-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 4 ++ .../hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 | 4 ++ .../HDFS/2.1.0.2.0/package/scripts/utils.py | 3 -- .../0.12.0.2.0/package/scripts/hive_service.py | 3 -- .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../hadoop-metrics2-hivemetastore.properties.j2 | 4 ++ .../hadoop-metrics2-hiveserver2.properties.j2 | 4 ++ .../templates/hadoop-metrics2-llapdaemon.j2 | 4 ++ .../templates/hadoop-metrics2-llaptaskscheduler.j2 | 4 ++ .../YARN/2.1.0.2.0/package/scripts/service.py | 3 -- .../stack-hooks/before-START/scripts/params.py | 2 + .../templates/hadoop-metrics2.properties.j2 | 4 ++ .../HDP/2.0.6/properties/stack_features.json | 5 ++ .../configuration/hadoop-metrics2.properties.xml | 5 ++ .../server/upgrade/UpgradeCatalog270Test.java | 58 ++++++++++++++++++++++ 26 files changed, 228 insertions(+), 17 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 443d639..f25b2b5 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -123,3 +123,4 @@ class StackFeature: KAFKA_EXTENDED_SASL_SUPPORT = "kafka_extended_sasl_support" OOZIE_EXTJS_INCLUDED = "oozie_extjs_included" MULTIPLE_ENV_SH_FILES_SUPPORT = "multiple_env_sh_files_support" + AMS_LEGACY_HADOOP_SINK = "ams_legacy_hadoop_sink" diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index 5e6b2d6..f89b9df 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -389,7 +389,10 @@ <defaultFilemode>644</defaultFilemode> <defaultUsername>root</defaultUsername> <defaultGroupname>root</defaultGroupname> - + <preinstallScriptlet> + <scriptFile>${project.build.directory}/resources/rpm/sink/preinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </preinstallScriptlet> <postinstallScriptlet> <scriptFile>${project.build.directory}/resources/rpm/sink/postinstall.sh</scriptFile> <fileEncoding>utf-8</fileEncoding> diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/package/deb/control/preinst b/ambari-metrics/ambari-metrics-assembly/src/main/package/deb/control/preinst new file mode 100644 index 0000000..7560fb9 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/package/deb/control/preinst @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +JAR_FILES_LEGACY_FOLDER="/usr/lib/ambari-metrics-sink-legacy" + +HADOOP_SINK_LINK="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" + +HADOOP_LEGACY_LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar" + +if [ -f ${HADOOP_SINK_LINK} ] +then + old_jar=$(readlink -f ${HADOOP_SINK_LINK}) + version_part=$(basename ${old_jar} | awk -F"-" '{print $7}') + IFS=. version=(${version_part}) + unset IFS + + if [[ ${version[0]} -le 2 && ${version[1]} -lt 7 ]] # backup only required on upgrade from version < 2.7 + then + if [ ! -d "$JAR_FILES_LEGACY_FOLDER" ] + then + mkdir -p "$JAR_FILES_LEGACY_FOLDER" + fi + echo "Backing up Ambari metrics hadoop sink jar ${old_jar} -> $JAR_FILES_LEGACY_FOLDER/" + cp "${old_jar}" "${JAR_FILES_LEGACY_FOLDER}/" + + HADOOP_SINK_LEGACY_JAR="$JAR_FILES_LEGACY_FOLDER/$(basename ${old_jar})" + echo "Creating symlink for backup jar $HADOOP_LEGACY_LINK_NAME -> $HADOOP_SINK_LEGACY_JAR" + rm -f "${HADOOP_LEGACY_LINK_NAME}" ; ln -s "${HADOOP_SINK_LEGACY_JAR}" "${HADOOP_LEGACY_LINK_NAME}" + fi +fi + +exit 0 diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/sink/preinstall.sh b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/sink/preinstall.sh new file mode 100644 index 0000000..7560fb9 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/sink/preinstall.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +JAR_FILES_LEGACY_FOLDER="/usr/lib/ambari-metrics-sink-legacy" + +HADOOP_SINK_LINK="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" + +HADOOP_LEGACY_LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar" + +if [ -f ${HADOOP_SINK_LINK} ] +then + old_jar=$(readlink -f ${HADOOP_SINK_LINK}) + version_part=$(basename ${old_jar} | awk -F"-" '{print $7}') + IFS=. version=(${version_part}) + unset IFS + + if [[ ${version[0]} -le 2 && ${version[1]} -lt 7 ]] # backup only required on upgrade from version < 2.7 + then + if [ ! -d "$JAR_FILES_LEGACY_FOLDER" ] + then + mkdir -p "$JAR_FILES_LEGACY_FOLDER" + fi + echo "Backing up Ambari metrics hadoop sink jar ${old_jar} -> $JAR_FILES_LEGACY_FOLDER/" + cp "${old_jar}" "${JAR_FILES_LEGACY_FOLDER}/" + + HADOOP_SINK_LEGACY_JAR="$JAR_FILES_LEGACY_FOLDER/$(basename ${old_jar})" + echo "Creating symlink for backup jar $HADOOP_LEGACY_LINK_NAME -> $HADOOP_SINK_LEGACY_JAR" + rm -f "${HADOOP_LEGACY_LINK_NAME}" ; ln -s "${HADOOP_SINK_LEGACY_JAR}" "${HADOOP_LEGACY_LINK_NAME}" + fi +fi + +exit 0 diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java index 43bbc87..8659082 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java @@ -1062,6 +1062,7 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog { updateSolrConfigurations(); updateAmsConfigs(); updateStormConfigs(); + clearHadoopMetrics2Content(); } protected void renameAmbariInfra() { @@ -1942,4 +1943,30 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog { } } } + + protected void clearHadoopMetrics2Content() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + String hadoopMetrics2ContentProperty = "content"; + String hadoopMetrics2ContentValue = ""; + String hadoopMetrics2ConfigType = "hadoop-metrics2.properties"; + for (final Cluster cluster : clusterMap.values()) { + Config config = cluster.getDesiredConfigByType(hadoopMetrics2ConfigType); + if (config != null) { + Map<String, String> hadoopMetrics2Configs = config.getProperties(); + if (hadoopMetrics2Configs.containsKey(hadoopMetrics2ContentProperty)) { + LOG.info("Updating " + hadoopMetrics2ContentProperty); + Map<String, String> updateProperty = Collections.singletonMap(hadoopMetrics2ContentProperty, hadoopMetrics2ContentValue); + updateConfigurationPropertiesForCluster(cluster, hadoopMetrics2ConfigType, updateProperty, Collections.EMPTY_SET, + true, false); + } + } + } + } + } + } } diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py index d87d9c2..80cb4c2 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py @@ -150,6 +150,7 @@ if has_metric_collector: metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + metric_legacy_hadoop_sink = check_stack_feature(StackFeature.AMS_LEGACY_HADOOP_SINK, stack_version_formatted) pass metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 index 282f904..13def8e 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 @@ -24,7 +24,11 @@ {% if has_metric_collector %} +{% if metric_legacy_hadoop_sink %} +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar +{% else %} *.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +{% endif %} *.sink.timeline.slave.host.name={{hostname}} accumulo.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink accumulo.period={{metrics_collection_period}} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase_service.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase_service.py index 6105139..d322f51 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase_service.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase_service.py @@ -20,7 +20,6 @@ limitations under the License. from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions.format import format -from ambari_commons.repo_manager.repo_manager_helper import check_installed_metrics_hadoop_sink_version def hbase_service( name, @@ -34,8 +33,6 @@ def hbase_service( no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") if action == 'start': - # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 - check_installed_metrics_hadoop_sink_version() daemon_cmd = format("{cmd} start {role}") diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 index 4a6cd29..a7c41a5 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -59,4 +59,4 @@ hbase.sink.timeline.truststore.password = {{metric_truststore_password}} # Switch off metrics generation on a per region basis *.source.filter.class=org.apache.hadoop.metrics2.filter.RegexFilter -hbase.*.source.filter.exclude=.*(Regions|Users|Tables).* \ No newline at end of file +hbase.*.source.filter.exclude=.*(Regions|Users|Tables).* diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py index 6928182..474c74c 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py @@ -22,7 +22,6 @@ from resource_management.libraries.functions.format import format from resource_management.libraries.functions.show_logs import show_logs from resource_management.core.shell import as_sudo from resource_management.core.resources.system import Execute, File -from ambari_commons.repo_manager.repo_manager_helper import check_installed_metrics_hadoop_sink_version def hbase_service( name, @@ -37,8 +36,6 @@ def hbase_service( no_op_test = as_sudo(["test", "-f", pid_file]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") if action == 'start': - # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 - check_installed_metrics_hadoop_sink_version() daemon_cmd = format("{cmd} start {role}") diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 0552275..0f01ea3 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -186,6 +186,7 @@ if has_metric_collector: metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + metric_legacy_hadoop_sink = check_stack_feature(StackFeature.AMS_LEGACY_HADOOP_SINK, version_for_stack_feature_checks) pass metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index 66796b4..2811d72 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -49,7 +49,11 @@ hbase.extendedperiod = 3600 {% if has_metric_collector %} +{% if metric_legacy_hadoop_sink %} +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar +{% else %} *.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +{% endif %} *.sink.timeline.slave.host.name={{hostname}} hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index 4ed68ba..d15c842 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -48,7 +48,11 @@ hbase.extendedperiod = 3600 {% if has_metric_collector %} +{% if metric_legacy_hadoop_sink %} +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar +{% else %} *.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +{% endif %} *.sink.timeline.slave.host.name={{hostname}} hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period={{metrics_collection_period}} diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py index f6cea7a..9d146e0 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py @@ -37,7 +37,6 @@ from resource_management.libraries.functions.curl_krb_request import curl_krb_re from resource_management.libraries.script.script import Script from resource_management.libraries.functions.namenode_ha_utils import get_namenode_states from resource_management.libraries.functions.show_logs import show_logs -from ambari_commons.repo_manager.repo_manager_helper import check_installed_metrics_hadoop_sink_version from ambari_commons.inet_utils import ensure_ssl_using_protocol from zkfc_slave import ZkfcSlaveDefault @@ -270,8 +269,6 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False, daemon_cmd = as_user(cmd, user) if action == "start": - # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 - check_installed_metrics_hadoop_sink_version() # remove pid file from dead process File(pid_file, action="delete", not_if=process_id_exists_command) diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index 9bc2601..4a9ecc9 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -36,7 +36,6 @@ from resource_management.libraries.functions.stack_features import check_stack_f from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from ambari_commons.repo_manager.repo_manager_helper import check_installed_metrics_hadoop_sink_version @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -73,8 +72,6 @@ def hive_service(name, action='start', upgrade_type=None): process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") if action == 'start': - # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 - check_installed_metrics_hadoop_sink_version() if name == 'hiveserver2': check_fs_root(params.hive_server_conf_dir, params.execute_path) diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index d0a9d3d..77ad09a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -574,6 +574,7 @@ if has_metric_collector: metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + metric_legacy_hadoop_sink = check_stack_feature(StackFeature.AMS_LEGACY_HADOOP_SINK, version_for_stack_feature_checks) metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 index d4573c3..6602cee 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 @@ -37,7 +37,11 @@ {% if has_metric_collector %} *.period={{metrics_collection_period}} + {% if metric_legacy_hadoop_sink %} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar + {% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + {% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 index c67d002..a4c9968 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 @@ -37,7 +37,11 @@ {% if has_metric_collector %} *.period={{metrics_collection_period}} + {% if metric_legacy_hadoop_sink %} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar + {% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + {% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 index cd23e8a..0ce7d8e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 @@ -37,7 +37,11 @@ {% if has_metric_collector %} *.period={{metrics_collection_period}} + {% if metric_legacy_hadoop_sink %} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar + {% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + {% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 index 674d3cc..023e689 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 @@ -37,7 +37,11 @@ {% if has_metric_collector %} *.period={{metrics_collection_period}} + {% if metric_legacy_hadoop_sink %} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar + {% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + {% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py index cc790e9..d684080 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py @@ -21,7 +21,6 @@ Ambari Agent from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from ambari_commons.repo_manager.repo_manager_helper import check_installed_metrics_hadoop_sink_version from resource_management.core.shell import as_user, as_sudo from resource_management.libraries.functions.show_logs import show_logs from resource_management.libraries.functions.format import format @@ -65,8 +64,6 @@ def service(componentName, action='start', serviceName='yarn'): cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}") if action == 'start': - # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 - check_installed_metrics_hadoop_sink_version() daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file]) diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py index e3fa476..0f29e24 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py @@ -162,8 +162,10 @@ if has_metric_collector: metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + metric_legacy_hadoop_sink = check_stack_feature(StackFeature.AMS_LEGACY_HADOOP_SINK, version_for_stack_feature_checks) pass + metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stack-hooks/before-START/templates/hadoop-metrics2.properties.j2 index 281ac27..49be9c4 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/templates/hadoop-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/stack-hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -67,7 +67,11 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue {% if has_metric_collector %} *.period={{metrics_collection_period}} +{% if metric_legacy_hadoop_sink %} +*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar +{% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +{% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 6d622ec..417f9bb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -458,6 +458,11 @@ "description": "ExtJS is included in the repository and automatically installed by Ambari", "min_version": "2.2.0.0", "max_version": "2.6.0.0" + }, + { + "name": "ams_legacy_hadoop_sink", + "description": "Legacy AMS hadoop sink should be used", + "max_version": "2.6.99.99" } ] } diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml index 02be755..4fc8bd0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml @@ -78,7 +78,11 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue {% if has_metric_collector %} *.period={{metrics_collection_period}} +{% if metric_legacy_hadoop_sink %} +*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink-legacy.jar +{% else %} *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +{% endif %} *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 @@ -124,6 +128,7 @@ namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}} </value> <value-attributes> <type>content</type> + <empty-value-valid>true</empty-value-valid> </value-attributes> <on-ambari-upgrade add="false"/> </property> diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java index 68c6b1f..477da85 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java @@ -330,6 +330,7 @@ public class UpgradeCatalog270Test { Method updateSolrConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateSolrConfigurations"); Method updateAmsConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateAmsConfigs"); Method updateStormConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateStormConfigs"); + Method clearHadoopMetrics2Content = UpgradeCatalog270.class.getDeclaredMethod("clearHadoopMetrics2Content"); UpgradeCatalog270 upgradeCatalog270 = createMockBuilder(UpgradeCatalog270.class) .addMockedMethod(showHcatDeletedUserMessage) @@ -346,6 +347,7 @@ public class UpgradeCatalog270Test { .addMockedMethod(updateSolrConfigurations) .addMockedMethod(updateAmsConfigurations) .addMockedMethod(updateStormConfigurations) + .addMockedMethod(clearHadoopMetrics2Content) .createMock(); @@ -389,6 +391,9 @@ public class UpgradeCatalog270Test { upgradeCatalog270.updateStormConfigs(); expectLastCall().once(); + upgradeCatalog270.clearHadoopMetrics2Content(); + expectLastCall().once(); + replay(upgradeCatalog270); upgradeCatalog270.executeDMLUpdates(); @@ -1664,4 +1669,57 @@ public class UpgradeCatalog270Test { assertTrue(Maps.difference(newStormProperties, updatedProperties).areEqual()); } + + @Test + public void testClearHadoopMetrics2Content() throws Exception { + + Map<String, String> oldContentProperty = new HashMap<String, String>() { + { + put("content", "# Licensed to the Apache Software Foundation (ASF) under one or more..."); + } + }; + Map<String, String> newContentProperty = new HashMap<String, String>() { + { + put("content", ""); + } + }; + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + Config mockHadoopMetrics2Properties = easyMockSupport.createNiceMock(Config.class); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getDesiredConfigByType("hadoop-metrics2.properties")).andReturn(mockHadoopMetrics2Properties).atLeastOnce(); + expect(mockHadoopMetrics2Properties.getProperties()).andReturn(oldContentProperty).anyTimes(); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + + replay(injector, clusters, mockHadoopMetrics2Properties, cluster); + + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .withConstructor(createNiceMock(ActionManager.class), clusters, injector) + .createNiceMock(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + Capture<Map> propertiesCapture = EasyMock.newCapture(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector2); + new UpgradeCatalog270(injector2).clearHadoopMetrics2Content(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedProperties = propertiesCapture.getValue(); + assertTrue(Maps.difference(newContentProperty, updatedProperties).areEqual()); + + } }