Repository: ambari Updated Branches: refs/heads/trunk bdae70187 -> e671d0fe7
AMBARI-18238. HBase Master doesn't start after upgrading from HDP 2.3.6 using Ambari (dgrinenko via dlisnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e671d0fe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e671d0fe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e671d0fe Branch: refs/heads/trunk Commit: e671d0fe7afa68de9d0a6b55e748b68d2272c70b Parents: bdae701 Author: Lisnichenko Dmitro <[email protected]> Authored: Tue Aug 23 19:26:09 2016 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Tue Aug 23 19:29:14 2016 +0300 ---------------------------------------------------------------------- .../HBaseEnvMaxDirectMemorySizeAction.java | 87 +++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 4 + .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 4 + .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 9 + .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 9 + .../HBaseEnvMaxDirectMemorySizeActionTest.java | 194 +++++++++++++++++++ 6 files changed, 307 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java new file mode 100644 index 0000000..b238bca --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.upgrades; + +import com.google.inject.Inject; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Computes HBase Env content property. + * This class is only used when moving from HDP-2.3 to HDP-2.4 and HDP-2.3 to HDP-2.5 + */ +public class HBaseEnvMaxDirectMemorySizeAction extends AbstractServerAction { + private static final String SOURCE_CONFIG_TYPE = "hbase-env"; + private static final String CONTENT_NAME = "content"; + private static final String APPEND_CONTENT_LINE = "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\""; + private static final String CHECK_REGEX = "^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$"; + private static final Pattern REGEX = Pattern.compile(CHECK_REGEX, Pattern.MULTILINE); + + @Inject + private Clusters clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = clusters.getCluster(clusterName); + Config config = cluster.getDesiredConfigByType(SOURCE_CONFIG_TYPE); + + if (config == null) { + return createCommandReport(0, HostRoleStatus.FAILED,"{}", + String.format("Source type %s not found", SOURCE_CONFIG_TYPE), ""); + } + + Map<String, String> properties = config.getProperties(); + String content = properties.get(CONTENT_NAME); + + + if (content == null) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + String.format("The %s/%s property was not found. The -XX:MaxDirectMemorySize will not be added", SOURCE_CONFIG_TYPE, CONTENT_NAME), ""); + } + + Matcher m = REGEX.matcher(content); + + if (m.find()){ + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + String.format("The %s/%s property already contains an entry for %s and will not be modified", SOURCE_CONFIG_TYPE, CONTENT_NAME, APPEND_CONTENT_LINE), ""); + } + + String appendedContent = content + "\n" + APPEND_CONTENT_LINE; + properties.put(CONTENT_NAME, appendedContent); + + config.setProperties(properties); + config.persist(false); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + String.format("The %s/%s property was appended with %s", SOURCE_CONFIG_TYPE, CONTENT_NAME, APPEND_CONTENT_LINE),""); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index 450e350..eb6f333 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -273,6 +273,10 @@ <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/> + </execute-stage> + <!-- TEZ --> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index b9067c7..f5f06c0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -299,6 +299,10 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/> + </execute-stage> + <!-- TEZ --> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> <task xsi:type="configure" id="hdp_2_5_0_0_tez_client_adjust_tez_lib_uris_property"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 4e241f0..6583010 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -133,6 +133,15 @@ </service> </group> + <group name="HBASE" title="Update HBase Configuration"> + <skippable>true</skippable> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/> + </execute-stage> + + </group> + <group name="KAFKA" title="Kafka"> <skippable>true</skippable> <service name="KAFKA"> http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 7237a6a..09bd2ac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -150,6 +150,15 @@ </service> </group> + <group name="HBASE" title="Update HBase Configuration"> + <skippable>true</skippable> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/> + </execute-stage> + + </group> + <!-- This group should exist for all RUs that cross a major stack version. --> <group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs"> <direction>UPGRADE</direction> http://git-wip-us.apache.org/repos/asf/ambari/blob/e671d0fe/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java new file mode 100644 index 0000000..4c1d7a3 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import com.google.inject.Injector; +import junit.framework.Assert; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigImpl; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.easymock.EasyMock.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests HiveEnvClasspathAction logic + */ +public class HBaseEnvMaxDirectMemorySizeActionTest { + private Injector injector; + private Clusters clusters; + private Field m_clusterField; + + @Before + public void setup() throws Exception { + injector = EasyMock.createMock(Injector.class); + clusters = EasyMock.createMock(Clusters.class); + Cluster cluster = EasyMock.createMock(Cluster.class); + + Config hbaseEnv = new ConfigImpl("hbase-env") { + Map<String, String> mockProperties = new HashMap<String, String>() {{ + put("content","# Set environment variables here.\n" + + "\n" + + "# The java implementation to use. Java 1.6 required.\n" + + "export JAVA_HOME={{java64_home}}\n" + + "\n" + + "# HBase Configuration directory\n" + + "export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n" + + "\n" + + "# Extra Java CLASSPATH elements. Optional.\n" + + "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" + + "\n" + + "# The maximum amount of heap to use, in MB. Default is 1000.\n" + + "# export HBASE_HEAPSIZE=1000\n" + + "\n" + + "# Extra Java runtime options.\n" + + "# Below are what we set by default. May only work with SUN JVM.\n" + + "# For more on why as well as other possible settings,\n" + + "# see http://wiki.apache.org/hadoop/PerformanceTuning\n"; + + "export SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n" + + "# Uncomment below to enable java garbage collection logging.\n" + + "# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" + + "\n" + + "# Uncomment and adjust to enable JMX exporting\n" + + "# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n" + + "# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n"; + + "#\n" + + "# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n" + + "# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n" + + "# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n" + + "# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n" + + "\n" + + "# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\n" + + "export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n" + + "\n" + + "# Extra ssh options. Empty by default.\n" + + "# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n" + + "\n" + + "# Where log files are stored. $HBASE_HOME/logs by default.\n" + + "export HBASE_LOG_DIR={{log_dir}}\n" + + "\n" + + "# A string representing this instance of hbase. $USER by default.\n" + + "# export HBASE_IDENT_STRING=$USER\n" + + "\n" + + "# The scheduling priority for daemon processes. See 'man nice'.\n" + + "# export HBASE_NICENESS=10\n" + + "\n" + + "# The directory where pid files are stored. /tmp by default.\n" + + "export HBASE_PID_DIR={{pid_dir}}\n" + + "\n" + + "# Seconds to sleep between slave commands. Unset by default. This\n" + + "# can be useful in large clusters, where, e.g., slave rsyncs can\n" + + "# otherwise arrive faster than the master can service them.\n" + + "# export HBASE_SLAVE_SLEEP=0.1\n" + + "\n" + + "# Tell HBase whether it should manage it's own instance of Zookeeper or not.\n" + + "export HBASE_MANAGES_ZK=false\n" + + "\n" + + "{% if security_enabled %}\n" + + "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}} -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" + + "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\n" + + "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n" + + "{% else %}\n" + + "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" + + "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\n" + + "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" + + "{% endif %}"); + }}; + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map<String, String> properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + + + expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv).atLeastOnce(); + + expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce(); + + replay(injector, clusters, cluster); + + m_clusterField = HBaseEnvMaxDirectMemorySizeAction.class.getDeclaredField("clusters"); + m_clusterField.setAccessible(true); + } + + + @Test + public void testAction() throws Exception { + Pattern regex = Pattern.compile("^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$", Pattern.MULTILINE); + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put("clusterName", "c1"); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(hrc); + + HBaseEnvMaxDirectMemorySizeAction action = new HBaseEnvMaxDirectMemorySizeAction(); + + m_clusterField.set(action, clusters); + + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("hbase-env"); + Map<String, String> map = config.getProperties(); + + Assert.assertTrue(map.containsKey("content")); + String content = map.get("content"); + Matcher m = regex.matcher(content); + + assertEquals(true, m.find()); + } + +}
