http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer.log ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer.log b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer.log new file mode 100644 index 0000000..2010c02 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer.log @@ -0,0 +1,29 @@ +Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved +Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB +Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB +Jul 28, 2014 5:04:07 PM 2 0 B 5.40 GB 9.79 GB +Jul 28, 2014 5:05:14 PM 3 0 B 5.06 GB 9.79 GB +Jul 28, 2014 5:05:50 PM 4 0 B 5.06 GB 9.79 GB +Jul 28, 2014 5:06:56 PM 5 0 B 4.81 GB 9.79 GB +Jul 28, 2014 5:07:33 PM 6 0 B 4.80 GB 9.79 GB +Jul 28, 2014 5:09:11 PM 7 0 B 4.29 GB 9.79 GB +Jul 28, 2014 5:09:47 PM 8 0 B 4.29 GB 9.79 GB +Jul 28, 2014 5:11:24 PM 9 0 B 3.89 GB 9.79 GB +Jul 28, 2014 5:12:00 PM 10 0 B 3.86 GB 9.79 GB +Jul 28, 2014 5:13:37 PM 11 0 B 3.23 GB 9.79 GB +Jul 28, 2014 5:15:13 PM 12 0 B 2.53 GB 9.79 GB +Jul 28, 2014 5:15:49 PM 13 0 B 2.52 GB 9.79 GB +Jul 28, 2014 5:16:25 PM 14 0 B 2.51 GB 9.79 GB +Jul 28, 2014 5:17:01 PM 15 0 B 2.39 GB 9.79 GB +Jul 28, 2014 5:17:37 PM 16 0 B 2.38 GB 9.79 GB +Jul 28, 2014 5:18:14 PM 17 0 B 2.31 GB 9.79 GB +Jul 28, 2014 5:18:50 PM 18 0 B 2.30 GB 9.79 GB +Jul 28, 2014 5:19:26 PM 19 0 B 2.21 GB 9.79 GB +Jul 28, 2014 5:20:02 PM 20 0 B 2.10 GB 9.79 GB +Jul 28, 2014 5:20:38 PM 21 0 B 2.06 GB 9.79 GB +Jul 28, 2014 5:22:14 PM 22 0 B 1.68 GB 9.79 GB +Jul 28, 2014 5:23:20 PM 23 0 B 1.00 GB 9.79 GB +Jul 28, 2014 5:23:56 PM 24 0 B 1016.16 MB 9.79 GB +Jul 28, 2014 5:25:33 PM 25 0 B 30.55 MB 9.79 GB +The cluster is balanced. Exiting... +Balancing took 24.858033333333335 minutes
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py new file mode 100644 index 0000000..df173fe --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import time +import sys +from threading import Thread + + +def write_function(path, handle, interval): + with open(path) as f: + for line in f: + handle.write(line) + handle.flush() + time.sleep(interval) + +thread = Thread(target = write_function, args = ('balancer.log', sys.stdout, 1)) +thread.start() + +threaderr = Thread(target = write_function, args = ('balancer-err.log', sys.stderr, 0.3)) +threaderr.start() + +thread.join() + + +def rebalancer_out(): + write_function('balancer.log', sys.stdout) + +def rebalancer_err(): + write_function('balancer-err.log', sys.stdout) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_rebalance.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_rebalance.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_rebalance.py new file mode 100644 index 0000000..1dc545e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_rebalance.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import re + +class HdfsParser(): + def __init__(self): + self.initialLine = None + self.state = None + + def parseLine(self, line): + hdfsLine = HdfsLine() + type, matcher = hdfsLine.recognizeType(line) + if(type == HdfsLine.LineType.HeaderStart): + self.state = 'PROCESS_STARTED' + elif (type == HdfsLine.LineType.Progress): + self.state = 'PROGRESS' + hdfsLine.parseProgressLog(line, matcher) + if(self.initialLine == None): self.initialLine = hdfsLine + + return hdfsLine + elif (type == HdfsLine.LineType.ProgressEnd): + self.state = 'PROCESS_FINISED' + return None + +class HdfsLine(): + + class LineType: + HeaderStart, Progress, ProgressEnd, Unknown = range(4) + + + MEMORY_SUFFIX = ['B','KB','MB','GB','TB','PB','EB'] + MEMORY_PATTERN = '(?P<memmult_%d>(?P<memory_%d>(\d+)(.|,)?(\d+)?) (?P<mult_%d>'+"|".join(MEMORY_SUFFIX)+'))' + + HEADER_BEGIN_PATTERN = re.compile('Time Stamp\w+Iteration#\w+Bytes Already Moved\w+Bytes Left To Move\w+Bytes Being Moved') + PROGRESS_PATTERN = re.compile( + "(?P<date>.*?)\s+" + + "(?P<iteration>\d+)\s+" + + MEMORY_PATTERN % (1,1,1) + "\s+" + + MEMORY_PATTERN % (2,2,2) + "\s+" + + MEMORY_PATTERN % (3,3,3) + ) + PROGRESS_END_PATTERN = re.compile('(The cluster is balanced. Exiting...|The cluster is balanced. Exiting...)') + + def __init__(self): + self.date = None + self.iteration = None + self.bytesAlreadyMoved = None + self.bytesLeftToMove = None + self.bytesBeingMoved = None + self.bytesAlreadyMovedStr = None + self.bytesLeftToMoveStr = None + self.bytesBeingMovedStr = None + + def recognizeType(self, line): + for (type, pattern) in ( + (HdfsLine.LineType.HeaderStart, self.HEADER_BEGIN_PATTERN), + (HdfsLine.LineType.Progress, self.PROGRESS_PATTERN), + (HdfsLine.LineType.ProgressEnd, self.PROGRESS_END_PATTERN) + ): + m = re.match(pattern, line) + if m: + return type, m + return HdfsLine.LineType.Unknown, None + + def parseProgressLog(self, line, m): + ''' + Parse the line of 'hdfs rebalancer' output. The example output being parsed: + + Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved + Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB + Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB + + Throws AmbariException in case of parsing errors + + ''' + m = re.match(self.PROGRESS_PATTERN, line) + if m: + self.date = m.group('date') + self.iteration = int(m.group('iteration')) + + self.bytesAlreadyMoved = self.parseMemory(m.group('memory_1'), m.group('mult_1')) + self.bytesLeftToMove = self.parseMemory(m.group('memory_2'), m.group('mult_2')) + self.bytesBeingMoved = self.parseMemory(m.group('memory_3'), m.group('mult_3')) + + self.bytesAlreadyMovedStr = m.group('memmult_1') + self.bytesLeftToMoveStr = m.group('memmult_2') + self.bytesBeingMovedStr = m.group('memmult_3') + else: + raise AmbariException("Failed to parse line [%s]") + + def parseMemory(self, memorySize, multiplier_type): + try: + factor = self.MEMORY_SUFFIX.index(multiplier_type) + except ValueError: + raise AmbariException("Failed to memory value [%s %s]" % (memorySize, multiplier_type)) + + return float(memorySize) * (1024 ** factor) + def toJson(self): + return { + 'timeStamp' : self.date, + 'iteration' : self.iteration, + + 'dataMoved': self.bytesAlreadyMovedStr, + 'dataLeft' : self.bytesLeftToMoveStr, + 'dataBeingMoved': self.bytesBeingMovedStr, + + 'bytesMoved': self.bytesAlreadyMoved, + 'bytesLeft' : self.bytesLeftToMove, + 'bytesBeingMoved': self.bytesBeingMoved, + } + def __str__(self): + return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py index 1978881..3b320d1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py @@ -20,6 +20,13 @@ limitations under the License. from resource_management import * from hdfs_namenode import namenode from hdfs import hdfs +import time +import json +import subprocess +import hdfs_rebalance +import sys +import os +from datetime import datetime class NameNode(Script): @@ -66,6 +73,51 @@ class NameNode(Script): env.set_params(params) namenode(action="decommission") pass + + def rebalancehdfs(self, env): + import params + env.set_params(params) + name_node_parameters = json.loads( params.name_node_params ) + threshold = name_node_parameters['threshold'] + print "Starting balancer with threshold = %s" % threshold + + def calculateCompletePercent(first, current): + return 1.0 - current.bytesLeftToMove/first.bytesLeftToMove + + + def startRebalancingProcess(threshold): + rebalanceCommand = format('hadoop --config {hadoop_conf_dir} balancer -threshold {threshold}') + return ['su','-',params.hdfs_user,'-c', rebalanceCommand] + + command = startRebalancingProcess(threshold) + + basedir = os.path.join(env.config.basedir, 'scripts') + if(threshold == 'DEBUG'): #FIXME TODO remove this on PROD + basedir = os.path.join(env.config.basedir, 'scripts', 'balancer-emulator') + command = ['python','hdfs-command.py'] + + print "Executing command %s" % command + + parser = hdfs_rebalance.HdfsParser() + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=False, + close_fds=True, + cwd=basedir + ) + for line in iter(proc.stdout.readline, ''): + sys.stdout.write('[balancer] %s %s' % (str(datetime.now()), line )) + pl = parser.parseLine(line) + if pl: + res = pl.toJson() + res['completePercent'] = calculateCompletePercent(parser.initialLine, pl) + + self.put_structured_out(res) + elif parser.state == 'PROCESS_FINISED' : + sys.stdout.write('[balancer] %s %s' % (str(datetime.now()), 'Process is finished' )) + self.put_structured_out({'completePercent' : 1}) + break + + if __name__ == "__main__": NameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py index 8f6c7e4..9a5e393 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py @@ -184,3 +184,4 @@ if not "com.hadoop.compression.lzo" in io_compression_codecs: exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"] else: exclude_packages = [] +name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java index 17e8724..60ed592 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java @@ -46,7 +46,9 @@ import org.apache.ambari.server.agent.ActionQueue; import org.apache.ambari.server.agent.AgentCommand; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper; import org.apache.ambari.server.controller.HostsMap; import org.apache.ambari.server.serveraction.ServerAction; import org.apache.ambari.server.serveraction.ServerActionManagerImpl; @@ -742,6 +744,74 @@ public class TestActionScheduler { Assert.assertEquals(HostRoleStatus.PENDING, stages.get(3).getHostRoleStatus(hostname3, "DATANODE")); Assert.assertEquals(HostRoleStatus.PENDING, stages.get(4).getHostRoleStatus(hostname4, "GANGLIA_MONITOR")); } + /** + * Verifies that ActionScheduler allows to execute background tasks in parallel + */ + @Test + public void testBackgroundStagesExecutionEnable() throws Exception { + ActionQueue aq = new ActionQueue(); + Clusters fsm = mock(Clusters.class); + Cluster oneClusterMock = mock(Cluster.class); + Service serviceObj = mock(Service.class); + ServiceComponent scomp = mock(ServiceComponent.class); + ServiceComponentHost sch = mock(ServiceComponentHost.class); + UnitOfWork unitOfWork = mock(UnitOfWork.class); + RequestFactory requestFactory = mock(RequestFactory.class); + when(fsm.getCluster(anyString())).thenReturn(oneClusterMock); + when(oneClusterMock.getService(anyString())).thenReturn(serviceObj); + when(serviceObj.getServiceComponent(anyString())).thenReturn(scomp); + when(scomp.getServiceComponentHost(anyString())).thenReturn(sch); + when(serviceObj.getCluster()).thenReturn(oneClusterMock); + + String hostname1 = "ahost.ambari.apache.org"; + String hostname2 = "bhost.ambari.apache.org"; + HashMap<String, ServiceComponentHost> hosts = + new HashMap<String, ServiceComponentHost>(); + hosts.put(hostname1, sch); + hosts.put(hostname2, sch); + when(scomp.getServiceComponentHosts()).thenReturn(hosts); + + List<Stage> stages = new ArrayList<Stage>(); + Stage backgroundStage = null; + stages.add(//stage with background command + backgroundStage = getStageWithSingleTask( + hostname1, "cluster1", Role.NAMENODE, RoleCommand.CUSTOM_COMMAND, "REBALANCEHDFS", Service.Type.HDFS, 1, 1, 1)); + + Assert.assertEquals(AgentCommandType.BACKGROUND_EXECUTION_COMMAND ,backgroundStage.getExecutionCommands(hostname1).get(0).getExecutionCommand().getCommandType()); + + stages.add( // Stage with the same hostname, should be scheduled + getStageWithSingleTask( + hostname1, "cluster1", Role.GANGLIA_MONITOR, + RoleCommand.START, Service.Type.GANGLIA, 2, 2, 2)); + + stages.add( + getStageWithSingleTask( + hostname2, "cluster1", Role.DATANODE, + RoleCommand.START, Service.Type.HDFS, 3, 3, 3)); + + + ActionDBAccessor db = mock(ActionDBAccessor.class); + when(db.getStagesInProgress()).thenReturn(stages); + + Properties properties = new Properties(); + properties.put(Configuration.PARALLEL_STAGE_EXECUTION_KEY, "true"); + Configuration conf = new Configuration(properties); + ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3, + new HostsMap((String) null), new ServerActionManagerImpl(fsm), + unitOfWork, conf); + + ActionManager am = new ActionManager( + 2, 2, aq, fsm, db, new HostsMap((String) null), + new ServerActionManagerImpl(fsm), unitOfWork, + requestFactory, conf); + + scheduler.doWork(); + + Assert.assertEquals(HostRoleStatus.QUEUED, stages.get(0).getHostRoleStatus(hostname1, "NAMENODE")); + Assert.assertEquals(HostRoleStatus.QUEUED, stages.get(2).getHostRoleStatus(hostname2, "DATANODE")); + + Assert.assertEquals(HostRoleStatus.QUEUED, stages.get(1).getHostRoleStatus(hostname1, "GANGLIA_MONITOR")); + } @Test @@ -1234,6 +1304,19 @@ public class TestActionScheduler { return stage; } + private Stage getStageWithSingleTask(String hostname, String clusterName, Role role, RoleCommand roleCommand, + String customCommandName, Service.Type service, int taskId, int stageId, int requestId) { + Stage stage = getStageWithSingleTask(hostname, clusterName, role, roleCommand, service, taskId, stageId, requestId); + + HostRoleCommand cmd = stage.getHostRoleCommand(hostname, role.name()); + if (cmd != null) { + cmd.setCustomCommandName(customCommandName); + } + + stage.getExecutionCommandWrapper(hostname, role.toString()).getExecutionCommand().setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND); + return stage; + } + private void addInstallTaskToStage(Stage stage, String hostname, String clusterName, Role role, RoleCommand roleCommand, Service.Type service, http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 172a2d6..9f0ccb0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -1232,13 +1232,18 @@ public class AmbariMetaInfoTest { ccd = findCustomCommand("YET_ANOTHER_PARENT_COMMAND", component); Assert.assertEquals("scripts/yet_another_parent_command.py", ccd.getCommandScript().getScript()); - - Assert.assertEquals(2, component.getCustomCommands().size()); + + ccd = findCustomCommand("REBALANCEHDFS", component); + Assert.assertEquals("scripts/namenode.py", + ccd.getCommandScript().getScript()); + Assert.assertTrue(ccd.isBackground()); + + Assert.assertEquals(3, component.getCustomCommands().size()); // Test custom command script inheritance component = metaInfo.getComponent(STACK_NAME_HDP, "2.0.8", "HDFS", "NAMENODE"); - Assert.assertEquals(3, component.getCustomCommands().size()); + Assert.assertEquals(4, component.getCustomCommands().size()); ccd = findCustomCommand("YET_ANOTHER_PARENT_COMMAND", component); Assert.assertEquals("scripts/yet_another_parent_command.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 07730dd..43c49de 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -7244,8 +7244,9 @@ public class AmbariManagementControllerTest { StackServiceComponentResponse response = responses.iterator().next(); assertNotNull(response.getCustomCommands()); - assertEquals(1, response.getCustomCommands().size()); + assertEquals(2, response.getCustomCommands().size()); assertEquals("DECOMMISSION", response.getCustomCommands().get(0)); + assertEquals("REBALANCEHDFS", response.getCustomCommands().get(1)); StackServiceComponentRequest journalNodeRequest = new StackServiceComponentRequest( STACK_NAME, NEW_STACK_VERSION, SERVICE_NAME, "JOURNALNODE"); http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java new file mode 100644 index 0000000..d49cadd --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller; + +import static org.mockito.Matchers.any; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import junit.framework.Assert; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.internal.ComponentResourceProviderTest; +import org.apache.ambari.server.controller.internal.RequestResourceFilter; +import org.apache.ambari.server.controller.internal.ServiceResourceProviderTest; +import org.apache.ambari.server.orm.GuiceJpaInitializer; +import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.HostState; +import org.apache.ambari.server.state.State; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; + +@RunWith(MockitoJUnitRunner.class) +public class BackgroundCustomCommandExecutionTest { + private Injector injector; + private AmbariManagementController controller; + private AmbariMetaInfo ambariMetaInfo; + private Configuration configuration; + private Clusters clusters; + + + private static final String REQUEST_CONTEXT_PROPERTY = "context"; + + @Captor ArgumentCaptor<List<Stage>> stagesCaptor; + @Mock ActionManager am; + + @Before + public void setup() throws Exception { + InMemoryDefaultTestModule module = new InMemoryDefaultTestModule(){ + + + @Override + protected void configure() { + getProperties().put(Configuration.CUSTOM_ACTION_DEFINITION_KEY, "src/main/resources/custom_action_definitions"); + super.configure(); + bind(ActionManager.class).toInstance(am); + } + }; + injector = Guice.createInjector(module); + + + injector.getInstance(GuiceJpaInitializer.class); + controller = injector.getInstance(AmbariManagementController.class); + clusters = injector.getInstance(Clusters.class); + configuration = injector.getInstance(Configuration.class); + + Assert.assertEquals("src/main/resources/custom_action_definitions", configuration.getCustomActionDefinitionPath()); + + ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); + ambariMetaInfo.init(); + } + @After + public void teardown() { + injector.getInstance(PersistService.class).stop(); + } + + @SuppressWarnings("serial") + @Test + public void testRebalanceHdfsCustomCommand() { + try { + createClusterFixture(); + + Map<String, String> requestProperties = new HashMap<String, String>() { + { + put(REQUEST_CONTEXT_PROPERTY, "Refresh YARN Capacity Scheduler"); + put("command", "REBALANCEHDFS"); + put("namenode" , "{\"threshold\":13}");//case is important here + } + }; + + ExecuteActionRequest actionRequest = new ExecuteActionRequest("c1", + "REBALANCEHDFS", new HashMap<String, String>()); + actionRequest.getResourceFilters().add(new RequestResourceFilter("HDFS", "NAMENODE",Collections.singletonList("c6401"))); + + controller.createAction(actionRequest, requestProperties); + + Mockito.verify(am, Mockito.times(1)).sendActions(stagesCaptor.capture(), any(ExecuteActionRequest.class)); + + + List<Stage> stages = stagesCaptor.getValue(); + Assert.assertEquals(1, stages.size()); + Stage stage = stages.get(0); + + System.out.println(stage); + + Assert.assertEquals(1, stage.getHosts().size()); + + List<ExecutionCommandWrapper> commands = stage.getExecutionCommands("c6401"); + Assert.assertEquals(1, commands.size()); + + ExecutionCommand command = commands.get(0).getExecutionCommand(); + + Assert.assertEquals(AgentCommandType.BACKGROUND_EXECUTION_COMMAND, command.getCommandType()); + Assert.assertEquals("{\"threshold\":13}", command.getCommandParams().get("namenode")); + + } catch (AmbariException e) { + Assert.fail(e.getMessage()); + } + } + @SuppressWarnings("serial") + @Test + public void testCancelCommand() { + try { + createClusterFixture(); + + Map<String, String> requestProperties = new HashMap<String, String>() { + { + put(REQUEST_CONTEXT_PROPERTY, "Stop background command"); +// put("cancel_policy","SIGKILL"); +// put("cancel_task_id","19"); + } + }; + + ExecuteActionRequest actionRequest = new ExecuteActionRequest( + "c1", + "actionexecute","cancel_background_task", + null, + null, + new HashMap<String, String>(){{ + put("cancel_policy","SIGKILL"); // parameters/cancel_policy -- in request params + put("cancel_task_id","19"); + }}); + actionRequest.getResourceFilters().add(new RequestResourceFilter("HDFS", "NAMENODE", Collections.singletonList("c6401"))); + + controller.createAction(actionRequest, requestProperties); + + Mockito.verify(am, Mockito.times(1)).sendActions(stagesCaptor.capture(), any(ExecuteActionRequest.class)); + + List<Stage> stages = stagesCaptor.getValue(); + Assert.assertEquals(1, stages.size()); + Stage stage = stages.get(0); + + Assert.assertEquals(1, stage.getHosts().size()); + + List<ExecutionCommandWrapper> commands = stage.getExecutionCommands("c6401"); + Assert.assertEquals(1, commands.size()); + + ExecutionCommand command = commands.get(0).getExecutionCommand(); + + Assert.assertEquals(AgentCommandType.EXECUTION_COMMAND, command.getCommandType()); + Assert.assertEquals("ACTIONEXECUTE", command.getRoleCommand().name()); + Assert.assertEquals("cancel_background_task.py", command.getCommandParams().get("script")); + Assert.assertEquals("SIGKILL", command.getCommandParams().get("cancel_policy")); + Assert.assertEquals("19", command.getCommandParams().get("cancel_task_id")); + + + } catch (AmbariException e) { + Assert.fail(e.getMessage()); + } + } + + private void createClusterFixture() throws AmbariException { + createCluster("c1"); + addHost("c6401","c1"); + addHost("c6402","c1"); + + clusters.getCluster("c1"); + createService("c1", "HDFS", null); + + createServiceComponent("c1","HDFS","NAMENODE", State.INIT); + + createServiceComponentHost("c1","HDFS","NAMENODE","c6401", null); + } + private void addHost(String hostname, String clusterName) throws AmbariException { + clusters.addHost(hostname); + setOsFamily(clusters.getHost(hostname), "redhat", "6.3"); + clusters.getHost(hostname).setState(HostState.HEALTHY); + clusters.getHost(hostname).persist(); + if (null != clusterName) + clusters.mapHostToCluster(hostname, clusterName); + } + private void setOsFamily(Host host, String osFamily, String osVersion) { + Map<String, String> hostAttributes = new HashMap<String, String>(); + hostAttributes.put("os_family", osFamily); + hostAttributes.put("os_release_version", osVersion); + + host.setHostAttributes(hostAttributes); + } + + private void createCluster(String clusterName) throws AmbariException { + ClusterRequest r = new ClusterRequest(null, clusterName, State.INSTALLED.name(), "HDP-2.0.6", null); + controller.createCluster(r); + } + + private void createService(String clusterName, + String serviceName, State desiredState) throws AmbariException { + String dStateStr = null; + if (desiredState != null) { + dStateStr = desiredState.toString(); + } + ServiceRequest r1 = new ServiceRequest(clusterName, serviceName, dStateStr); + Set<ServiceRequest> requests = new HashSet<ServiceRequest>(); + requests.add(r1); + + ServiceResourceProviderTest.createServices(controller, requests); + } + + private void createServiceComponent(String clusterName, + String serviceName, String componentName, State desiredState) + throws AmbariException { + String dStateStr = null; + if (desiredState != null) { + dStateStr = desiredState.toString(); + } + ServiceComponentRequest r = new ServiceComponentRequest(clusterName, + serviceName, componentName, dStateStr); + Set<ServiceComponentRequest> requests = + new HashSet<ServiceComponentRequest>(); + requests.add(r); + ComponentResourceProviderTest.createComponents(controller, requests); + } + + private void createServiceComponentHost(String clusterName, String serviceName, String componentName, String hostname, State desiredState) throws AmbariException { + String dStateStr = null; + if (desiredState != null) { + dStateStr = desiredState.toString(); + } + ServiceComponentHostRequest r = new ServiceComponentHostRequest(clusterName, + serviceName, componentName, hostname, dStateStr); + Set<ServiceComponentHostRequest> requests = + new HashSet<ServiceComponentHostRequest>(); + requests.add(r); + controller.createHostComponents(requests); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/java/org/apache/ambari/server/customactions/ActionDefinitionManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/customactions/ActionDefinitionManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/customactions/ActionDefinitionManagerTest.java index 7108be2..ec84922 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/customactions/ActionDefinitionManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/customactions/ActionDefinitionManagerTest.java @@ -27,27 +27,15 @@ import org.junit.Test; public class ActionDefinitionManagerTest { - private final String customActionDefinitionRoot = "./src/test/resources/custom_action_definitions/". - replaceAll("/", File.separator); + private final String customActionDefinitionRoot = "./src/test/resources/custom_action_definitions/"; @Test public void testReadCustomActionDefinitions() throws Exception { ActionDefinitionManager manager = new ActionDefinitionManager(); manager.readCustomActionDefinitions(new File(customActionDefinitionRoot)); - Assert.assertEquals(3, manager.getAllActionDefinition().size()); - ActionDefinition ad = manager.getActionDefinition("ambari_hdfs_rebalancer"); - Assert.assertNotNull(ad); - Assert.assertEquals("ambari_hdfs_rebalancer", ad.getActionName()); - Assert.assertEquals("HDFS Rebalance", ad.getDescription()); - Assert.assertEquals("threshold,[principal],[keytab]", ad.getInputs()); - Assert.assertEquals("NAMENODE", ad.getTargetComponent()); - Assert.assertEquals("HDFS", ad.getTargetService()); - Assert.assertEquals(600, (int)ad.getDefaultTimeout()); - Assert.assertEquals(TargetHostType.ANY, ad.getTargetType()); - Assert.assertEquals(ActionType.SYSTEM, ad.getActionType()); - - ad = manager.getActionDefinition("customAction1"); + Assert.assertEquals(2, manager.getAllActionDefinition().size()); + ActionDefinition ad = manager.getActionDefinition("customAction1"); Assert.assertNotNull(ad); Assert.assertEquals("customAction1", ad.getActionName()); Assert.assertEquals("A random test", ad.getDescription()); http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index 0dfad30..d268f9d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -451,3 +451,10 @@ class TestNamenode(RMFTestCase): recursive = True, mode = 0755, ) +# def test_rebalance_hdfs(self): ##Does not work because of exectuteScript Framework does not works with strcuturedoutput +# self.executeScript("2.0.6/services/HDFS/package/scripts/namenode.py", +# classname = "NameNode", +# command = "rebalancehdfs", +# config_file="rebalancehdfs_default.json" +# ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json new file mode 100644 index 0000000..ba11bb5 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json @@ -0,0 +1,388 @@ +{ + "roleCommand": "CUSTOM_COMMAND", + "clusterName": "pacan", + "hostname": "c6402.ambari.apache.org", + "hostLevelParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "ambari_db_rca_password": "mapred", + "java_home": "/usr/lib/jvm/java-1.6.0-openjdk.x86_64", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", + "stack_name": "HDP", + "custom_command": "REBALANCEHDFS", + "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar";, + "repo_info": "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.3.0\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.1\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"REPLACE_WITH_CENTOS6_URL\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.3.0\"},{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.17/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.17\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.17/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.17/repos/centos6\"}]";, + "stack_version": "2.1", + "db_name": "ambari7", + "ambari_db_rca_driver": "org.postgresql.Driver", + "ambari_db_rca_username": "mapred", + "db_driver_filename": "mysql-connector-java.jar", + "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar"; + }, + "commandType": "BACKGROUND_EXECUTION_COMMAND", + "roleParams": { + "component_category": "MASTER" + }, + "serviceName": "HDFS", + "role": "NAMENODE", + "commandParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "service_package_folder": "HDP/2.0.6/services/HDFS/package", + "script": "scripts/namenode.py", + "hooks_folder": "HDP/2.0.6/hooks", + "command_timeout": "600", + "namenode": "{\"threshold\":\"DEBUG\"}", + "script_type": "PYTHON" + }, + "taskId": 104, + "public_hostname": "c6402.ambari.apache.org", + "configurations": { + "core-site": { + "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization", + "fs.trash.interval": "360", + "hadoop.security.authentication": "simple", + "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec", + "hadoop.proxyuser.falcon.hosts": "*", + "mapreduce.jobtracker.webinterface.trusted": "false", + "hadoop.security.authorization": "false", + "fs.defaultFS": "hdfs://c6402.ambari.apache.org:8020", + "ipc.client.connect.max.retries": "50", + "ipc.client.idlethreshold": "8000", + "io.file.buffer.size": "131072", + "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT", + "ipc.client.connection.maxidletime": "30000", + "hadoop.proxyuser.falcon.groups": "users" + }, + "capacity-scheduler": { + "yarn.scheduler.capacity.root.default.maximum-capacity": "100", + "yarn.scheduler.capacity.root.default.state": "RUNNING", + "yarn.scheduler.capacity.root.capacity": "100", + "yarn.scheduler.capacity.root.default.capacity": "100", + "yarn.scheduler.capacity.root.queues": "default", + "yarn.scheduler.capacity.maximum-applications": "10000", + "yarn.scheduler.capacity.root.default.user-limit-factor": "1", + "yarn.scheduler.capacity.node-locality-delay": "40", + "yarn.scheduler.capacity.root.default.acl_submit_applications": "*", + "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2", + "yarn.scheduler.capacity.root.acl_administer_queue": "*", + "yarn.scheduler.capacity.root.default.acl_administer_jobs": "*" + }, + "hadoop-env": { + "security_enabled": "false", + "namenode_opt_maxnewsize": "200m", + "hdfs_log_dir_prefix": "/var/log/hadoop", + "ignore_groupsusers_create": "false", + "namenode_heapsize": "1024m", + "namenode_opt_newsize": "200m", + "kerberos_domain": "EXAMPLE.COM", + "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}\n\n# Hadoop Configuration Directory\n#TODO: if env var set that can cause problems\nexport HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by defaul t.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime Stamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nHADOOP_DATANODE_OPTS=\"-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize }} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER={{hdfs_user}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where ha doop code should be rsync'd from. Unset by default.\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add lib raries required by mysql connector\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\nif [ -d \"/usr/lib/tez\" ]; then\n export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf\nfi\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n#Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64", + "hdfs_user": "hdfs", + "user_group": "hadoop", + "dtnode_heapsize": "1024m", + "proxyuser_group": "users", + "smokeuser": "ambari-qa", + "hadoop_heapsize": "1024", + "hadoop_pid_dir_prefix": "/var/run/hadoop" + }, + "zoo.cfg": {}, + "tez-env": { + "content": "\n# Tez specific configuration\nexport TEZ_CONF_DIR={{config_dir}}\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}", + "tez_user": "tez" + }, + "tez-site": { + "tez.am.shuffle-vertex-manager.max-src-fraction": "0.4", + "tez.task.get-task.sleep.interval-ms.max": "200", + "tez.runtime.intermediate-input.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec", + "tez.yarn.ats.enabled": "true", + "tez.am.log.level": "INFO", + "tez.runtime.intermediate-output.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec", + "tez.runtime.intermediate-output.should-compress": "false", + "tez.lib.uris": "hdfs:///apps/tez/,hdfs:///apps/tez/lib/", + "tez.am.grouping.max-size": "1073741824", + "tez.runtime.intermediate-input.is-compressed": "false", + "tez.am.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC", + "tez.am.env": "LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/`$JAVA_HOME/bin/java -d32 -version &> /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`", + "tez.am.grouping.split-waves": "1.4", + "tez.am.grouping.min-size": "16777216", + "tez.am.container.reuse.enabled": "true", + "tez.session.am.dag.submit.timeout.secs": "300", + "tez.session.client.timeout.secs": "180", + "tez.staging-dir": "/tmp/${user.name}/staging", + "tez.am.am-rm.heartbeat.interval-ms.max": "250", + "tez.am.shuffle-vertex-manager.min-src-fraction": "0.2", + "tez.am.container.reuse.non-local-fallback.enabled": "true", + "tez.am.container.reuse.rack-fallback.enabled": "true", + "tez.am.container.reuse.locality.delay-allocation-millis": "250", + "tez.am.resource.memory.mb": "682", + "tez.am.container.session.delay-allocation-millis": "10000" + }, + "mapred-site": { + "mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020", + "mapreduce.cluster.administrators": " hadoop", + "mapreduce.reduce.input.buffer.percent": "0.0", + "mapreduce.output.fileoutputformat.compress": "false", + "mapreduce.framework.name": "yarn", + "mapreduce.map.speculative": "false", + "mapreduce.reduce.shuffle.merge.percent": "0.66", + "yarn.app.mapreduce.am.resource.mb": "682", + "mapreduce.map.java.opts": "-Xmx546m", + "mapreduce.application.classpath": "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*", + "mapreduce.job.reduce.slowstart.completedmaps": "0.05", + "mapreduce.output.fileoutputformat.compress.type": "BLOCK", + "mapreduce.reduce.speculative": "false", + "mapreduce.reduce.java.opts": "-Xmx546m", + "mapreduce.am.max-attempts": "2", + "yarn.app.mapreduce.am.admin-command-opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN", + "mapreduce.reduce.log.level": "INFO", + "mapreduce.map.sort.spill.percent": "0.7", + "mapreduce.task.timeout": "300000", + "mapreduce.map.memory.mb": "682", + "mapreduce.task.io.sort.factor": "100", + "mapreduce.jobhistory.intermediate-done-dir": "/mr-history/tmp", + "mapreduce.reduce.memory.mb": "682", + "mapreduce.jobhistory.keytab.file": "/etc/security/keytabs/jhs.service.keytab", + "yarn.app.mapreduce.am.log.level": "INFO", + "mapreduce.map.log.level": "INFO", + "mapreduce.shuffle.port": "13562", + "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/`$JAVA_HOME/bin/java -d32 -version &> /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`", + "mapreduce.map.output.compress": "false", + "yarn.app.mapreduce.am.staging-dir": "/user", + "mapreduce.reduce.shuffle.parallelcopies": "30", + "mapreduce.reduce.shuffle.input.buffer.percent": "0.7", + "mapreduce.jobhistory.webapp.address": "c6402.ambari.apache.org:19888", + "mapreduce.jobhistory.done-dir": "/mr-history/done", + "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN", + "mapreduce.task.io.sort.mb": "273", + "yarn.app.mapreduce.am.command-opts": "-Xmx546m", + "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN" + }, + "hdfs-site": { + "dfs.namenode.avoid.write.stale.datanode": "true", + "dfs.namenode.checkpoint.txns": "1000000", + "dfs.block.access.token.enable": "true", + "dfs.support.append": "true", + "dfs.datanode.address": "0.0.0.0:50010", + "dfs.cluster.administrators": " hdfs", + "dfs.datanode.balance.bandwidthPerSec": "6250000", + "dfs.namenode.safemode.threshold-pct": "1.0f", + "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}", + "dfs.permissions.enabled": "true", + "dfs.namenode.kerberos.principal": "nn/[email protected]", + "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", + "dfs.https.port": "50470", + "dfs.namenode.https-address": "c6402.ambari.apache.org:50470", + "dfs.secondary.namenode.kerberos.https.principal": "HTTP/[email protected]", + "dfs.blockreport.initialDelay": "120", + "dfs.journalnode.edits.dir": "/grid/0/hdfs/journal", + "dfs.blocksize": "134217728", + "dfs.client.read.shortcircuit": "true", + "dfs.datanode.max.transfer.threads": "1024", + "dfs.heartbeat.interval": "3", + "dfs.replication": "3", + "dfs.namenode.handler.count": "40", + "dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab", + "fs.permissions.umask-mode": "022", + "dfs.namenode.stale.datanode.interval": "30000", + "dfs.datanode.ipc.address": "0.0.0.0:8010", + "dfs.namenode.name.dir": "/hadoop/hdfs/namenode", + "dfs.datanode.data.dir": "/hadoop/hdfs/data", + "dfs.namenode.http-address": "c6402.ambari.apache.org:50070", + "dfs.webhdfs.enabled": "true", + "dfs.datanode.failed.volumes.tolerated": "0", + "dfs.namenode.accesstime.precision": "0", + "dfs.namenode.avoid.read.stale.datanode": "true", + "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090", + "dfs.datanode.kerberos.principal": "dn/[email protected]", + "dfs.datanode.http.address": "0.0.0.0:50075", + "dfs.datanode.du.reserved": "1073741824", + "dfs.client.read.shortcircuit.streams.cache.size": "4096", + "dfs.namenode.kerberos.https.principal": "HTTP/[email protected]", + "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab", + "dfs.web.authentication.kerberos.principal": "HTTP/[email protected]", + "dfs.secondary.namenode.kerberos.principal": "nn/[email protected]", + "dfs.datanode.keytab.file": "/etc/security/keytabs/dn.service.keytab", + "dfs.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab", + "dfs.permissions.superusergroup": "hdfs", + "dfs.journalnode.http-address": "0.0.0.0:8480", + "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket", + "dfs.namenode.write.stale.datanode.ratio": "1.0f", + "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude", + "dfs.datanode.data.dir.perm": "750", + "dfs.namenode.name.dir.restore": "true", + "dfs.replication.max": "50", + "dfs.namenode.checkpoint.period": "21600" + }, + "yarn-env": { + "yarn_pid_dir_prefix": "/var/run/hadoop-yarn", + "apptimelineserver_heapsize": "1024", + "nodemanager_heapsize": "1024", + "content": "\nexport HADOOP_YARN_HOME={{hadoop_yarn_home}}\nexport YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER\nexport YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\nexport JAVA_HOME={{java64_home}}\n\n# User for YARN daemons\nexport HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}\n\n# resolve links - $0 may be a softlink\nexport YARN_CONF_DIR=\"${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}\"\n\n# some Java parameters\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\nif [ \"$JAVA_HOME\" != \"\" ]; then\n #echo \"run java in $JAVA_HOME\"\n JAVA_HOME=$JAVA_HOME\nfi\n\nif [ \"$JAVA_HOME\" = \"\" ]; then\n echo \"Error: JAVA_HOME is not set.\"\n exit 1\nfi\n\nJAVA=$JAVA_HOME/bin/java\nJAVA_HEAP_MAX=-Xmx1000m\n\n# For setting YARN specific HEAP sizes please use this\n# Parameter and set appropriately\nYARN_HEAPSIZE={{yarn_heapsize}}\n\n# check envvars which might override default args\nif [ \"$YARN_HEAPSIZE\" != \"\" ]; then\n JAVA_HEAP_M AX=\"-Xmx\"\"$YARN_HEAPSIZE\"\"m\"\nfi\n\n# Resource Manager specific parameters\n\n# Specify the max Heapsize for the ResourceManager using a numerical value\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n# the value to 1000.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_RESOURCEMANAGER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}\n\n# Specify the JVM options to be used when starting the ResourceManager.\n# These options will be appended to the options specified as YARN_OPTS\n# and therefore may override any similar flags set in YARN_OPTS\n#export YARN_RESOURCEMANAGER_OPTS=\n\n# Node Manager specific parameters\n\n# Specify the max Heapsize for the NodeManager using a numerical value\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n# the value to 1000.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_NODEMANAGER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}\n\n# Specify the max Heapsize for the HistoryManager using a numerical value\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n# the value to 1024.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_HISTORYSERVER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}\n\n# Specify the JVM options to be used when starting the NodeManager.\n# These options will be appended to the options specified as YARN_OPTS\n# and therefore may override any similar flags set in YARN_OPTS\n#export YARN_NODEMANAGER_OPTS=\n\n# so that filenames w/ spaces are handled correctly in loops below\nIFS=\n\n\n# default log directory and file\nif [ \"$YARN_LOG_DIR\" = \"\" ]; then\n YARN_LOG_DIR=\"$HADOOP_YARN_HOME/logs\"\nfi\nif [ \"$YARN_LOGFILE\" = \"\" ]; then\n YARN_LOGFILE='yarn.log'\nfi\n\n# default policy file for service-level authorization\nif [ \"$YARN_POLICYFILE\" = \"\" ]; then\n YARN_POLICYFILE=\"hadoop-policy.xml\"\nfi\n\n# restore ordinary behaviour\nunset IFS\n\n\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR\"\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING\"\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.root.logger=$ {YARN_ROOT_LOGGER:-INFO,console}\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\nif [ \"x$JAVA_LIBRARY_PATH\" != \"x\" ]; then\n YARN_OPTS=\"$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH\"\nfi\nYARN_OPTS=\"$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE\"", + "yarn_heapsize": "1024", + "yarn_user": "yarn", + "resourcemanager_heapsize": "1024", + "yarn_log_dir_prefix": "/var/log/hadoop-yarn" + }, + "yarn-log4j": { + "content": "" + }, + "mapreduce2-log4j": {}, + "hdfs-log4j": { + "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\nhadoop.root.logger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n # Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache. log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logger}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFA S=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppe nder\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxF ileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounte r\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN" + }, + "yarn-site": { + "yarn.nodemanager.resource.memory-mb": "2048", + "yarn.nodemanager.disk-health-checker.min-healthy-disks": "0.25", + "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor", + "yarn.nodemanager.local-dirs": "/hadoop/yarn/local", + "yarn.resourcemanager.resource-tracker.address": "c6402.ambari.apache.org:8025", + "yarn.nodemanager.remote-app-log-dir-suffix": "logs", + "yarn.resourcemanager.hostname": "c6402.ambari.apache.org", + "yarn.nodemanager.health-checker.script.timeout-ms": "60000", + "yarn.resourcemanager.scheduler.class": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler", + "yarn.resourcemanager.nodes.exclude-path": "/etc/hadoop/conf/yarn.exclude", + "yarn.scheduler.minimum-allocation-mb": "682", + "yarn.resourcemanager.address": "c6402.ambari.apache.org:8050", + "yarn.resourcemanager.scheduler.address": "c6402.ambari.apache.org:8030", + "yarn.log-aggregation.retain-seconds": "2592000", + "yarn.scheduler.maximum-allocation-mb": "2048", + "yarn.timeline-service.ttl-ms": "2678400000", + "yarn.log-aggregation-enable": "true", + "yarn.nodemanager.address": "0.0.0.0:45454", + "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms": "300000", + "yarn.nodemanager.container-monitor.interval-ms": "3000", + "yarn.timeline-service.webapp.address": "c6402.ambari.apache.org:8188", + "yarn.timeline-service.enabled": "true", + "yarn.timeline-service.address": "c6402.ambari.apache.org:10200", + "yarn.nodemanager.log.retain-second": "604800", + "yarn.nodemanager.delete.debug-delay-sec": "0", + "yarn.timeline-service.store-class": "org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore", + "yarn.nodemanager.log-aggregation.compression-type": "gz", + "yarn.nodemanager.log-dirs": "/hadoop/yarn/log", + "yarn.timeline-service.ttl-enable": "true", + "yarn.nodemanager.health-checker.interval-ms": "135000", + "yarn.resourcemanager.am.max-attempts": "2", + "yarn.nodemanager.remote-app-log-dir": "/app-logs", + "yarn.nodemanager.admin-env": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX", + "yarn.nodemanager.aux-services": "mapreduce_shuffle", + "yarn.nodemanager.vmem-check-enabled": "false", + "yarn.nodemanager.vmem-pmem-ratio": "2.1", + "yarn.admin.acl": "", + "yarn.resourcemanager.webapp.address": "c6402.ambari.apache.org:8088", + "yarn.timeline-service.leveldb-timeline-store.path": "/hadoop/yarn/timeline", + "yarn.nodemanager.linux-container-executor.group": "hadoop", + "yarn.acl.enable": "false", + "yarn.log.server.url": "http://c6402.ambari.apache.org:19888/jobhistory/logs";, + "yarn.application.classpath": "/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*", + "yarn.resourcemanager.admin.address": "c6402.ambari.apache.org:8141", + "yarn.timeline-service.generic-application-history.store-class": "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore", + "yarn.timeline-service.webapp.https.address": "c6402.ambari.apache.org:8190", + "yarn.nodemanager.aux-services.mapreduce_shuffle.class": "org.apache.hadoop.mapred.ShuffleHandler" + }, + "zookeeper-env": { + "clientPort": "2181", + "zk_user": "zookeeper", + "zk_log_dir": "/var/log/zookeeper", + "syncLimit": "5", + "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}", + "initLimit": "10", + "zk_pid_dir": "/var/run/zookeeper", + "zk_data_dir": "/hadoop/zookeeper", + "tickTime": "2000" + }, + "zookeeper-log4j": { + "content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4 j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n" + }, + "mapred-env": { + "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.", + "mapred_pid_dir_prefix": "/var/run/hadoop-mapreduce", + "mapred_user": "mapred", + "jobhistory_heapsize": "900", + "mapred_log_dir_prefix": "/var/log/hadoop-mapreduce" + } + }, + "configurationTags": { + "capacity-scheduler": { + "tag": "version1" + }, + "tez-site": { + "tag": "version1" + }, + "zoo.cfg": { + "tag": "version1" + }, + "tez-env": { + "tag": "version1" + }, + "mapreduce2-log4j": { + "tag": "version1" + }, + "mapred-site": { + "tag": "version1" + }, + "hdfs-site": { + "tag": "version1" + }, + "yarn-site": { + "tag": "version1" + }, + "yarn-env": { + "tag": "version1" + }, + "yarn-log4j": { + "tag": "version1" + }, + "core-site": { + "tag": "version1" + }, + "hdfs-log4j": { + "tag": "version1" + }, + "hadoop-env": { + "tag": "version1" + }, + "zookeeper-env": { + "tag": "version1" + }, + "zookeeper-log4j": { + "tag": "version1" + }, + "mapred-env": { + "tag": "version1" + } + }, + "commandId": "11-1", + "clusterHostInfo": { + "snamenode_host": [ + "c6402.ambari.apache.org" + ], + "nm_hosts": [ + "c6402.ambari.apache.org" + ], + "app_timeline_server_hosts": [ + "c6402.ambari.apache.org" + ], + "all_ping_ports": [ + "8670" + ], + "rm_host": [ + "c6402.ambari.apache.org" + ], + "all_hosts": [ + "c6402.ambari.apache.org" + ], + "slave_hosts": [ + "c6402.ambari.apache.org" + ], + "namenode_host": [ + "c6402.ambari.apache.org" + ], + "ambari_server_host": [ + "c6401.ambari.apache.org" + ], + "zookeeper_hosts": [ + "c6402.ambari.apache.org" + ], + "hs_host": [ + "c6402.ambari.apache.org" + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/resources/custom_action_definitions/cust_action_definitions1.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/custom_action_definitions/cust_action_definitions1.xml b/ambari-server/src/test/resources/custom_action_definitions/cust_action_definitions1.xml index a4db87d..9cee575 100644 --- a/ambari-server/src/test/resources/custom_action_definitions/cust_action_definitions1.xml +++ b/ambari-server/src/test/resources/custom_action_definitions/cust_action_definitions1.xml @@ -42,14 +42,4 @@ <targetComponent>TASKTRACKER</targetComponent> <description>A random test</description> </actionDefinition> - <actionDefinition> - <actionName>ambari_hdfs_rebalancer</actionName> - <actionType>SYSTEM</actionType> - <inputs>threshold,[principal],[keytab]</inputs> - <targetService>HDFS</targetService> - <targetComponent>NAMENODE</targetComponent> - <description>HDFS Rebalance</description> - <targetType>ANY</targetType> - <defaultTimeout>600</defaultTimeout> - </actionDefinition> </actionDefinitions> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/resources/custom_action_definitions/system_action_definitions.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/custom_action_definitions/system_action_definitions.xml b/ambari-server/src/test/resources/custom_action_definitions/system_action_definitions.xml deleted file mode 100644 index ee07900..0000000 --- a/ambari-server/src/test/resources/custom_action_definitions/system_action_definitions.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="action_definition.xsl"?> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<actionDefinitions> - <actionDefinition> - <actionName>ambari_hdfs_rebalancer</actionName> - <actionType>SYSTEM</actionType> - <inputs>threshold,[principal],[keytab]</inputs> - <targetService>HDFS</targetService> - <targetComponent>NAMENODE</targetComponent> - <defaultTimeout>600</defaultTimeout> - <description>HDFS Rebalance</description> - <targetType>ANY</targetType> - </actionDefinition> -</actionDefinitions> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml index be999e6..782aa65 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml @@ -42,6 +42,15 @@ <timeout>600</timeout> </commandScript> </customCommand> + <customCommand> + <name>REBALANCEHDFS</name> + <background>true</background> + <commandScript> + <script>scripts/namenode.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </customCommand> + </customCommands> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-web/app/controllers/global/background_operations_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/global/background_operations_controller.js b/ambari-web/app/controllers/global/background_operations_controller.js index b6dc603..17540c5 100644 --- a/ambari-web/app/controllers/global/background_operations_controller.js +++ b/ambari-web/app/controllers/global/background_operations_controller.js @@ -172,6 +172,12 @@ App.BackgroundOperationsController = Em.Controller.extend({ task.Tasks.status = data.Tasks.status; task.Tasks.stdout = data.Tasks.stdout; task.Tasks.stderr = data.Tasks.stderr; + + // Put some command information to task object + task.Tasks.command = data.Tasks.command; + task.Tasks.custom_command_name = data.Tasks.custom_command_name; + task.Tasks.structured_out = data.Tasks.structured_out; + task.Tasks.output_log = data.Tasks.output_log; task.Tasks.error_log = data.Tasks.error_log; this.set('serviceTimestamp', App.dateTime()); http://git-wip-us.apache.org/repos/asf/ambari/blob/cb662f49/ambari-web/app/controllers/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js index 0aaf78c..32c8f5d 100644 --- a/ambari-web/app/controllers/main/service/item.js +++ b/ambari-web/app/controllers/main/service/item.js @@ -230,15 +230,11 @@ App.MainServiceItemController = Em.Controller.extend({ * @param event */ refreshYarnQueues : function (event) { - var self = this; + var controller = this; return App.showConfirmationPopup(function() { - self.refreshYarnQueuesPrimary(); - }); - }, - refreshYarnQueuesPrimary : function(){ App.ajax.send({ name : 'service.item.refreshQueueYarnRequest', - sender : this, + sender: controller, data : { command : "REFRESHQUEUES", context : Em.I18n.t('services.service.actions.run.yarnRefreshQueues.context') , @@ -250,6 +246,7 @@ App.MainServiceItemController = Em.Controller.extend({ success : 'refreshYarnQueuesSuccessCallback', error : 'refreshYarnQueuesErrorCallback' }); + }); }, refreshYarnQueuesSuccessCallback : function(data, ajaxOptions, params) { if (data.Requests.id) { @@ -269,6 +266,72 @@ App.MainServiceItemController = Em.Controller.extend({ App.showAlertPopup(Em.I18n.t('services.service.actions.run.yarnRefreshQueues.error'), error); console.warn('Error during refreshYarnQueues:'+error); }, + /** + * On click handler for rebalance Hdfs command from items menu + */ + rebalanceHdfsNodes: function () { + var controller = this; + App.ModalPopup.show({ + classNames: ['fourty-percent-width-modal'], + header: Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.context'), + primary: Em.I18n.t('common.start'), + secondary: Em.I18n.t('common.cancel'), + inputValue: 0, + errorMessage: Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.promptError'), + isInvalid: function () { + var intValue = Number(this.get('inputValue')); + if (this.get('inputValue')!=='DEBUG' && (isNaN(intValue) || intValue < 0 || intValue > 100)) { + return true; + } + return false; + }.property('inputValue'), + onPrimary: function () { + if (this.get('isInvalid')) { + return; + } + App.ajax.send({ + name : 'service.item.rebalanceHdfsNodes', + sender: controller, + data : { + hosts : App.Service.find('HDFS').get('hostComponents').findProperty('componentName', 'NAMENODE').get('hostName'), + threshold: this.get('inputValue') + }, + success : 'rebalanceHdfsNodesSuccessCallback', + error : 'rebalanceHdfsNodesErrorCallback' + }); + this.hide(); + }, + bodyClass: Ember.View.extend({ + templateName: require('templates/common/prompt_popup'), + text: Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.prompt'), + didInsertElement: function () { + App.tooltip(this.$(".prompt-input"), { + placement: "bottom", + title: Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.promptTooltip') + }); + } + }) + }); + }, + rebalanceHdfsNodesSuccessCallback: function (data) { + if (data.Requests.id) { + App.router.get('backgroundOperationsController').showPopup(); + } else { + console.warn('Error during runRebalanceHdfsNodes'); + } + }, + rebalanceHdfsNodesErrorCallback : function(data) { + var error = Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.error'); + if(data && data.responseText){ + try { + var json = $.parseJSON(data.responseText); + error += json.message; + } catch (err) { + } + } + App.showAlertPopup(Em.I18n.t('services.service.actions.run.rebalanceHdfsNodes.error'), error); + console.warn('Error during runRebalanceHdfsNodes:'+error); + }, /** * On click callback for <code>run compaction</code> button
