Ottomata has submitted this change and it was merged.
Change subject: Remove {templates,files}/hadoop files
......................................................................
Remove {templates,files}/hadoop files
These are now in analytics_cluster role
Change-Id: I58395a476fc88e2541d74a619eaae9151b671496
---
D files/hadoop/check_hadoop_yarn_node_state
D files/hadoop/hdfs-balancer
M modules/role/manifests/analytics_cluster/hadoop/logstash.pp
R modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
D templates/hadoop/fair-scheduler.xml.erb
D templates/hadoop/net-topology.py.erb
6 files changed, 1 insertion(+), 192 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/files/hadoop/check_hadoop_yarn_node_state
b/files/hadoop/check_hadoop_yarn_node_state
deleted file mode 100755
index b342e3a..0000000
--- a/files/hadoop/check_hadoop_yarn_node_state
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# Usage:
-# hadoop_yarn_node_status [node_id]
-#
-# If node_id is not given, then the $(hostname -f):8041 will be used.
-#
-
-node_id="${1}"
-
-if [ -z "${node_id}" ]; then
- node_id=$(hostname -f):8041
-fi
-
-node_status_output=$(/usr/bin/yarn node -status ${node_id} 2>&1)
-# check that this node exists in the node status output
-echo "${node_status_output}" | grep -qE "(Could not find the node report for
node id|Invalid NodeId)"
-if [ $? -eq 0 ]; then
- node_state=$(echo "${node_status_output}" | head -n 1)
-else
- node_state=$(echo "${node_status_output}" | grep Node-State | awk -F ' : '
'{print $2}')
-fi
-
-
-if [ "x${node_state}" = "xRUNNING" ]; then
- check_status="OK"
-# Unhealthy is just a warning. Likely disk space is getting low.
-elif [ "x${node_state}" = "xUNHEALTHY" ]; then
- check_status="WARNING"
-# Anything else is critical.
-else
- check_status="CRITICAL"
-fi
-
-echo "${check_status}: YARN NodeManager ${node_id} Node-State: ${node_state}"
diff --git a/files/hadoop/hdfs-balancer b/files/hadoop/hdfs-balancer
deleted file mode 100755
index e210774..0000000
--- a/files/hadoop/hdfs-balancer
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# NOTE: This file is managed by Puppet.
-
-#
-# Runs hdfs balancer as long as another is not already running.
-#
-
-(
- # make sure balancer is not already running.
- lockfile-check /tmp/hdfs-balancer && \
- echo "$(date \'+\%y/\%m/\%d \%H:\%M:\%S\') WARN Not starting hdfs
balancer, it is already running (or the lockfile exists)."
-) || \
-(
- # Create a lockfile
- lockfile-create /tmp/hdfs-balancer && \
- # Set the balancer bandwidth
- hdfs dfsadmin -setBalancerBandwidth $((40*1048576)) && \
- # Run the hdfs balancer.
- /usr/bin/hdfs balancer
-
- # Remove the lockfile
- lockfile-remove /tmp/hdfs-balancer
-)
diff --git a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
b/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
index b3cf939..8f386f1 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
@@ -5,7 +5,7 @@
Class['cdh::hadoop'] -> Class['role::analytics_cluster::hadoop::logstash']
file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
- content => template('hadoop/hadoop-yarn-logging-helper.erb'),
+ content =>
template('role/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb'),
mode => '0744',
}
diff --git a/templates/hadoop/hadoop-yarn-logging-helper.erb
b/modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
similarity index 100%
rename from templates/hadoop/hadoop-yarn-logging-helper.erb
rename to
modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
diff --git a/templates/hadoop/fair-scheduler.xml.erb
b/templates/hadoop/fair-scheduler.xml.erb
deleted file mode 100644
index 1db4ef1..0000000
--- a/templates/hadoop/fair-scheduler.xml.erb
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0"?>
-<allocations>
-
- <queue name="default">
- <weight>1.0</weight>
- <maxRunningApps>50</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
- </queue>
-
- <queue name="oozie">
- <!--
- The oozie queue is for oozie:launcher jobs. These jobs
- run only one task, and don't need many resources.
- Trying out DRF queue for this.
- -->
- <weight>1.0</weight>
- <maxRunningApps>50</maxRunningApps>
- <schedulingMode>drf</schedulingMode>
- </queue>
-
- <queue name="priority">
- <!--
- The priority queue is for non-adhoc jobs that should get some priority.
- This queue has a higher weight than default, but will never preempt.
- -->
- <weight>5.0</weight>
- <maxRunningApps>50</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
- </queue>
-
- <queue name="production">
- <schedulingMode>fair</schedulingMode>
- <aclSubmitApps>hdfs</aclSubmitApps>
-
- <!--
- The production queue has a higher priority than default,
- and it will start killing (preempting) jobs in other queues
- if it can't get its minimum share within 10 minutes.
- -->
- <weight>5.0</weight>
- <minSharePreemptionTimeout>600</minSharePreemptionTimeout>
- <maxRunningApps>50</maxRunningApps>
- </queue>
-
- <!-- essential jobs will aggressively preempt jobs in other queues -->
- <queue name="essential">
- <!--
- Use FIFO for essential queue. We want jobs submitted
- here to run in sequential order.
- -->
- <schedulingMode>fifo</schedulingMode>
- <aclSubmitApps>hdfs</aclSubmitApps>
-
- <!--
- The essential queue has a much higher priority than production,
- and it will start killing (preempting) jobs in other queues,
- first after 60 seconds if it can't get its minimum share,
- and then more after 10 minutes if it can't get its fair share.
- -->
- <weight>10.0</weight>
- <minSharePreemptionTimeout>60</minSharePreemptionTimeout>
- <fairSharePreemptionThreshold>600</fairSharePreemptionThreshold>
- <maxRunningApps>50</maxRunningApps>
- </queue>
-
-</allocations>
diff --git a/templates/hadoop/net-topology.py.erb
b/templates/hadoop/net-topology.py.erb
deleted file mode 100755
index 67c96f6..0000000
--- a/templates/hadoop/net-topology.py.erb
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-
-# NOTE: This file is managed by Puppet.
-
-# Returns a rack/row name for the given node name.
-
-# Usage:
-# net-topology.py <ipaddr|fqdn>
-
-import socket
-from sys import argv
-
-# Maps fqdn to a unique /datacenter/rack/row id.
-nodes = {
- 'analytics1001.eqiad.wmnet': '/eqiad/C/4',
- 'analytics1002.eqiad.wmnet': '/eqiad/D/2',
-
- 'analytics1028.eqiad.wmnet': '/eqiad/C/2',
- 'analytics1029.eqiad.wmnet': '/eqiad/C/2',
- 'analytics1030.eqiad.wmnet': '/eqiad/C/2',
- 'analytics1031.eqiad.wmnet': '/eqiad/C/2',
-
- 'analytics1032.eqiad.wmnet': '/eqiad/C/3',
- 'analytics1033.eqiad.wmnet': '/eqiad/C/3',
- 'analytics1034.eqiad.wmnet': '/eqiad/C/3',
-
- 'analytics1035.eqiad.wmnet': '/eqiad/D/2',
- 'analytics1036.eqiad.wmnet': '/eqiad/D/2',
- 'analytics1037.eqiad.wmnet': '/eqiad/D/2',
-
- 'analytics1038.eqiad.wmnet': '/eqiad/D/4',
- 'analytics1039.eqiad.wmnet': '/eqiad/D/4',
- 'analytics1040.eqiad.wmnet': '/eqiad/D/4',
- 'analytics1041.eqiad.wmnet': '/eqiad/D/4',
-
- 'analytics1042.eqiad.wmnet': '/eqiad/D/2',
- 'analytics1043.eqiad.wmnet': '/eqiad/D/2',
- 'analytics1044.eqiad.wmnet': '/eqiad/D/2',
- 'analytics1045.eqiad.wmnet': '/eqiad/D/2',
-
- 'analytics1046.eqiad.wmnet': '/eqiad/B/3',
- 'analytics1047.eqiad.wmnet': '/eqiad/B/3',
- 'analytics1048.eqiad.wmnet': '/eqiad/B/3',
- 'analytics1049.eqiad.wmnet': '/eqiad/B/3',
- 'analytics1050.eqiad.wmnet': '/eqiad/B/3',
- 'analytics1051.eqiad.wmnet': '/eqiad/B/3',
-
- 'analytics1052.eqiad.wmnet': '/eqiad/A/3',
- 'analytics1053.eqiad.wmnet': '/eqiad/A/3',
- 'analytics1054.eqiad.wmnet': '/eqiad/A/3',
- 'analytics1055.eqiad.wmnet': '/eqiad/A/3',
- 'analytics1056.eqiad.wmnet': '/eqiad/A/3',
- 'analytics1057.eqiad.wmnet': '/eqiad/A/3'
-}
-
-if len(argv) < 2:
- node = 'default'
-else:
- node = argv[1]
-
-# If the node doesn't have a key in nodes, perhaps it is a
-# short hostname or IP address. Look it up!
-if node not in nodes:
- node = socket.getfqdn(node)
-
-print(nodes.get(node, '/eqiad/default/rack'))
--
To view, visit https://gerrit.wikimedia.org/r/270963
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I58395a476fc88e2541d74a619eaae9151b671496
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
