mikewalch closed pull request #234: Several update to Muchos (#233)
URL: https://github.com/apache/fluo-muchos/pull/234
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ansible/common.yml b/ansible/common.yml
index b9214f4..8a17ee8 100644
--- a/ansible/common.yml
+++ b/ansible/common.yml
@@ -13,6 +13,3 @@
- import_tasks: roles/common/tasks/ssh.yml
- import_tasks: roles/common/tasks/os.yml
- import_tasks: roles/common/tasks/drives.yml
- handlers:
- - name: "update network settings"
- command: /sbin/ifup-local {{ network_interface }}
diff --git a/ansible/roles/accumulo/templates/accumulo.properties
b/ansible/roles/accumulo/templates/accumulo.properties
index 44edeca..2379c18 100644
--- a/ansible/roles/accumulo/templates/accumulo.properties
+++ b/ansible/roles/accumulo/templates/accumulo.properties
@@ -20,7 +20,7 @@
## Time to wait on I/O for simple, short RPC calls
general.rpc.timeout=240s
-## A secret unique to a given instnace that servers must know to communicate
+## A secret unique to a given instance that servers must know to communicate
instance.secret=muchos
## Sets location in HDFS where Accumulo will store data
diff --git a/ansible/roles/common/files/ifup-local
b/ansible/roles/common/files/ifup-local
deleted file mode 100755
index 29102ab..0000000
--- a/ansible/roles/common/files/ifup-local
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-/sbin/ethtool -K $1 sg off
-/sbin/ethtool -K $1 tso off
diff --git a/ansible/roles/common/tasks/main.yml
b/ansible/roles/common/tasks/main.yml
index 3b2d094..5c5216e 100644
--- a/ansible/roles/common/tasks/main.yml
+++ b/ansible/roles/common/tasks/main.yml
@@ -1,16 +1,17 @@
- name: "enable epel yum repo"
yum: name=epel-release state=present
- name: "install packages"
- yum: name={{ item }} state=present
- with_items:
- - vim
- - git
- - wget
- - gcc-c++
- - collectd
- - screen
- - patch
- - java-1.8.0-openjdk-devel
+ yum:
+ name:
+ - vim
+ - git
+ - wget
+ - gcc-c++
+ - collectd
+ - screen
+ - patch
+ - java-1.8.0-openjdk-devel
+ state: present
- name: "configure node shutdown"
shell: shutdown +{{ shutdown_delay_minutes }} &> {{ cluster_basedir
}}/.shutdown creates={{ cluster_basedir}}/.shutdown
when: shutdown_delay_minutes > 0
diff --git a/ansible/roles/common/tasks/os.yml
b/ansible/roles/common/tasks/os.yml
index 2d27ba8..ff98ea4 100644
--- a/ansible/roles/common/tasks/os.yml
+++ b/ansible/roles/common/tasks/os.yml
@@ -4,9 +4,6 @@
file: path=/etc/security/limits.d/20-nproc.conf state=absent
- name: "copy new limits.conf"
copy: src=roles/common/files/limits.conf dest=/etc/security/limits.conf
-- name: "configure network settings for spark"
- copy: src=roles/common/files/ifup-local dest=/sbin/ifup-local mode=0755
- notify: "update network settings"
- name: "configure user shell"
template: src=roles/common/templates/{{ item }} dest=/home/{{ cluster_user
}}/.{{ item }} owner={{ cluster_user }} group={{ cluster_user }} mode=0644
with_items:
diff --git a/ansible/roles/grafana/tasks/main.yml
b/ansible/roles/grafana/tasks/main.yml
index 4af4114..e6da759 100644
--- a/ansible/roles/grafana/tasks/main.yml
+++ b/ansible/roles/grafana/tasks/main.yml
@@ -19,7 +19,7 @@
- fluo-recent-dashboard.json
- fluo-historical-dashboard.json
- name: "configure accumulo dashboards"
- get_url:
url=https://raw.githubusercontent.com/apache/fluo-uno/master/conf/grafana/accumulo-dashboard.json
dest=/etc/grafana/dashboards/accumulo-dashboard.json
+ get_url:
url=https://raw.githubusercontent.com/apache/fluo-uno/master/plugins/influx-metrics/accumulo-dashboard.json
dest=/etc/grafana/dashboards/accumulo-dashboard.json
- name: "ensure owner and group are correct for /etc/grafana"
file: path=/etc/grafana/ recurse=yes owner=grafana group=grafana
- name: "ensure grafana is running (and enable it at boot)"
diff --git a/ansible/roles/hadoop/tasks/main.yml
b/ansible/roles/hadoop/tasks/main.yml
index fdb63e5..b37dee8 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -7,7 +7,16 @@
- hdfs-site.xml
- yarn-site.xml
- mapred-site.xml
+- name: "configure hadoop 2"
+ template: src={{ item }} dest={{ hadoop_home }}/etc/hadoop/{{ item }}
+ with_items:
- slaves
+ when: hadoop_major_version == '2'
+- name: "configure hadoop 3"
+ template: src={{ item }} dest={{ hadoop_home }}/etc/hadoop/{{ item }}
+ with_items:
+ - workers
+ when: hadoop_major_version == '3'
- name: "copy spark yarn shuffle jar to hadoop lib"
command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar
{{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home
}}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
when: "'spark' in groups"
diff --git a/ansible/roles/hadoop/tasks/start-yarn.yml
b/ansible/roles/hadoop/tasks/start-yarn.yml
index 7477e87..cd4a4ea 100644
--- a/ansible/roles/hadoop/tasks/start-yarn.yml
+++ b/ansible/roles/hadoop/tasks/start-yarn.yml
@@ -1,4 +1,11 @@
-- name: "start yarn"
+- name: "start hadoop yarn 2.x"
command: "{{ hadoop_home }}/sbin/start-yarn.sh"
register: start_yarn_result
changed_when: start_yarn_result.stdout | search("starting
(:?resource|node)manager")
+ when: hadoop_major_version == '2'
+- name: "start hadoop yarn 3.x"
+ command: "nohup {{ hadoop_home }}/sbin/start-yarn.sh"
+ register: start_yarn_result
+ changed_when: start_yarn_result.rc == 0
+ failed_when: start_yarn_result.rc >= 2
+ when: hadoop_major_version == '3'
diff --git a/ansible/roles/hadoop/templates/workers
b/ansible/roles/hadoop/templates/workers
new file mode 100644
index 0000000..cd3348c
--- /dev/null
+++ b/ansible/roles/hadoop/templates/workers
@@ -0,0 +1,3 @@
+{% for host in groups['workers'] %}
+{{ host }}
+{% endfor %}
diff --git a/conf/checksums b/conf/checksums
index af0c4aa..cdc605f 100644
--- a/conf/checksums
+++ b/conf/checksums
@@ -1,3 +1,4 @@
+accumulo:2.0.0-alpha-1:baa5e0929248ff0d96355bc7fb42a5b75d183a83364519296e07b0adbb089180
accumulo:1.9.2:c23c147e6abde5e6b851cf27f91b813705dc41d07c2bfea798a86abb144255d5
accumulo:1.9.0:f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe
accumulo:1.8.1:eba3bfe823935ca7901ea7c2bd59c84a68b9381361699c7e260bbd9191f237f4
@@ -13,6 +14,7 @@
hadoop:2.8.3:e8bf9a53337b1dca3b152b0a5b5e277dc734e76520543e525c301a050bb27eae
hadoop:2.7.6:f2327ea93f4bc5a5d7150dee8e0ede196d3a77ff8526a7dd05a48a09aae25669
hadoop:2.7.5:0bfc4d9b04be919be2fdf36f67fa3b4526cdbd406c512a7a1f5f1b715661f831
hadoop:2.6.5:001ad18d4b6d0fe542b15ddadba2d092bc97df1c4d2d797381c8d12887691898
+spark:2.3.2:3387107155d62f04ccf6bcaf2e00a69a0de5ae5df875348d93147743c206f0a8
spark:2.2.2:023b2fea378b3dd0fee2d5d1de6bfaf2d8349aefe7be97a9cbcf03bbacc428d7
zookeeper:3.4.13:7ced798e41d2027784b8fd55c908605ad5bd94a742d5dab2506be8f94770594d
zookeeper:3.4.12:c686f9319050565b58e642149cb9e4c9cc8c7207aacc2cb70c5c0672849594b9
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index a98057b..c797155 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -25,8 +25,6 @@ cluster_basedir = /home/centos
# public IP if launching in EC2. If not launching in EC2, node must have
public IP that can be reached
# from your machine. Hostname can be chosen from "nodes" section below.
proxy_hostname = leader1
-# Cluster network interface (leave default if launching in AWS)
-network_interface=eth0
# If set, a SOCKS proxy will be created on the specified port when connecting
to proxy using 'muchos ssh <cluster>'
#proxy_socks_port = 38585
# Accumulo Instance name
@@ -34,12 +32,12 @@ accumulo_instance = muchos
# Accumluo Password
accumulo_password = secret
# Software versions (set sha-256 in conf/checksums)
-hadoop_version = 2.8.4
-zookeeper_version = 3.4.12
-spark_version = 2.2.2
+hadoop_version = 3.1.1
+zookeeper_version = 3.4.13
+spark_version = 2.3.2
fluo_version = 1.2.0
fluo_yarn_version = 1.0.0
-accumulo_version = 1.9.2
+accumulo_version = 2.0.0-alpha-1
[ec2]
# AWS machine image to use. The default below is for a CentOS 7 image (in
us-east-1).
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 2aa8996..5f8a399 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -346,7 +346,6 @@ def print_property(self, key):
'metrics_drive_ids': None,
'mount_root': None,
'node_type_map': None,
- 'network_interface': None,
'spark_sha256': None,
'shutdown_delay_minutes': None,
'twill_reserve_mem_mb': None,
diff --git a/lib/muchos/main.py b/lib/muchos/main.py
index 0d8d0f4..a7824f5 100644
--- a/lib/muchos/main.py
+++ b/lib/muchos/main.py
@@ -200,7 +200,7 @@ def sync(self):
if name in play_vars:
play_vars[name] = value
- play_vars['accumulos_sha256'] = config.checksum('accumulo')
+ play_vars['accumulo_sha256'] = config.checksum('accumulo')
play_vars['fluo_sha256'] = config.checksum('fluo')
play_vars['fluo_yarn_sha256'] = config.checksum('fluo_yarn')
play_vars['hadoop_sha256'] = config.checksum('hadoop')
diff --git a/lib/tests/test_config.py b/lib/tests/test_config.py
index 4502cd6..f708162 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/test_config.py
@@ -19,7 +19,7 @@ def test_defaults():
c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'../conf/checksums', 'mycluster')
assert c.checksum_ver('accumulo', '1.9.0') ==
'f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe'
- assert c.checksum('accumulo') ==
'c23c147e6abde5e6b851cf27f91b813705dc41d07c2bfea798a86abb144255d5'
+ assert c.checksum('accumulo') ==
'baa5e0929248ff0d96355bc7fb42a5b75d183a83364519296e07b0adbb089180'
assert c.get('ec2', 'default_instance_type') == 'm5d.large'
assert c.get('ec2', 'worker_instance_type') == 'm5d.large'
assert c.get('ec2', 'aws_ami') == 'ami-9887c6e7'
@@ -46,9 +46,9 @@ def test_defaults():
assert c.get_public_ip('leader1') == '23.0.0.0'
assert c.get_private_ip('leader1') == '10.0.0.0'
assert c.cluster_name == 'mycluster'
- assert c.version("accumulo").startswith('1.')
+ assert c.version("accumulo").startswith('2.')
assert c.version("fluo").startswith('1.')
- assert c.version("hadoop").startswith('2.')
+ assert c.version("hadoop").startswith('3.')
assert c.version("zookeeper").startswith('3.')
assert c.get_service_private_ips("worker") == ['10.0.0.2', '10.0.0.3',
'10.0.0.4', '10.0.0.5']
assert c.get('general', 'proxy_hostname') == "leader1"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
