Repository: incubator-myriad
Updated Branches:
refs/heads/master f5e7406f8 -> 47c7be0de
MYRIAD-263 Add HDFS as system service in DC/OS vagrant testbed
The local development environment based on vagrant-libvirt (DC/OS)
needs deploy HDFS as foundation for Myriad. We have available HDFS
as Mesos framework at Mesosphere Universe, however running this
framework is not straightforward. So I propose deploy HDFS as systemd services,
this is easy and allow us focus the in our development (Myriad).
JIRA:
[MYRIAD-263] https://issues.apache.org/jira/browse/MYRIAD-263
Pull Request:
Closes #110
Author: Javi Roman <jroman.espi...@gmail.com>
Date: Sat Apr 21 13:11:48 2018 +0200
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/47c7be0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/47c7be0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/47c7be0d
Branch: refs/heads/master
Commit: 47c7be0de1c91470c766dda06c432c282b22bb01
Parents: f5e7406
Author: Javi Roman <jroman.espi...@gmail.com>
Authored: Sat Apr 21 13:11:48 2018 +0200
Committer: Javi Roman <javiro...@apache.org>
Committed: Sat May 5 09:40:51 2018 +0200
----------------------------------------------------------------------
vagrant/libvirt/dcos/Vagrantfile | 6 +-
.../provisioning/group_vars/all/defaults.yml | 26 ++++++++
.../provisioning/roles/common/tasks/main.yml | 3 +
.../roles/common/templates/java.sh.j2 | 23 +++++++
.../roles/dcos-setup/tasks/main.yml | 1 -
.../provisioning/roles/devel/tasks/main.yml | 42 +++++++++++++
.../roles/hadoop/files/core-site.xml | 23 +++++++
.../hadoop/files/hadoop-hdfs-datanode.service | 16 +++++
.../hadoop/files/hadoop-hdfs-namenode.service | 15 +++++
.../roles/hadoop/files/hdfs-site.xml | 34 ++++++++++
.../roles/hadoop/tasks/datanode.yml | 42 +++++++++++++
.../roles/hadoop/tasks/download.yml | 51 +++++++++++++++
.../provisioning/roles/hadoop/tasks/main.yml | 56 +++++++++++++++++
.../roles/hadoop/tasks/namenode.yml | 65 ++++++++++++++++++++
.../roles/jbod-disks/tasks/main.yml | 64 +++++++++++++++++++
.../roles/nfs-client/tasks/main.yml | 30 +++++++++
.../dcos/provisioning/roles/nfs/tasks/main.yml | 41 ++++++++++++
.../provisioning/roles/nfs/templates/exports.j2 | 34 ++++++++++
vagrant/libvirt/dcos/provisioning/site.yml | 19 +++++-
19 files changed, 583 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/Vagrantfile
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/Vagrantfile b/vagrant/libvirt/dcos/Vagrantfile
index 886c227..0f0022f 100644
--- a/vagrant/libvirt/dcos/Vagrantfile
+++ b/vagrant/libvirt/dcos/Vagrantfile
@@ -19,10 +19,8 @@
# specific language governing permissions and limitations
# under the License.
#
-
-unless defined? VAGRANTFILE_API_VERSION
- VAGRANTFILE_API_VERSION = "2"
-end
+#
+VAGRANTFILE_API_VERSION = "2"
# Tested with Vagrant version:
Vagrant.require_version ">= 1.7.2"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
b/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
index 07e69e5..b8ce22e 100644
--- a/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
+++ b/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
@@ -33,6 +33,7 @@ system_packages:
- xz
- unzip
- curl
+ - nfs-utils
- ipset
- sysstat
@@ -42,3 +43,28 @@ system_epel_packages:
with_dcos_version: "{{ lookup('env','DCOS_VERSION') }}"
with_hadoop_version: "{{ lookup('env','HADOOP_VERSION') }}"
+
+# NFS enablement for sharing folders between nodes
+nfs_packages:
+ - rpcbind
+ - nfs-server
+ - nfs-lock
+ - nfs-idmap
+nfs_shared_folder: /opt/shared
+
+# Source code location for Mesos and Myriad
+myriad_cloned_folder: myriad
+myriad_repository: https://github.com/myriad-framework/myriad
+myriad_repo_branch: master
+
+# Hadoop configrations
+hadoop_version: "{{ lookup('env','HADOOP_VERSION') }}"
+hadoop_binary_download_url: https://archive.apache.org/dist/hadoop/common/
+hadoop_binary_package: "{{ hadoop_binary_download_url }}/hadoop-{{
hadoop_version }}/hadoop-{{ hadoop_version }}.tar.gz"
+
+# Hadoop folders
+hadoop_data_root_folder: /cluster
+hadoop_namenode_folder: "{{ hadoop_data_root_folder }}/nn"
+hadoop_logs_folder: "{{ nfs_shared_folder }}/hadoop/logs"
+hadoop_etc_folder: "{{ nfs_shared_folder }}/hadoop/etc/hadoop"
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
index f814085..9c772b2 100644
--- a/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
+++ b/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
@@ -67,6 +67,9 @@
- name: Restarting SSH daemon
service: name=sshd state=restarted
+- name: Setup wide JAVA_HOME at /etc/profile.d
+ template: src=java.sh.j2 dest=/etc/profile.d/java.sh
+
- name: Configure /etc/host files
template: src=hosts.j2 dest=/etc/hosts
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
----------------------------------------------------------------------
diff --git
a/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
b/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
new file mode 100644
index 0000000..dba8aaa
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{% if ansible_fqdn in groups['bootstrap'] %}
+export JAVA_HOME=/usr/lib/jvm/java
+{% else %}
+export JAVA_HOME=/opt/mesosphere/active/java/usr/java
+{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
index 35e8af7..a826dec 100644
--- a/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
+++ b/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
@@ -102,7 +102,6 @@
- name: Downloading DC/OS CLI
shell: >
curl -fLsS --retry 20 -Y 100000 -y 60
- https://downloads.dcos.io/binaries/cli/linux/x86-64/dcos-1.11/dcos
https://downloads.dcos.io/binaries/cli/linux/x86-64/{{
versions[with_dcos_version].cli }}/dcos
-o dcos && touch .cliskip
args:
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
new file mode 100644
index 0000000..95b8581
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Install the 'Development tools' package group
+ yum:
+ name: "@Development tools"
+ state: present
+
+- name: Install OpenJDK for Myriad Development
+ yum: name=java-1.8.0-openjdk-devel state=present
+
+- name: Setup shared folder permissions
+ file:
+ path: "{{ nfs_shared_folder }}"
+ state: directory
+ owner: vagrant
+ group: vagrant
+ mode: 0755
+
+- name: Checkout Apache Myriad repo
+ become_user: vagrant
+ git:
+ repo: "{{ myriad_repository }}"
+ dest: "{{ nfs_shared_folder }}/{{ myriad_cloned_folder }}"
+ version: "{{ myriad_repo_branch }}"
+ update: no
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
new file mode 100644
index 0000000..002d4e3
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+<configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>hdfs://m1:9001</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
----------------------------------------------------------------------
diff --git
a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
new file mode 100644
index 0000000..572e61d
--- /dev/null
+++
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Apache Hadoop HDFS Datanode
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Environment="JAVA_HOME=/opt/mesosphere/active/java/usr/java"
+Environment="HADOOP_PID_DIR=/var/run/hadoop/hdfs"
+Environment="HADOOP_LOG_DIR=/opt/shared/hadoop/logs/"
+ExecStart=/opt/shared/hadoop/sbin/hadoop-daemon.sh --config
/opt/shared/hadoop/etc/hadoop start datanode
+User=hdfs
+Group=hadoop
+Type=forking
+Type=forking
+
+[Install]
+WantedBy=multi-user.target
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
----------------------------------------------------------------------
diff --git
a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
new file mode 100644
index 0000000..9395ad8
--- /dev/null
+++
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Apache Hadoop HDFS Namenode
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Environment="JAVA_HOME=/opt/mesosphere/active/java/usr/java"
+Environment="HADOOP_PID_DIR=/var/run/hadoop/hdfs"
+Environment="HADOOP_LOG_DIR=/opt/shared/hadoop/logs/"
+ExecStart=/opt/shared/hadoop/sbin/hadoop-daemon.sh --config
/opt/shared/hadoop/etc/hadoop start namenode
+User=hdfs
+Group=hadoop
+Type=forking
+
+[Install]
+WantedBy=multi-user.target
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
new file mode 100644
index 0000000..aaa8f64
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/cluster/nn</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/cluster/1/dn,/cluster/2/dn</value>
+ </property>
+
+ <property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
new file mode 100644
index 0000000..9b8543e
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Set right permissions for data folders
+ file:
+ path: "{{ item }}"
+ state: directory
+ owner: hdfs
+ group: hadoop
+ mode: 0755
+ with_items:
+ - "{{ hadoop_data_root_folder }}/1/dn"
+ - "{{ hadoop_data_root_folder }}/2/dn"
+ tags: hadoop
+
+- name: Copy Systemd DataNode Unit File
+ copy:
+ src: hadoop-hdfs-datanode.service
+ dest: /etc/systemd/system/hadoop-hdfs-datanode.service
+ tags: hadoop
+
+- name: Starting DataNode System Daemon
+ service: name=hadoop-hdfs-datanode state=started enabled=yes
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
new file mode 100644
index 0000000..9491c8d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Check that you can connect (GET) to a page and it returns a status 200
+ uri:
+ url: "{{ hadoop_binary_download_url }}"
+ tags: download_hadoop
+
+- name: Download Hadoop binary from Apache releases
+ get_url:
+ url: "{{ hadoop_binary_package }}"
+ dest: "{{ nfs_shared_folder }}"
+ tags: download_hadoop
+
+- name: Check if already moved
+ stat: path="{{ nfs_shared_folder }}/hadoop"
+ register: hadoop_stat
+ tags: download_hadoop
+
+- name: Unarchive a file that is already on the remote machine
+ unarchive:
+ src: "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}.tar.gz"
+ dest: "{{ nfs_shared_folder }}"
+ remote_src: yes
+ owner: vagrant
+ group: vagrant
+ when: hadoop_stat.stat.isdir is not defined
+ tags: download_hadoop
+
+- name: Rename folder name
+ command: mv "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}" "{{
nfs_shared_folder }}/hadoop"
+ when: hadoop_stat.stat.isdir is not defined
+ tags: download_hadoop
+
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
new file mode 100644
index 0000000..47bb5ac
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- include: download.yml
+ when: inventory_hostname == 'bt'
+ tags: hadoop
+
+- name: Create Hadoop group
+ group:
+ name: hadoop
+ state: present
+ tags: hadoop
+
+- name: Create Hadoop users
+ user:
+ name: "{{ item }}"
+ password: "{{ 'vagrant' | password_hash('sha512') }}"
+ shell: /bin/bash
+ group: hadoop
+ with_items:
+ - yarn
+ - hdfs
+ - mapred
+ tags: hadoop
+
+- name: Create Hadoop PID file folder
+ file:
+ path: /var/run/hadoop/hdfs
+ state: directory
+ mode: 0755
+ owner: hdfs
+ group: hadoop
+
+- include: namenode.yml
+ when: inventory_hostname in groups['dcos-masters']
+ tags: hadoop
+
+- include: datanode.yml
+ when: inventory_hostname in groups['dcos-agents-private'] and
inventory_hostname != 'a1'
+ tags: hadoop
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
new file mode 100644
index 0000000..7ac742d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create Hadoop metadata folder
+ file:
+ path: "{{ hadoop_namenode_folder }}"
+ state: directory
+ owner: hdfs
+ group: hadoop
+ mode: 0755
+ tags: hadoop
+
+- name: Create local logs folder for Hadoop daemons
+ file:
+ path: "{{ hadoop_logs_folder }}"
+ state: directory
+ owner: hdfs
+ group: hadoop
+ mode: 0775
+ tags: hadoop
+
+- name: Copy minimal (core,hdfs)-site.xml files
+ copy:
+ src: "{{ item }}"
+ dest: "{{ hadoop_etc_folder }}"
+ with_items:
+ - core-site.xml
+ - hdfs-site.xml
+ tags: hadoop
+
+- name: Format Hadoop filesystem
+ shell : "source /etc/profile.d/java.sh && {{ nfs_shared_folder
}}/hadoop/bin/hdfs namenode -format && touch .skipformat"
+ become: yes
+ become_user: hdfs
+ args:
+ creates: .skipformat
+ run_once: yes
+ tags: hadoop
+
+- name: Copy Systemd NameNode Unit File
+ copy:
+ src: hadoop-hdfs-namenode.service
+ dest: /etc/systemd/system/hadoop-hdfs-namenode.service
+ tags: hadoop
+
+- name: Starting NameNode System Daemon
+ service: name=hadoop-hdfs-namenode state=started enabled=yes
+
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
new file mode 100644
index 0000000..e13911e
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create partitions on JBOD disks
+ parted:
+ device: "{{ item }}"
+ number: 1
+ state: present
+ with_items:
+ - /dev/vdb
+ - /dev/vdc
+ tags: disk
+
+- name: Create a xfs filesystems on JBOD disk
+ filesystem:
+ fstype: xfs
+ dev: "{{ item }}"
+ with_items:
+ - /dev/vdb1
+ - /dev/vdc1
+ tags: disk
+
+- name: Create Hadoop data folders
+ file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0755
+ with_items:
+ - "{{ hadoop_data_root_folder }}/1/dn"
+ - "{{ hadoop_data_root_folder }}/2/dn"
+ tags: disk
+
+- name: Mount up JBOD devices 1
+ mount:
+ path: "{{ hadoop_data_root_folder }}/1/dn"
+ src: /dev/vdb1
+ fstype: xfs
+ state: mounted
+ tags: disk
+
+- name: Mount up JBOD devices 2
+ mount:
+ path: "{{ hadoop_data_root_folder }}/2/dn"
+ src: /dev/vdc1
+ fstype: xfs
+ state: mounted
+ tags: disk
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
new file mode 100644
index 0000000..e571220
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create NFS mount point
+ file:
+ path: "{{ nfs_shared_folder }}"
+ state: directory
+
+- name: Mount NFS shared folder
+ mount:
+ path: "{{ nfs_shared_folder }}"
+ src: "bt:{{ nfs_shared_folder }}"
+ fstype: nfs
+ state: mounted
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
b/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
new file mode 100644
index 0000000..9e8111d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Creates directory
+ file:
+ path: /opt/shared
+ state: directory
+ owner: vagrant
+ group: vagrant
+ mode: 0775
+ when: inventory_hostname != 'a1'
+
+- name: Copy exports file.
+ template:
+ src: exports.j2
+ dest: /etc/exports
+ owner: root
+ group: root
+ mode: 0644
+ when: inventory_hostname == 'bt'
+
+- name: Enable NFS in all nodes
+ service: name={{ item }} state=started enabled=yes
+ with_items:
+ - "{{ nfs_packages }}"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
b/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
new file mode 100644
index 0000000..aa2b5dd
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# /etc/exports: the access control list for filesystems which may be exported
+# to NFS clients. See exports(5).
+#
+# Example for NFSv2 and NFSv3:
+# /srv/homes hostname1(rw,sync,no_subtree_check)
hostname2(ro,sync,no_subtree_check)
+#
+# Example for NFSv4:
+# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
+# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
+#
+{# % for item in play_hosts %}
+{{ nfs_shared_folder}} {{ hostvars[item]['ansible_host']
}}(rw,sync,no_root_squash)
+{% endfor % #}
+
+{{ nfs_shared_folder }} *(rw,sync,no_root_squash)
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/site.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/site.yml
b/vagrant/libvirt/dcos/provisioning/site.yml
index 416aadf..530371d 100644
--- a/vagrant/libvirt/dcos/provisioning/site.yml
+++ b/vagrant/libvirt/dcos/provisioning/site.yml
@@ -27,34 +27,47 @@
# non in special order, you cannot control the order
# of roles within a play.
-# Play 0
+# Play 1
- hosts: all
roles:
- common
- docker
+ - { role: jbod-disks,
+ when: "inventory_hostname in groups['dcos-agents-private'] and
inventory_hostname != 'a1'" }
-# Play 1
+# Play 2
- hosts: bootstrap
roles:
- passwordless
+ - devel
+ - nfs
- dcos-setup
- registry
+ - hadoop
-# Play 2
+# Play 3
- hosts: dcos-masters
strategy: free
vars:
node_type: master
roles:
- dcos-install
+ - nfs
+ - nfs-client
+ - hadoop
+# Play 4
- hosts: dcos-agents-private
strategy: free
vars:
node_type: slave
roles:
- dcos-install
+ - nfs
+ - nfs-client
+ - hadoop
+# Play 4
- hosts: dcos-agents-public
strategy: free
vars:
