http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/config/cluster-1m4a.yaml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/config/cluster-1m4a.yaml b/vagrant/mesos/libvirt/mesos/config/cluster-1m4a.yaml deleted file mode 100644 index 3164e9f..0000000 --- a/vagrant/mesos/libvirt/mesos/config/cluster-1m4a.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: build - type: build - box: centos/7 - ip: 100.0.10.100 - mem: 4096 - cpu: 2 -- name: mesos-m1 - type: master - box: centos/7 - ip: 100.0.10.101 - mem: 2048 - cpu: 2 -- name: mesos-a1 - type: agent - box: centos/7 - ip: 100.0.10.102 - mem: 2048 - cpu: 2 -- name: mesos-a2 - type: agent - box: centos/7 - ip: 100.0.10.103 - mem: 2048 - cpu: 2 - aditional_disk: - - 50G - - 50G -- name: mesos-a3 - type: agent - box: centos/7 - mem: 2048 - ip: 100.0.10.104 - aditional_disk: - - 50G - - 50G - cpu: 2 -- name: mesos-a4 - type: agent - box: centos/7 - mem: 2048 - ip: 100.0.10.105 - aditional_disk: - - 50G - - 50G - cpu: 2
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/config/cluster-3m4a.yaml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/config/cluster-3m4a.yaml b/vagrant/mesos/libvirt/mesos/config/cluster-3m4a.yaml deleted file mode 100644 index cb51758..0000000 --- a/vagrant/mesos/libvirt/mesos/config/cluster-3m4a.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: build - type: build - box: centos/7 - ip: 100.0.10.100 - mem: 2048 - cpu: 2 -- name: mesos-m1 - type: master - box: centos/7 - ip: 100.0.10.101 - mem: 2048 - cpu: 2 -- name: mesos-m2 - type: master - box: centos/7 - ip: 100.0.10.102 - mem: 2048 - cpu: 2 -- name: mesos-m3 - type: master - box: centos/7 - ip: 100.0.10.103 - mem: 2048 - cpu: 2 -- name: mesos-a1 - type: agent - box: centos/7 - ip: 100.0.10.104 - mem: 2048 - cpu: 2 -- name: mesos-a2 - type: agent - box: centos/7 - ip: 100.0.10.105 - mem: 2048 - cpu: 2 - aditional_disk: - - 50G - - 50G -- name: mesos-a3 - type: agent - box: centos/7 - ip: 100.0.10.106 - mem: 1024 - cpu: 2 - aditional_disk: - - 50G - - 50G -- name: mesos-a4 - type: agent - box: centos/7 - mem: 2048 - ip: 100.0.10.107 - aditional_disk: - - 50G - - 50G - cpu: 2 http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/docs/ansible.txt ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/docs/ansible.txt b/vagrant/mesos/libvirt/mesos/docs/ansible.txt deleted file mode 100644 index d2c1d5b..0000000 --- a/vagrant/mesos/libvirt/mesos/docs/ansible.txt +++ /dev/null @@ -1 +0,0 @@ -ansible-playbook --private-key $HOME/.vagrant.d/insecure_private_key -i provisioning/inventory.py provisioning/site.yml --tags hadoop http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/docs/build-mesos.md ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/docs/build-mesos.md b/vagrant/mesos/libvirt/mesos/docs/build-mesos.md deleted file mode 100644 index 99981f4..0000000 --- a/vagrant/mesos/libvirt/mesos/docs/build-mesos.md +++ /dev/null @@ -1,135 +0,0 @@ -# Building Apache Mesos - -Note: Last tested Mesos version for Myriad -> 0.28.1 - -Notes about build Apache Mesos and some development hacking notes. - -# Building with autotools system - -``` -# Change working directory. -$ cd mesos - -# Bootstrap (Only required if building from git repository). -$ ./bootstrap - -# Configure and build. -$ mkdir build -$ cd build -$ ../configure -$ make - -# Run test suite. -$ make check - -# Install (Optional). -$ make install -``` - -# Building with new CMake system - -``` -curl -O https://cmake.org/files/v3.9/cmake-3.9.4-Linux-x86_64.tar.gz -tar xvzf cmake-3.9.4-Linux-x86_64.tar.gz -export PATH=$PATH:$HOME/cmake-3.9.4-Linux-x86_64/bin - -cd mesos -mkdir build && cd build -cmake .. -cmake --build . - - -For cleaning generated build system files: - -rm build/CMakeCache.txt -``` - -# Running Apache Mesos - -## Master - -``` -cd /home/vagrant/mesos.git/build -sudo ./bin/mesos-master.sh --ip=100.0.10.101 --work_dir=/var/lib/mesos -``` - -## Agents - -``` -cd /home/vagrant/mesos.git/build -sudo ./bin/mesos-agent.sh --master=100.0.10.101:5050 --work_dir=/var/lib/mesos -``` - -## Running with 3rdparty shipped Zookeeper - -At master: - -``` -cd /home/vagrant/mesos.git/build/3rdparty/zookeeper-3.4.8 -cp conf/zoo_sample.cfg conf/zoo.cfg -echo "server.1=mesos-m1:2888:3888" >> conf/zoo.cfg -bin/zkServer.sh start -echo ruok | nc 127.0.0.1 2181 - -sudo ./bin/mesos-master.sh --ip=100.0.10.101 --work_dir=/var/lib/mesos --zk=zk://mesos-m1:2181/mesos --quorum=1 -``` - -At agents: - -``` -sudo ./bin/mesos-agent.sh --master=zk://mesos-m1:2181/mesos --work_dir=/var/lib/mesos -``` - -# Debugging - -``` -gdbserver -> exec ${LIBTOOL} --mode=execute gdbserver 100.0.10.101:1234 \ - /home/vagrant/mesos.git/build/src/mesos-master "${@}" - - -(gdb) target remote mesos-m1:1234 -(gdb) continue -(gdb) monitor exit -``` - -# Development Environment with Vim - - -## Building clang-format - -``` -svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm -cd llvm/tools -svn co http://llvm.org/svn/llvm-project/cfe/trunk clang -cd ../.. -mkdir build -cd build -cmake -G "Unix Makefiles" ../llvm -``` - -## Code navigation: cscope and tags - -``` -sudo yum install cscope ctags -y - -cd build -make cscope -make ctags -``` - -## Vimrc with clang-format for C++11 and Google Style - -``` -cd -curl -O https://raw.githubusercontent.com/javiroman/reproducible-research/master/vim-tmux/vimrc -mv vimrc .vimrc -mkdir -p .vim/bundle -git clone https://github.com/VundleVim/Vundle.vim.git ~/.vim/bundle/Vundle.vim -vim -:PluginInstall -or from command line: vim +PluginInstall +qall -``` - - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/docs/notes-myriad.txt ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/docs/notes-myriad.txt b/vagrant/mesos/libvirt/mesos/docs/notes-myriad.txt deleted file mode 100644 index 1dd51d5..0000000 --- a/vagrant/mesos/libvirt/mesos/docs/notes-myriad.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copy the Myriad Scheduler -cp -vf myriad/myriad-scheduler/build/libs/* hadoop/share/hadoop/yarn/lib/ - -# Copy the Myriad Executor -cp -vf myriad/myriad-executor/build/libs/myriad-executor-0.2.0.jar hadoop/share/hadoop/yarn/lib/ - -# Copy the myriad-config-default.yml -cp myriad/myriad-scheduler/build/resources/main/myriad-config-default.yml hadoop/etc/hadoop/ - -#Hadoop 2.7.x: -#sbin/yarn-daemon.sh --config /opt/hadoop/etc/hadoop/ start resourcemanager http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/docs/steps-hadoop.txt ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/docs/steps-hadoop.txt b/vagrant/mesos/libvirt/mesos/docs/steps-hadoop.txt deleted file mode 100644 index 54e2c55..0000000 --- a/vagrant/mesos/libvirt/mesos/docs/steps-hadoop.txt +++ /dev/null @@ -1,145 +0,0 @@ -$ cat /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/java - -$ cat /opt/hadoop/etc/hadoop/workers -mesos-a2 -mesos-a3 -mesos-a4 - -- Set NameNode Location - -$ cat /opt/hadoop/etc/hadoop/core-site.xml -<configuration> - <property> - <name>fs.default.name</name> - <value>hdfs://mesos-m1:9000</value> - </property> -</configuration> - -- Set path for HDFS - -https://hortonworks.com/blog/hdfs-metadata-directories-explained/ - -NN: dfs.namenode.name.dir - -Determines where on the local filesystem the DFS name node should store the -name table(fsimage). If this is a comma-delimited list of directories -then the name table is replicated in all of the directories, for redundancy. - -DN: dfs.datanode.data.dir - -Although DataNodes do not contain metadata about the directories and files -stored in an HDFS cluster, they do contain a small amount of metadata about -the DataNode itself and its relationship to a cluster. - -- directories on the datanode's local filesystem that store HDFS blocks. -- dfs.datanode.data.dir: directory on your datanodes as the data directory. -dfs.datanode.data.dir=/volumes/disk1/hadoop/data/ - -$ cat /opt/hadoop/etc/hadoop/hdfs-site.xml - -<configuration> - <property> - <name>dfs.namenode.name.dir</name> - <value>/cluster/nn</value> - </property> - - <property> - <name>dfs.datanode.data.dir</name> - <value>/cluster/1/dn/data,/data/2/dn/data</value> - </property> - - <property> - <name>dfs.replication</name> - <value>3</value> - </property> -</configuration> - -- Set YARN as Job Scheduler -$ cat /opt/hadoop/etc/hadoop/mapred-site.xml -<configuration> - <property> - <name>mapreduce.framework.name</name> - <value>yarn</value> - </property> -</configuration> - -- Configure YARN -$ cat /opt/hadoop/etc/hadoop/yarn-site.xml -<configuration> - <property> - <name>yarn.acl.enable</name> - <value>0</value> - </property> - - <property> - <name>yarn.resourcemanager.hostname</name> - <value>mesos-m1</value> - </property> - - <property> - <name>yarn.nodemanager.aux-services</name> - <value>mapreduce_shuffle</value> - </property> -</configuration> - -- Duplicate Config Files on Each Node: NFS shared for this - -- HDFS needs to be formatted like any classical file system. On node-master, run the following command: - -- Permission, user and local folders: - -Master (NN): - -groupadd hadoop -useradd -g hadoop yarn -useradd -g hadoop hdfs -useradd -g hadoop mapred - -mkdir /opt/hadoop/logs -sudo chown -R hdfs:hadoop /opt/hadoop/logs - -sudo mkdir -p /cluster/nn/ -sudo chown -R hdfs:hadoop /cluster/nn/ - -- Format HDFS: - -For the HDFS NameNode to start, it needs to initialize the directory where it -will hold its data. - -sudo su - hdfs -cd /opt/hadoop -bin/hdfs namenode -format - -- Running HDFS cluster: - -sudo su - hdfs -sbin/start-dfs.sh - - -- Testing HDFS: - -sudo su - hdfs -bin/hdfs dfsadmin -printTopology -bin/hdfs dfsadmin -report - -- Running Yarn cluster: - -sudo su - yarn -sbin/start-yarn.sh - -- Testing Yarn: - -bin/yarn node -list -bin/yarn application -list - -For testing with Myriad we run only RM: - -sudo -E -u yarn bin/yarn --daemon start resourcemanager -sudo -E -u yarn jps - - - - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/group_vars/all/defaults.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/group_vars/all/defaults.yml b/vagrant/mesos/libvirt/mesos/provisioning/group_vars/all/defaults.yml deleted file mode 100644 index a509c11..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/group_vars/all/defaults.yml +++ /dev/null @@ -1,100 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -# Minimal CentOS system packages -system_packages: - - nc - - tar - - wget - - git - - docker-python - - epel-release - - vim - - tree - - ntp - - net-tools - - bind-utils - - xz - - unzip - - curl - - ipset - - nfs-utils - - gdb-gdbserver - - gcc-c++ - - sysstat - -# Dependencies for building Apache Mesos in CentOS -mesos_dependencies_packages: - - apache-maven - - python-devel - - python-six - - python-virtualenv - - java-1.8.0-openjdk-devel - - zlib-devel - - libcurl-devel - - openssl-devel - - cyrus-sasl-devel - - cyrus-sasl-md5 - - apr-devel - - subversion-devel - - apr-util-devel - - libevent - - libevent-devel - -system_epel_packages: - - pdsh - - sshpass - -# NFS enablement for sharing folders between nodes -nfs_packages: - - rpcbind - - nfs-server - - nfs-lock - - nfs-idmap -nfs_shared_folder: /opt - -# Source code location for Mesos and Myriad -mesos_cloned_folder: mesos -myriad_cloned_folder: myriad -mesos_repository: https://git-wip-us.apache.org/repos/asf/mesos.git -myriad_repository: https://github.com/myriad-framework/myriad -mesos_repo_branch: master -myriad_repo_branch: master - -# Zookeeper configrations -zookeeper_version: "{{ lookup('env','ZOOKEEPER_VERSION') }}" -zookeeper_binary_download_url: http://www-eu.apache.org/dist/zookeeper -zookeeper_binary_package: "{{ zookeeper_binary_download_url }}/zookeeper-{{ zookeeper_version }}/zookeeper-{{ zookeeper_version }}.tar.gz" - -# Hadoop configrations -hadoop_version: "{{ lookup('env','HADOOP_VERSION') }}" -hadoop_binary_download_url: https://archive.apache.org/dist/hadoop/common/ -hadoop_binary_package: "{{ hadoop_binary_download_url }}/hadoop-{{ hadoop_version }}/hadoop-{{ hadoop_version }}.tar.gz" - -# Hadoop folders -hadoop_data_root_folder: /cluster -hadoop_namenode_folder: "{{ hadoop_data_root_folder }}/nn" -hadoop_logs_folder: "{{ nfs_shared_folder }}/hadoop/logs" -hadoop_etc_folder: "{{ nfs_shared_folder }}/hadoop/etc/hadoop" - - - - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/inventory.py ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/inventory.py b/vagrant/mesos/libvirt/mesos/provisioning/inventory.py deleted file mode 100755 index b794c95..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/inventory.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import yaml -import os -import sys -import paramiko -import socket -import time -import logging - - -def get_cluster_config_file(): - '''Returns the file used as MESOS architecture''' - try: - ret = os.environ["MESOS_ARCH"] - except KeyError: - logging.debug( - "[Ansible] Please set environment variable MESOS_ARCH") - sys.exit(1) - - return "config/cluster-" + ret + ".yaml" - - -def get_cluster_config_yml(): - '''Returns a list with hostname and IP''' - with open(get_cluster_config_file(), 'r') as f: - ret = yaml.load(f) - - return ret - - -def check_ssh(ip, user, key_file, initial_wait=0, interval=0, retries=1): - logging.debug("[Ansible] checking SSH availability for %s", ip) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - time.sleep(initial_wait) - - for x in range(retries): - try: - ssh.connect(ip, username=user, key_filename=key_file) - return True - except (paramiko.BadHostKeyException, - paramiko.AuthenticationException, - paramiko.SSHException, - socket.error) as e: - logging.debug(e) - time.sleep(interval) - - return False - - -def check_ssh_available(cluster_yml): - ssh_key_path = os.environ["HOME"] + "/.vagrant.d/insecure_private_key" - - # check if the hosts are ssh accesibles - for item in cluster_yml: - if check_ssh(item.get('ip'), - "vagrant", - ssh_key_path, - 2, 2, 3): - logging.debug( - "[Ansible] %s: SSH is OK for provisioning", item) - else: - logging.debug("[Ansible] %s: SSH not ready", item) - return False - - return True - - -logging.basicConfig(filename='.vagrant/inventory.log', level=logging.DEBUG) - -all_vm_accesibles = False -logging.debug('[Ansible] getting host list from configuration') -cluster_yml = get_cluster_config_yml() - - -logging.debug("[Ansible] Sanity check loop for Ansible hosts") -while not all_vm_accesibles: - logging.debug("[Ansible] Waiting for SSH to become available in all hosts") - if check_ssh_available(cluster_yml): - all_vm_accesibles = True - - -class InventoryTemplate: - ''' - { - "all": { - "hosts": ["bt","a1","a2","p1","m1","m2","m3"], - "vars": { - "ansible_user": "vagrant", - "ansible_become": "true" - } - }, - "build-host": { - "hosts": ["build"] - }, - "mesos-masters": { - "hosts": ["master-m1","master-m3","master-m2"] - }, - "mesos-agents": { - "hosts": ["agent-a1","agent-a2"] - }, - "_meta": { - "hostvars": { - "build": {"ansible_host": "192.168.121.56"}, - "master-a1": {"ansible_host": "192.168.121.30"}, - "master-a2": {"ansible_host": "192.168.121.248"}, - "master-a3": {"ansible_host": "192.168.121.112"}, - "agent-a1": {"ansible_host": "192.168.121.80"}, - "agent-a2": {"ansible_host": "192.168.121.105"}, - "agent-a3": {"ansible_host": "192.168.121.150"} - } - } - } - ''' - - _template = """ - { - "all": { - "hosts": [%(_get_all|_pattern_a)s], - "vars": { - "ansible_user": "vagrant", - "ansible_become": "true" - } - }, - "build-host": { - "hosts": ["build"] - }, - "mesos-masters": { - "hosts": [%(_get_masters|_pattern_a)s] - }, - "mesos-agents": { - "hosts": [%(_get_agents|_pattern_a)s] - }, - "_meta": { - "hostvars": { - %(_get_all_hostip|_pattern_b)s - } - } - } - """ - - def __init__(self, dict={}): - self.dict = dict - - def __str__(self): - return self._template % self - - def __getitem__(self, key): - return self._process(key.split("|")) - - def _process(self, l): - arg = l[0] - if len(l) == 1: - if arg in self.dict: - return self.dict[arg] - elif hasattr(self, arg) and callable(getattr(self, arg)): - return getattr(self, arg)() - else: - raise KeyError(arg) - else: - func = l[1] - return getattr(self, func)(self._process([arg])) - - def _get_all(self): - cad = [] - for i in cluster_yml: - cad.append(i.get('name')) - return cad - - def _get_masters(self): - cad = [] - for i in cluster_yml: - if i.get('type') == 'master': - cad.append(i.get('name')) - return cad - - def _get_agents(iself): - cad = [] - for i in cluster_yml: - if i.get('type') == 'agent': - cad.append(i.get('name')) - return cad - - def _get_all_hostip(self): - cad = [] - for i in cluster_yml: - cad.append("\"%s\": {\"ansible_host\": \"%s\"}" % (i.get('name'), - i.get('ip'))) - return cad - - def _pattern_a(self, l): - return ",".join(["\"%s\"" % x for x in l]) - - def _pattern_b(self, l): - return ",".join(["%s" % x for x in l]) - - -print InventoryTemplate() http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/handlers/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/handlers/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/handlers/main.yml deleted file mode 100644 index 1768796..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/handlers/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -- name: Restart ntp - service: name=ntpd state=restarted http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/tasks/main.yml deleted file mode 100644 index b174b46..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/tasks/main.yml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Sanity checking Ansible version - run_once: true - delegate_to: localhost - assert: - that: - - ansible_version.full|version_compare('2.2.0.0', '>') - msg: "Please install at least ansible 2.2.x.x in your host" - -- name: Fetch the Apache Maven repo file - yum_repository: - name: epel-apache-maven - description: EPEL Apache Maven Repository - baseurl: http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-$releasever/$basearch/ - gpgcheck: no - -- name: Install CentOS prerequisites - yum: name={{ item }} state=latest - with_items: - - "{{ system_packages }}" - - "{{ mesos_dependencies_packages }}" - -- name: Install CentOS EPEL prerequisites - yum: name={{ item }} state=latest - with_items: - - "{{ system_epel_packages }}" - -- name: Disabling SELinux (anti-pattern) - selinux: - policy: targeted - state: permissive - -- name: Effective disabling of SELinux (avoiting reboot) - command: /usr/sbin/setenforce 0 - -- name: Stopping Firewall rules (anti-pattner) - service: name=firewalld state=stopped enabled=no - -- name: Effective disabling of iptables rules - command: /usr/sbin/iptables -F - -- name: Disabling Chronyd NTP server - service: name=chronyd state=stopped enabled=no - -- name: Passwordless pre-step PasswordAuthentication enabling - lineinfile: - dest: /etc/ssh/sshd_config - backup: yes - regexp: '^PasswordAuthentication' - line: 'PasswordAuthentication yes' - -- name: Restarting SSH daemon - service: name=sshd state=restarted - -- name: Configure /etc/host files - template: src=hosts.j2 dest=/etc/hosts - -- name: Configure ntp servers - template: src=ntp.conf.j2 dest=/etc/ntp.conf - notify: Restart ntp - -- name: Enabled ntpd service - service: name=ntpd enabled=yes - -# 'Mesos > 0.21.0' requires 'subversion > 1.8' devel package, -# which is not available in the default repositories. -# Create a WANdisco SVN repo file to install the correct version -- name: WANdisco SVN repo - blockinfile: - dest: /etc/yum.repos.d/wandisco-svn.repo - backup: yes - create: yes - block: | - [WANdiscoSVN] - name=WANdisco SVN Repo 1.9 - enabled=1 - baseurl=http://opensource.wandisco.com/centos/7/svn-1.9/RPMS/\$basearch/ - gpgcheck=1 - gpgkey=http://opensource.wandisco.com/RPM-GPG-KEY-WANdisco - -- name: Enable NFS in all nodes - service: name={{ item }} state=started - with_items: - - "{{ nfs_packages }}" - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hostname-fix.j2 ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hostname-fix.j2 b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hostname-fix.j2 deleted file mode 100644 index 6a39a55..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hostname-fix.j2 +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -if ! grep -e "^127\.0\.1\.1.*`hostname`.*" /etc/hosts > /dev/null ; then - echo "127.0.1.1 `hostname`" >> /etc/hosts -fi http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hosts.j2 ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hosts.j2 b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hosts.j2 deleted file mode 100644 index 100be34..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/hosts.j2 +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# {{ ansible_managed }} -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 - -{% for item in play_hosts %} -{% set short_name = item.split('.') %} -{{ hostvars[item]['ansible_host'] }} {{ item }} -{% endfor %} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/ntp.conf.j2 ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/ntp.conf.j2 b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/ntp.conf.j2 deleted file mode 100644 index d88de96..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/templates/ntp.conf.j2 +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -driftfile /var/lib/ntp/ntp.drift - -server 0.pool.ntp.org -server 1.pool.ntp.org -server 2.pool.ntp.org -server 3.pool.ntp.org http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/common/vars/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/vars/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/common/vars/main.yml deleted file mode 100644 index cd21505..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/common/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/handlers/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/handlers/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/handlers/main.yml deleted file mode 100644 index 7210200..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/handlers/main.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# -# This tasks can spend a lot of time, you have -# to take this into account. -# -- name: Restart NFS Server - service: name=nfs-server state=restarted - - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/tasks/main.yml deleted file mode 100644 index 67bfc8e..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/tasks/main.yml +++ /dev/null @@ -1,70 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Install the 'Development tools' package group - yum: - name: "@Development tools" - state: present - -- name: Setup shared folder permissions - file: - path: "{{ nfs_shared_folder }}" - state: directory - owner: vagrant - group: vagrant - mode: 0755 - -- name: Checkout Apache Mesos repo - become_user: vagrant - git: - repo: "{{ mesos_repository }}" - dest: "{{ nfs_shared_folder }}/{{ mesos_cloned_folder }}" - version: "{{ mesos_repo_branch }}" - update: no - -- name: Checkout Apache Myriad repo - become_user: vagrant - git: - repo: "{{ myriad_repository }}" - dest: "{{ nfs_shared_folder }}/{{ myriad_cloned_folder }}" - version: "{{ myriad_repo_branch }}" - update: no - -- name: Copy exports file. - template: - src: exports.j2 - dest: /etc/exports - owner: root - group: root - mode: 0644 - notify: Restart NFS Server - -- name: Start rpcbind - service: name=rpcbind state=started enabled=yes - -- name: Start NFS Server - service: name=nfs-server state=started enabled=yes - -- name: Start nfs-lock - service: name=nfs-lock state=started enabled=yes - -- name: Start nfs-idmap - service: name=nfs-idmap state=started enabled=yes - -- meta: flush_handlers http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/templates/exports.j2 ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/templates/exports.j2 b/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/templates/exports.j2 deleted file mode 100644 index aa2b5dd..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/devel/templates/exports.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# /etc/exports: the access control list for filesystems which may be exported -# to NFS clients. See exports(5). -# -# Example for NFSv2 and NFSv3: -# /srv/homes hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check) -# -# Example for NFSv4: -# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check) -# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check) -# -{# % for item in play_hosts %} -{{ nfs_shared_folder}} {{ hostvars[item]['ansible_host'] }}(rw,sync,no_root_squash) -{% endfor % #} - -{{ nfs_shared_folder }} *(rw,sync,no_root_squash) http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/dfs-start/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/dfs-start/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/dfs-start/tasks/main.yml deleted file mode 100644 index 066cfd6..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/dfs-start/tasks/main.yml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Format Hadoop filesystem - shell : "source /etc/profile.d/java.sh && {{ nfs_shared_folder }}/hadoop/bin/hdfs namenode -format && touch .skipformat" - become: yes - become_user: hdfs - args: - creates: .skipformat - run_once: yes - tags: hadoop - -- name: Start DFS cluster - shell : "source /etc/profile.d/java.sh && {{ nfs_shared_folder }}/hadoop/sbin/start-dfs.sh && touch .skipstartdfs" - become: yes - become_user: hdfs - args: - creates: .skipstartdfs - run_once: yes - tags: hadoop - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/docker/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/docker/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/docker/tasks/main.yml deleted file mode 100644 index e81f7d7..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/docker/tasks/main.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Setup OverlayFS storage driver automatic loading - lineinfile: - dest: /etc/modules-load.d/overlay.conf - create: yes - state: present - line: overlay - -- name: Load overlay kernel module (avoiting reboot) - modprobe: name=overlay state=present - -- name: Setup offial Docker repository - blockinfile: - dest: /etc/yum.repos.d/docker.repo - backup: yes - create: yes - block: | - [dockerrepo] - name=Docker Repository - baseurl=https://yum.dockerproject.org/repo/main/centos/7/ - enabled=1 - gpgcheck=1 - gpgkey=https://yum.dockerproject.org/gpg - -- name: Configure systemd to run the Docker Daemon with OverlayFS (folder) - file: path=/etc/systemd/system/docker.service.d state=directory - -- name: Configure systemd to run the Docker Daemon with OverlayFS (control file) - blockinfile: - dest: /etc/systemd/system/docker.service.d/override.conf - backup: yes - create: yes - block: | - [Service] - ExecStart= - ExecStart=/usr/bin/docker daemon --storage-driver=overlay -H fd:// - -- name: Installing Docker 1.11.12 version - yum: name=docker-engine-1.11.2 state=installed - -- name: Starting Docker daemon - service: name=docker state=started - -- name: Enabling Docker daemon - service: name=docker enabled=yes http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/core-site.xml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/core-site.xml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/core-site.xml deleted file mode 100644 index 6685a83..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/core-site.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- Put site-specific property overrides in this file. --> -<configuration> - <property> - <name>fs.default.name</name> - <value>hdfs://mesos-m1:9000</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/hdfs-site.xml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/hdfs-site.xml deleted file mode 100644 index aaa8f64..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/hdfs-site.xml +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - <property> - <name>dfs.namenode.name.dir</name> - <value>/cluster/nn</value> - </property> - - <property> - <name>dfs.datanode.data.dir</name> - <value>/cluster/1/dn,/cluster/2/dn</value> - </property> - - <property> - <name>dfs.replication</name> - <value>3</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/java.sh ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/java.sh b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/java.sh deleted file mode 100644 index b2fca19..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/java.sh +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -export JAVA_HOME=/usr/lib/jvm/java -export MESOS_NATIVE_JAVA_LIBRARY=/opt/mesos/build/src/.libs/libmesos.so http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/mapred-site.xml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/mapred-site.xml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/mapred-site.xml deleted file mode 100644 index 7e90cb5..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/mapred-site.xml +++ /dev/null @@ -1,24 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - <property> - <name>mapreduce.framework.name</name> - <value>yarn</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/myriad-config-default.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/myriad-config-default.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/myriad-config-default.yml deleted file mode 100644 index ef27ac1..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/myriad-config-default.yml +++ /dev/null @@ -1,76 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -## -mesosMaster: mesos-m1:5050 -checkpoint: false -frameworkFailoverTimeout: 43200000 -frameworkName: MyriadAlpha -#frameworkRole: "*" -frameworkRole: "test" -frameworkUser: yarn # User the Node Manager runs as, required if nodeManagerURI set, otherwise defaults to the user - # running the resource manager. -frameworkSuperUser: root # To be deprecated, currently permissions need set by a superuser due to Mesos-1790. Must be - # root or have passwordless sudo. Required if nodeManagerURI set, ignored otherwise. -#nativeLibrary: /usr/local/lib/libmesos.so -nativeLibrary: /opt/mesos/build/src/.libs/libmesos.so -zkServers: localhost:2181 -zkTimeout: 20000 -restApiPort: 8192 -#servedConfigPath: dist/config.tgz -#servedBinaryPath: dist/hadoop-2.6.0.tgz -profiles: - zero: # NMs launched with this profile dynamically obtain cpu/mem from Mesos - cpu: 0 - mem: 0 - small: - cpu: 1 - mem: 512 - medium: - cpu: 4 - mem: 4096 - large: - cpu: 10 - mem: 12288 -nmInstances: # NMs to start with. Requires at least 1 NM with a non-zero profile. - small: 1 # <profile_name : instances> -rebalancer: false -haEnabled: false -nodemanager: - jvmMaxMemoryMB: 1024 - cpus: 0.2 - cgroups: false -executor: - jvmMaxMemoryMB: 256 - #path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar - path: file:///opt/hadoop/share/hadoop/yarn/lib/myriad-executor-0.2.0.jar - #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. - #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz - #configUri: http://127.0.0.1/api/arifacts/config.tgz - #jvmUri: https://downloads.mycompany.com/java/jre-7u76-linux-x64.tar.gz -yarnEnvironment: - #YARN_HOME: /usr/local/hadoop - YARN_HOME: /opt/hadoop - #HADOOP_CONF_DIR=config - #HADOOP_TMP_DIR=$MESOS_SANDBOX - #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set - #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary - #JAVA_HOME: jre1.7.0_76 # Path to JRE distribution, relative to sandbox directory - JAVA_HOME: /usr/lib/jvm/java - #JAVA_LIBRARY_PATH: /opt/mycompany/lib -#mesosAuthenticationPrincipal: -#mesosAuthenticationSecretFilename: http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/slaves ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/slaves b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/slaves deleted file mode 100644 index bd6d62a..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/slaves +++ /dev/null @@ -1,3 +0,0 @@ -mesos-a2 -mesos-a3 -mesos-a4 http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/workers ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/workers b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/workers deleted file mode 100644 index bd6d62a..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/workers +++ /dev/null @@ -1,3 +0,0 @@ -mesos-a2 -mesos-a3 -mesos-a4 http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/yarn-site.xml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/yarn-site.xml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/yarn-site.xml deleted file mode 100644 index 82c0288..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/files/yarn-site.xml +++ /dev/null @@ -1,40 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<configuration> - <property> - <name>yarn.acl.enable</name> - <value>0</value> - </property> - - <property> - <name>yarn.resourcemanager.hostname</name> - <value>mesos-m1</value> - </property> - - <property> - <name>yarn.nodemanager.aux-services</name> - <value>mapreduce_shuffle,myriad_executor</value> - </property> - - <property> - <name>yarn.resourcemanager.scheduler.class</name> - <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value> - </property> - - <property> - <name>yarn.nodemanager.aux-services.myriad_executor.class</name> - <value>org.apache.myriad.executor.MyriadExecutorAuxService</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/build.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/build.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/build.yml deleted file mode 100644 index 9491c8d..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/build.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Check that you can connect (GET) to a page and it returns a status 200 - uri: - url: "{{ hadoop_binary_download_url }}" - tags: download_hadoop - -- name: Download Hadoop binary from Apache releases - get_url: - url: "{{ hadoop_binary_package }}" - dest: "{{ nfs_shared_folder }}" - tags: download_hadoop - -- name: Check if already moved - stat: path="{{ nfs_shared_folder }}/hadoop" - register: hadoop_stat - tags: download_hadoop - -- name: Unarchive a file that is already on the remote machine - unarchive: - src: "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}.tar.gz" - dest: "{{ nfs_shared_folder }}" - remote_src: yes - owner: vagrant - group: vagrant - when: hadoop_stat.stat.isdir is not defined - tags: download_hadoop - -- name: Rename folder name - command: mv "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}" "{{ nfs_shared_folder }}/hadoop" - when: hadoop_stat.stat.isdir is not defined - tags: download_hadoop - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/datanode.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/datanode.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/datanode.yml deleted file mode 100644 index 171ebf9..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/datanode.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Create Hadoop group - group: - name: hadoop - state: present - tags: hadoop - -- name: Create Hadoop users - user: - name: "{{ item }}" - password: "{{ 'vagrant' | password_hash('sha512') }}" - shell: /bin/bash - group: hadoop - with_items: - - yarn - - hdfs - - mapred - tags: hadoop - -- name: Create Hadoop data folders - file: - path: "{{ item }}" - state: directory - owner: hdfs - group: hadoop - mode: 0755 - with_items: - - "{{ hadoop_data_root_folder }}/1/dn" - - "{{ hadoop_data_root_folder }}/2/dn" - tags: hadoop - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/main.yml deleted file mode 100644 index 2480b24..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/main.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Setup system wide JAVA_HOME - copy: - src: java.sh - dest: /etc/profile.d - tags: hadoop - -- include: build.yml - when: inventory_hostname in groups['build-host'] - -- include: namenode.yml - when: inventory_hostname in groups['mesos-masters'] - -- include: datanode.yml - when: inventory_hostname in groups['mesos-agents'] and inventory_hostname != 'mesos-a1' - - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/namenode.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/namenode.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/namenode.yml deleted file mode 100644 index 659693e..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/hadoop/tasks/namenode.yml +++ /dev/null @@ -1,68 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Create Hadoop group - group: - name: hadoop - state: present - tags: hadoop - -- name: Create Hadoop users - user: - name: "{{ item }}" - password: "{{ 'vagrant' | password_hash('sha512') }}" - shell: /bin/bash - group: hadoop - with_items: - - yarn - - hdfs - - mapred - tags: hadoop - -- name: Create Hadoop metadata folder - file: - path: "{{ hadoop_namenode_folder }}" - state: directory - owner: hdfs - group: hadoop - mode: 0755 - tags: hadoop - -- name: Create local logs folder for Hadoop daemons - file: - path: "{{ hadoop_logs_folder }}" - state: directory - owner: hdfs - group: hadoop - mode: 0775 - tags: hadoop - -- name: Copy minimal (core,hdfs,mapred,yarn)-site.xml files - copy: - src: "{{ item }}" - dest: "{{ hadoop_etc_folder }}" - with_items: - - core-site.xml - - hdfs-site.xml - - mapred-site.xml - - yarn-site.xml - - workers - - slaves - - myriad-config-default.yml - tags: hadoop http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/nfs-client/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/nfs-client/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/nfs-client/tasks/main.yml deleted file mode 100644 index 50f03a8..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/nfs-client/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Create NFS mount point - file: - path: "{{ nfs_shared_folder }}" - state: directory - -- name: Mount NFS shared folder - mount: - path: "{{ nfs_shared_folder }}" - src: "build:{{ nfs_shared_folder }}" - fstype: nfs - state: mounted - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/build.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/build.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/build.yml deleted file mode 100644 index e9d75d4..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/build.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Generate RSA key-pair for passwordless - shell: ssh-keygen -t rsa -f .ssh/id_rsa -q -N "" && touch .skiprsa - become: yes - become_user: vagrant - args: - chdir: /home/vagrant - creates: .skiprsa - -- name: Install boot host public key in a remote authorized_keys - shell: > - sshpass -p vagrant ssh-copy-id -i .ssh/id_rsa {{ item }} -o StrictHostKeyChecking=no - become: yes - become_user: vagrant - args: - chdir: /home/vagrant - with_inventory_hostnames: - - all - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/main.yml deleted file mode 100644 index 81ade41..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- include: build.yml - when: inventory_hostname in groups['build-host'] - -- include: master.yml - when: inventory_hostname == 'mesos-m1' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/master.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/master.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/master.yml deleted file mode 100644 index 4b0b574..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/passwordless/tasks/master.yml +++ /dev/null @@ -1,83 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Create hdfs .ssh folder - file: - path: /home/hdfs/.ssh - state: directory - owner: hdfs - group: hadoop - mode: 0700 - tags: hadoop - -- name: Generate RSA key-pair for hdfs user passwordless - shell: ssh-keygen -t rsa -f .ssh/id_rsa -q -N "" && touch .skiprsa - become: yes - become_user: hdfs - args: - chdir: /home/hdfs - creates: .skiprsa - tags: hadoop - -- name: Install boot host public key in a remote authorized_keys - shell: > - sshpass -p vagrant ssh-copy-id -i .ssh/id_rsa {{ item }} -o StrictHostKeyChecking=no - become: yes - become_user: hdfs - args: - chdir: /home/hdfs - with_items: - - "{{ groups['mesos-masters'] }}" - - "{{ groups['mesos-agents'] | reject('search','mesos-a1') | list }}" - - 0.0.0.0 - tags: hadoop - -- name: Create yarn .ssh folder - file: - path: /home/yarn/.ssh - state: directory - owner: yarn - group: hadoop - mode: 0700 - tags: hadoop - -- name: Generate RSA key-pair for hdfs user passwordless - shell: ssh-keygen -t rsa -f .ssh/id_rsa -q -N "" && touch .skiprsa - become: yes - become_user: yarn - args: - chdir: /home/yarn - creates: .skiprsa - tags: hadoop - -- name: Install boot host public key in a remote authorized_keys - shell: > - sshpass -p vagrant ssh-copy-id -i .ssh/id_rsa {{ item }} -o StrictHostKeyChecking=no - become: yes - become_user: yarn - args: - chdir: /home/yarn - with_items: - - "{{ groups['mesos-masters'] }}" - - "{{ groups['mesos-agents'] | reject('search','mesos-a1') | list }}" - - 0.0.0.0 - tags: hadoop - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/yarn-start/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/yarn-start/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/yarn-start/tasks/main.yml deleted file mode 100644 index 8a06d00..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/yarn-start/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Start Yarn cluster - shell : "source /etc/profile.d/java.sh && {{ nfs_shared_folder }}/hadoop/sbin/start-yarn.sh && touch .skipstartyarn" - become: yes - become_user: yarn - args: - creates: .skipstartyarn - run_once: yes - tags: hadoop - - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/roles/zookeeper/tasks/main.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/roles/zookeeper/tasks/main.yml b/vagrant/mesos/libvirt/mesos/provisioning/roles/zookeeper/tasks/main.yml deleted file mode 100644 index 5a91bba..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/roles/zookeeper/tasks/main.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -- name: Check that you can connect (GET) to a page and it returns a status 200 - uri: - url: "{{ zookeeper_binary_download_url }}" - tags: download_zookeeper - -- name: Download Zookeeper binary from Apache releases - get_url: - url: "{{ zookeeper_binary_package }}" - dest: "{{ nfs_shared_folder }}" - tags: download_zookeeper - -- name: Check if already moved - stat: path="{{ nfs_shared_folder }}/zookeeper" - register: zookeeper_stat - tags: download_zookeeper - -- name: Unarchive a file that is already on the remote machine - unarchive: - src: "{{ nfs_shared_folder }}/zookeeper-{{ zookeeper_version }}.tar.gz" - dest: "{{ nfs_shared_folder }}" - remote_src: yes - owner: vagrant - group: vagrant - when: zookeeper_stat.stat.isdir is not defined - tags: download_zookeeper - -- name: Rename folder name - command: mv "{{ nfs_shared_folder }}/zookeeper-{{ zookeeper_version }}" "{{ nfs_shared_folder }}/zookeeper" - when: zookeeper_stat.stat.isdir is not defined - tags: download_zookeeper - - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/provisioning/site.yml ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/provisioning/site.yml b/vagrant/mesos/libvirt/mesos/provisioning/site.yml deleted file mode 100644 index f13a2f0..0000000 --- a/vagrant/mesos/libvirt/mesos/provisioning/site.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -# - Plays are map roles to hosts. -# - Roles are just abstractions around tasks -# - Each "hosts:"-block (a play) will be executed -# one by one. So you can use multiple -# blocks to force the desired execution order. -# - The plays are executed from top to down order. -# - Multiple roles declarations per play are executed -# non in special order, you cannot control the order -# of roles within a play. - -# Play 0 -- hosts: all - roles: - - common - -# Play 1 -- hosts: build-host - roles: - - passwordless - - devel - - hadoop -# - zookeeper - -# Play 3 -- hosts: mesos-masters mesos-agents - roles: - - docker - - nfs-client - - hadoop - - passwordless - -# Play 4 -- hosts: mesos-masters - roles: - - dfs-start - -# FIXME: Yarn ResourceManager hangs with SIGHUP from Ansible, problably -# fixed with setsid or something similar for detaching controlling tty. - -# Play 5 -#- hosts: mesos-masters -# roles: -# - yarn-start - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/libvirt/mesos/setup-env ---------------------------------------------------------------------- diff --git a/vagrant/mesos/libvirt/mesos/setup-env b/vagrant/mesos/libvirt/mesos/setup-env deleted file mode 100644 index d17d2a7..0000000 --- a/vagrant/mesos/libvirt/mesos/setup-env +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -export MESOS_ARCH=1m4a -export HADOOP_VERSION=2.7.0 -export ZOOKEEPER_VERSION=3.4.11 - -sh check-env.sh http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/virtualbox/Vagrantfile ---------------------------------------------------------------------- diff --git a/vagrant/mesos/virtualbox/Vagrantfile b/vagrant/mesos/virtualbox/Vagrantfile deleted file mode 100644 index 4ccc8a0..0000000 --- a/vagrant/mesos/virtualbox/Vagrantfile +++ /dev/null @@ -1,79 +0,0 @@ -# -*- mode: ruby -*- -# # vi: set ft=ruby : - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -VAGRANTFILE_API_VERSION = "2" - -HADOOP_VERSION="2.7.0" -PRIVATE_IP="10.141.141.20" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/trusty64" - config.vm.box_url = "https://vagrantcloud.com/ubuntu/boxes/trusty64"; - config.vm.network :private_network, ip: "#{PRIVATE_IP}" - - # Configure VM resources - config.vm.provider :virtualbox do |vb| - vb.name = "myriad-dev" - vb.customize ["modifyvm", :id, "--memory", "4096"] - vb.customize ["modifyvm", :id, "--cpus", "2"] - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] - vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"] - end - - if Vagrant.has_plugin?("vagrant-cachier") - # Configure cached packages to be shared between instances of the same base box. - config.cache.scope = :box - end - - # Forward mesos master and slave ports - config.vm.network "forwarded_port", guest: 5005, host: 5005 - config.vm.network "forwarded_port", guest: 5050, host: 5050 - config.vm.network "forwarded_port", guest: 5051, host: 5051 - - # Forward myriad web and admin ports - config.vm.network "forwarded_port", guest: 8080, host: 8080 - config.vm.network "forwarded_port", guest: 8081, host: 8081 - - # Forward YARN/Hadoop ports - config.vm.network "forwarded_port", guest: 50070, host: 50070 - config.vm.network "forwarded_port", guest: 50075, host: 50075 - config.vm.network "forwarded_port", guest: 8088, host: 8088 - config.vm.network "forwarded_port", guest: 8042, host: 8042 - config.vm.network "forwarded_port", guest: 19888, host: 19888 - config.vm.network "forwarded_port", guest: 8192, host: 8192 - config.vm.network "forwarded_port", guest: 2181, host: 2181 - - # install software - config.vm.provision "shell", path: "provisioning/install_default_jdk.sh" - config.vm.provision "shell", path: "provisioning/install_mesos.sh" - config.vm.provision "shell", path: "provisioning/install_docker.sh" - config.vm.provision "shell", path: "provisioning/install_hadoop.sh", args: ["#{HADOOP_VERSION}"] - - # start things up - config.vm.provision "shell", path: "provisioning/start_mesos_master.sh" - config.vm.provision "shell", path: "provisioning/start_mesos_slave.sh" - config.vm.provision "shell", path: "provisioning/format_namenode.sh" - config.vm.provision "shell", path: "provisioning/start_namenode.sh" - config.vm.provision "shell", path: "provisioning/start_datanode.sh" - config.vm.provision "shell", path: "provisioning/start_historyserver.sh" - -end http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/virtualbox/provisioning/format_namenode.sh ---------------------------------------------------------------------- diff --git a/vagrant/mesos/virtualbox/provisioning/format_namenode.sh b/vagrant/mesos/virtualbox/provisioning/format_namenode.sh deleted file mode 100644 index e944a41..0000000 --- a/vagrant/mesos/virtualbox/provisioning/format_namenode.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -v -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e - -# Format NameNode -sudo -u hduser sh -c 'yes Y | /usr/local/hadoop/bin/hdfs namenode -format' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/virtualbox/provisioning/install_default_jdk.sh ---------------------------------------------------------------------- diff --git a/vagrant/mesos/virtualbox/provisioning/install_default_jdk.sh b/vagrant/mesos/virtualbox/provisioning/install_default_jdk.sh deleted file mode 100644 index 0a85c3c..0000000 --- a/vagrant/mesos/virtualbox/provisioning/install_default_jdk.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -v -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -PREFIX="JAVA Default JDK Provisioner:" -set -e - -# For installing Java 8 -apt-get -y update -apt-get -y install default-jdk - -if $(test -e /usr/lib/libjvm.so); then - rm /usr/lib/libjvm.so -fi - -ln -s /usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1c3857a8/vagrant/mesos/virtualbox/provisioning/install_docker.sh ---------------------------------------------------------------------- diff --git a/vagrant/mesos/virtualbox/provisioning/install_docker.sh b/vagrant/mesos/virtualbox/provisioning/install_docker.sh deleted file mode 100644 index a0a214f..0000000 --- a/vagrant/mesos/virtualbox/provisioning/install_docker.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -v -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e - -#Install docker -echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -apt-get update -apt-get -y install docker.io
