Repository: asterixdb Updated Branches: refs/heads/master e85902bd3 -> 235b21861
Ansible scripts for local cluster installation. This change includes the following parts: - Refactored aws scripts and share the common part with non-aws cluster installation; - Fixed aws_start.yml and aws_stop.yml to count based on the user-provided tag. Change-Id: I310f6f198c9ce41d7d7667dc4fbc21b3e34a77ab Reviewed-on: https://asterix-gerrit.ics.uci.edu/1525 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/235b2186 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/235b2186 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/235b2186 Branch: refs/heads/master Commit: 235b21861d7bcc081b0181cb642769928eadf016 Parents: e85902b Author: Yingyi Bu <[email protected]> Authored: Sun Feb 26 22:49:52 2017 -0800 Committer: Yingyi Bu <[email protected]> Committed: Mon Feb 27 08:15:14 2017 -0800 ---------------------------------------------------------------------- .../src/main/assembly/binary-assembly.xml | 20 ++++ .../asterix-server/src/main/opt/ansible/README | 60 +++++++++++ .../src/main/opt/ansible/bin/deploy.sh | 31 ++++++ .../src/main/opt/ansible/bin/erase.sh | 32 ++++++ .../src/main/opt/ansible/bin/start.sh | 31 ++++++ .../src/main/opt/ansible/bin/stop.sh | 32 ++++++ .../src/main/opt/ansible/conf/cc.conf | 24 +++++ .../main/opt/ansible/conf/instance_settings.yml | 45 ++++++++ .../src/main/opt/ansible/conf/inventory | 27 +++++ .../src/main/opt/ansible/yaml/deploy.yml | 29 +++++ .../src/main/opt/ansible/yaml/deploy_all.yml | 24 +++++ .../src/main/opt/ansible/yaml/erase.yml | 23 ++++ .../src/main/opt/ansible/yaml/erase_all.yml | 24 +++++ .../src/main/opt/ansible/yaml/gen_conf.yml | 37 +++++++ .../main/opt/ansible/yaml/instance_start.yml | 52 +++++++++ .../src/main/opt/ansible/yaml/instance_stop.yml | 46 ++++++++ .../src/main/opt/ansible/yaml/start_cc.yml | 33 ++++++ .../main/opt/ansible/yaml/start_ncservice.yml | 25 +++++ .../src/main/opt/ansible/yaml/wait_ssh.yml | 26 +++++ .../asterix-server/src/main/opt/aws/README | 20 ++-- .../src/main/opt/aws/ansible/aws_start.yml | 108 ------------------- .../src/main/opt/aws/ansible/aws_stop.yml | 35 ------ .../src/main/opt/aws/ansible/instance_start.yml | 73 ------------- .../src/main/opt/aws/bin/start.sh | 4 +- .../asterix-server/src/main/opt/aws/bin/stop.sh | 2 +- .../src/main/opt/aws/conf/instance_settings.yml | 45 -------- .../src/main/opt/aws/yaml/aws_start.yml | 102 ++++++++++++++++++ .../src/main/opt/aws/yaml/aws_stop.yml | 37 +++++++ .../src/main/opt/aws/yaml/install_jdk.yml | 26 +++++ .../src/main/opt/aws/yaml/instance_start.yml | 44 ++++++++ 30 files changed, 843 insertions(+), 274 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml index 48aa168..d0bd88a 100644 --- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml +++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml @@ -52,6 +52,26 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> + <directory>src/main/opt/ansible/yaml</directory> + <outputDirectory>opt/aws/yaml</outputDirectory> + <includes> + <include>**/*.yml</include> + </includes> + <excludes> + <exclude>**/instance_start.yml</exclude> + <exclude>**/gen_conf.yml</exclude> + </excludes> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>src/main/opt/ansible/conf</directory> + <outputDirectory>opt/aws/conf</outputDirectory> + <includes> + <include>**/instance_settings.yml</include> + </includes> + <filtered>true</filtered> + </fileSet> + <fileSet> <directory>target/appassembler/repo</directory> <outputDirectory>repo</outputDirectory> </fileSet> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/README ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/README b/asterixdb/asterix-server/src/main/opt/ansible/README new file mode 100644 index 0000000..32e7999 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/README @@ -0,0 +1,60 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +To start cluster-based instance, you need to do the following steps: + +1. (Pre-requisite) Install Ansible, boto, and boto3: + ansible: pip install ansible + boto: pip install boto + boto3: pip install boto3 + Make sure that the version of Ansible is no less than 2.2.1.0. + +2. Configure passwordless ssh from your current client that runs the scripts to + all nodes listed in conf/inventory. + +3. Edit the instance configuration file conf/cc.conf when necessary. + You can add/update whatever parameters in the [app] and [cc] sections. + +4. Edit the inventory file conf/inventory when necessary. + You mostly only need to sepecify the node DNS names (or IPs) for the cluster controller ([cc]) and node controllers + ([ncs]). + + If the ssh user account for target machines is different from your current username, please uncomment + and edit the following two lines: + ;[all:vars] + ;ansible_ssh_user=<fill with your ssh account username> + + If you want to specify advanced Ansible builtin variables, please refer to the following Ansible documentation: + http://docs.ansible.com/ansible/intro_inventory.html. + +5. Edit conf/instance_settings.yml to change the instance binary directories and commands when necessary. + By default, the binary directory will be under the home directory of the ssh user account on each node. + +6. Deploy the binary to all nodes: + bin/deploy.sh + +7. Launch your cluster instance: + bin/start.sh + Now you can use the cluster instance. + +8. If you want to stop the cluster instance, run the following script: + bin/stop.sh + +9. If you want to remove the binary on all nodes, run the following script: + bin/erase.sh http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/bin/deploy.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/deploy.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/deploy.sh new file mode 100644 index 0000000..eff1ad0 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/deploy.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +ANSB_PATH=`dirname "${SCRIPT_PATH}"` + +INVENTORY=$ANSB_PATH/conf/inventory + +# Deploy the installation binary to all nodes. +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/deploy_all.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/bin/erase.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/erase.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/erase.sh new file mode 100644 index 0000000..e7bf19d --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/erase.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +ANSB_PATH=`dirname "${SCRIPT_PATH}"` + +INVENTORY=$ANSB_PATH/conf/inventory + +# Erase the installation binary on all nodes. +# TODO(yingyi): erase all data/txn/log directories. +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/erase_all.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/bin/start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/start.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/start.sh new file mode 100755 index 0000000..55f5016 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/start.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +ANSB_PATH=`dirname "${SCRIPT_PATH}"` + +INVENTORY=$ANSB_PATH/conf/inventory + +# Start the cluster +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/instance_start.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/bin/stop.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/stop.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/stop.sh new file mode 100755 index 0000000..200d6e6 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/stop.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +ANSB_PATH=`dirname "${SCRIPT_PATH}"` + +INVENTORY=$ANSB_PATH/conf/inventory + +# Starts the cluster +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/instance_stop.yml + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf new file mode 100644 index 0000000..f5164ae --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf @@ -0,0 +1,24 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[app] +log.level=WARNING + +[nc] +txnlogdir=txnlog +iodevices=iodevice +command=asterixnc http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/conf/instance_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/opt/ansible/conf/instance_settings.yml new file mode 100644 index 0000000..912fac8 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/conf/instance_settings.yml @@ -0,0 +1,45 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# The name of the product (or extension) being used. +product: asterixdb + +# The script that starts a nc service. +ncsbin: "asterixncservice" + +# The script that starts a nc. +ncbin: "asterixnc" + +# The script that starts a cc. +ccbin: "asterixcc" + +# The parent directory for the working directory. +basedir: "{{ ansible_env.HOME }}" + +# The working directory. +binarydir: "{{ basedir }}/{{ product }}" + +# The full binary distribution directory. +dist: "../../../../" + +# The nc service command (script). +ncservice: "{{ binarydir}}/bin/{{ ncsbin }}" + +# The cc service command (script). +cc: "{{ binarydir}}/bin/{{ ccbin }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/conf/inventory ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/conf/inventory b/asterixdb/asterix-server/src/main/opt/ansible/conf/inventory new file mode 100644 index 0000000..140ed43 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/conf/inventory @@ -0,0 +1,27 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +; Sets the ansible ssh user account if it is different from the current user +; that runs the scripts. +;[all:vars] +;ansible_ssh_user=<fill with your ssh account username> + +[cc] +localhost + +[ncs] +localhost http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy.yml new file mode 100644 index 0000000..ff06614 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy.yml @@ -0,0 +1,29 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Deploy binaries + synchronize: + src: "{{ dist }}" + dest: "{{ binarydir }}" + archive: true + +- name: Ensure the log directory exists + file: + path: "{{ binarydir }}/logs" + state: directory http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_all.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_all.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_all.yml new file mode 100644 index 0000000..09ff365 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_all.yml @@ -0,0 +1,24 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: all + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: deploy.yml \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase.yml new file mode 100644 index 0000000..4ca4bcd --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase.yml @@ -0,0 +1,23 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Remove the binary directory + file: + path: "{{ binarydir }}" + state: absent \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase_all.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase_all.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase_all.yml new file mode 100644 index 0000000..272fabe --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/erase_all.yml @@ -0,0 +1,24 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: all + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: erase.yml \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml new file mode 100644 index 0000000..f1be8a1 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml @@ -0,0 +1,37 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + + +- name: Ensure that the local temporary directory exists + file: + path: "{{ temp_dir }}" + state: directory + +- name: Generate the cluster configuration file + shell: cp "{{ playbook_dir }}/../conf/cc.conf" "{{ ccconf }}" + +- name: Populate NCs to the cluster configuration file + shell: printf "\n[nc/{{ node.0 + 1 }}]\naddress={{ node.1 }}\n" >> "{{ ccconf }}" + with_indexed_items: "{{ groups['ncs'] }}" + loop_control: + loop_var: node + +- name: Populate CC to the cluster configuration file "{{ ccconf }}" + shell: printf "\n[cc]\ncluster.address={{ groups['cc'][0] }}\n\n" >> "{{ ccconf }}" + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml new file mode 100644 index 0000000..13de7cd --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml @@ -0,0 +1,52 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: [localhost,] + vars: + temp_dir: "/tmp/asterixdb" + inventory: "{{ temp_dir }}/inventory" + ccconf: "{{ temp_dir }}/cc.conf" + tasks: + - include: wait_ssh.yml + + - include_vars: ../conf/instance_settings.yml + + - include: gen_conf.yml + +- hosts: all + tasks: + - include_vars: ../conf/instance_settings.yml + + - name: Ensure the log directory exists + file: + path: "{{ binarydir }}/logs" + state: directory + +- hosts: ncs + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: start_ncservice.yml + +- hosts: cc + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: start_cc.yml + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_stop.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_stop.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_stop.yml new file mode 100644 index 0000000..7a942e9 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_stop.yml @@ -0,0 +1,46 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: all + tasks: + - name: Check CC and NC processes + shell: jps | egrep '(CDriver|NCService)' | awk '{print $1}' + register: procs + + - set_fact: + lines: "{{ procs.stdout_lines|length }}" + + - name: Stop the CC and NCs + shell: kill `jps | egrep '(CDriver|NCService)' | awk '{print $1}'` + when: lines != "0" + + - name: Wait two seconds + shell: sleep 2 + when: lines != "0" + + - name: Check leftover CC and NC processes + shell: jps | egrep '(CDriver|NCService)' | awk '{print $1}' + register: procs + + - set_fact: + lines: "{{ procs.stdout_lines|length }}" + + - name: Force stopping leftover CC and NC processes + shell: kill -9 `jps | egrep '(CDriver|NCService)' | awk '{print $1}'` + when: lines != "0" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml new file mode 100644 index 0000000..32b1579 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml @@ -0,0 +1,33 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Copy cluster config to CC + synchronize: + src: /tmp/asterixdb/cc.conf + dest: "{{ basedir }}/cc.conf" + +- name: Update cluster config + shell: find -P "{{ basedir }}/cc.conf" | xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g' + +- name: Start CC + shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" & + args: + chdir: "{{ binarydir }}" + async: 10 + poll: 0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_ncservice.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_ncservice.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_ncservice.yml new file mode 100644 index 0000000..0f2f85d --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_ncservice.yml @@ -0,0 +1,25 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Start NC Service + shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" & + args: + chdir: "{{ binarydir }}" + async: 10 + poll: 0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/ansible/yaml/wait_ssh.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/wait_ssh.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/wait_ssh.yml new file mode 100644 index 0000000..1e58cc7 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/wait_ssh.yml @@ -0,0 +1,26 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Wait for SSH to come up at the cluster controller + wait_for: host="{{ item }}" port=22 search_regex=OpenSSH delay=0 timeout=300 state=started + with_items: "{{ groups['cc'] }}" + +- name: Wait for SSH to come up at node controllers + wait_for: host="{{ item }}" port=22 search_regex=OpenSSH delay=0 timeout=300 state=started + with_items: "{{ groups['ncs'] }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/README ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/README b/asterixdb/asterix-server/src/main/opt/aws/README index 52d503a..d267720 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/README +++ b/asterixdb/asterix-server/src/main/opt/aws/README @@ -19,21 +19,27 @@ To start an AWS cluster, you need to do the following steps: -1. Create an AWS account and an IAM user. +1. (Pre-requisite) Install Ansible, boto, and boto3: + ansible: pip install ansible + boto: pip install boto + boto3: pip install boto3 + Make sure that the version of Ansible is no less than 2.2.1.0. + +2. Create an AWS account and an IAM user. Set up a security group that you'd like to use for your AWS cluster. The security group should at least allow all TCP connection from anywhere. -2. Retrieve your AWS EC2 key pair name and fill that after "keypair:" in conf/aws_settings.yml; +3. Retrieve your AWS EC2 key pair name and fill that after "keypair:" in conf/aws_settings.yml; retrieve your AWS IAM "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml; retrieve your AWS IAM "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml. Note that you can only read or download "access key ID" and "secret access key" once from your AWS console. If you forget them, you have to create new keys again and delete the old ones. -3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want. +4. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want. Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup in step 1. -4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry: +5. Configure your ssh setting by editing ~/.ssh/config and adding the following entry: Host *.amazonaws.com IdentityFile <path_of_private_key> Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key @@ -41,12 +47,6 @@ To start an AWS cluster, you need to do the following steps: Host *.amazonaws.com IdentityFile ~/.ssh/id_rsa -5. Install ansible, boto, and boto3: - ansible: pip install ansible - boto: pip install boto - boto3: pip install boto3 - Make sure that the version of ansible is no less than 2.2.1.0. - 6. Launch your cluster instance on AWS: bin/start.sh Now you can use the AWS-based instance. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml deleted file mode 100644 index 645f68f..0000000 --- a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml +++ /dev/null @@ -1,108 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- name: Start AWS cluster - hosts: localhost - gather_facts: false - vars: - temp_dir: "/tmp/asterixdb" - inventory: "{{ temp_dir }}/inventory" - ccconf: "{{ temp_dir }}/cc.conf" - home_dir: "/home/{{ user }}" - tasks: - - include_vars: ../conf/aws_settings.yml - - name: Launch all instances - ec2: - key_name: "{{ keypair }}" - group: "{{ group }}" - instance_type: "{{ instance_type }}" - image: "{{ image }}" - wait: true - region: "{{ region }}" - aws_access_key: "{{ access_key_id }}" - aws_secret_key: "{{ secret_access_key }}" - exact_count: "{{ count }}" - count_tag: - Name: "{{ tag }}" - instance_tags: - Name: "{{ tag }}" - register: ec2 - - - name: Create local temporary directory - file: - path: "{{ temp_dir }}" - state: directory - - - name: Set CC node - set_fact: - cc: "{{ ec2.instances[0] }}" - - - name: Populate CC to the cluster inventory file {{ inventory }} - shell: printf "[cc]\n{{ cc.public_dns_name }}\n" > "{{ inventory }}" - - - name: Start the NC section in the cluster inventory file {{ inventory }} - shell: printf "\n[ncs]\n" >> "{{ inventory }}" - - - name: Populate CC to the NC section (if needed) in the cluster configuration file "{{ ccconf }}" - shell: printf "[cc]\n{{ cc.private_ip }}\n\n" > "{{ ccconf }}" - when: cc_on_nc - - - name: Populate NCs to the cluster inventory file {{ inventory }} - shell: printf "{{ node.1.public_dns_name }}\n" >> "{{ inventory }}" - when: node.0 != 0 - with_indexed_items: "{{ ec2.instances }}" - loop_control: - loop_var: node - - - name: Populate NCs to the cluster inventory file {{ inventory }} - shell: printf "\n[all:vars]\nansible_ssh_user={{ user }}\n" >> "{{ inventory }}" - - - name: Populate CC to the cluster configuration file "{{ ccconf }}" - shell: printf "[cc]\ncluster.address={{ cc.private_ip }}\n\n" > "{{ ccconf }}" - - - name: Populate NCs to the cluster configuration file - shell: printf "[nc/1]\naddress={{ cc.private_ip }}\n\n" >> "{{ ccconf }}" - when: cc_on_nc - - - set_fact: - start: 0 - - - set_fact: - start: 1 - when: cc_on_nc - - - name: Populate NCs to the cluster configuration file - shell: printf "[nc/{{ node.0 + start }}]\naddress={{ node.1.private_ip }}\n\n" >> "{{ ccconf }}" - when: node.0 != 0 - with_indexed_items: "{{ ec2.instances }}" - loop_control: - loop_var: node - - - name: Populate the NC txnlogdir to the cluster configuration file {{ ccconf }} - shell: printf "[nc]\ntxnlogdir={{ home_dir }}/txnlog\n" >> "{{ ccconf }}" - - - name: Populate NC iodevices to the cluster configuration file {{ ccconf }} - shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}" - - - name: Populate the NC command to the cluster configuration file {{ ccconf }} - shell: printf "command=asterixnc\n" >> "{{ ccconf }}" - - - name: Wait for SSH to come up - wait_for: host="{{ item.public_ip }}" port=22 search_regex=OpenSSH delay=0 timeout=300 state=started - with_items: "{{ ec2.instances }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml deleted file mode 100644 index e1bdc86..0000000 --- a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- name: Stop AWS cluster - hosts: localhost - gather_facts: false - tasks: - - include_vars: ../conf/aws_settings.yml - - name: Stop all instance - ec2: - key_name: "{{ keypair }}" - group: "{{ group }}" - instance_type: "{{ instance_type }}" - image: "{{ image }}" - wait: true - region: "{{ region }}" - aws_access_key: "{{ access_key_id }}" - aws_secret_key: "{{ secret_access_key }}" - exact_count: 0 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml deleted file mode 100644 index e71066c..0000000 --- a/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml +++ /dev/null @@ -1,73 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- hosts: all - tasks: - - include_vars: ../conf/instance_settings.yml - - - name: Download JDK - shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \ - \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \ - \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""; - - - name: Install JDK - shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm - - - name: Deploy binaries - synchronize: - src: "{{ dist }}" - dest: "{{ binarydir }}" - archive: true - - - name: Ensure the log directory exists - file: - path: "{{ binarydir }}/logs" - state: directory - -- hosts: ncs - tasks: - - include_vars: ../conf/instance_settings.yml - - - - name: Ensure the io device directory exit - file: - path: "{{ basedir }}/iodevice" - state: directory - - - name: Start NC Service - shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" & - async: 10 - poll: 0 - -- hosts: cc - tasks: - - include_vars: ../conf/instance_settings.yml - - - name: Copy cluster config to CC - synchronize: - src: /tmp/asterixdb/cc.conf - dest: "{{ basedir }}/cc.conf" - - - name: Update cluster config - shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g' - - - name: Start CC - shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" & - async: 10 - poll: 0 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh index 2627a09..a836b4e 100755 --- a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh @@ -27,7 +27,7 @@ OPT_PATH=`dirname "${AWS_PATH}"` DIST_PATH=`dirname "${OPT_PATH}"` # Starts an AWS cluster. -ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_start.yml +ansible-playbook -i "localhost," $AWS_PATH/yaml/aws_start.yml # Generates an Ansible inventory file and an AsterixDB configuration file. temp=/tmp/asterixdb @@ -35,5 +35,5 @@ inventory=$temp/inventory # Installs asterixdb on all AWS instances. export ANSIBLE_HOST_KEY_CHECKING=false -ansible-playbook -i $inventory $AWS_PATH/ansible/instance_start.yml +ansible-playbook -i $inventory $AWS_PATH/yaml/instance_start.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh index 0f8bf98..48f4ff7 100755 --- a/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh @@ -25,4 +25,4 @@ popd > /dev/null AWS_PATH=`dirname "${SCRIPT_PATH}"` # Terminates an AWS cluster. -ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_stop.yml +ansible-playbook -i "localhost," $AWS_PATH/yaml/aws_stop.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml deleted file mode 100644 index cc517b2..0000000 --- a/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml +++ /dev/null @@ -1,45 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -# The name of the product (or extension) being used. -product: asterixdb - -# The script that starts a nc service. -ncsbin: "asterixncservice" - -# The script that starts a nc. -ncbin: "asterixnc" - -# The script that starts a cc. -ccbin: "asterixcc" - -# The parent directory for the working directory. -basedir: /home/ec2-user - -# The working directory. -binarydir: "{{ basedir }}/{{ product }}" - -# The full binary distribution directory. -dist: "../../../../" - -# The nc service command (script). -ncservice: "{{ binarydir}}/bin/{{ ncsbin }}" - -# The cc service command (script). -cc: "{{ binarydir}}/bin/{{ ccbin }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml new file mode 100644 index 0000000..e0abb12 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml @@ -0,0 +1,102 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Start AWS cluster + hosts: localhost + gather_facts: false + vars: + temp_dir: "/tmp/asterixdb" + inventory: "{{ temp_dir }}/inventory" + ccconf: "{{ temp_dir }}/cc.conf" + home_dir: "/home/{{ user }}" + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Launch all instances + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + count: "{{ count }}" + instance_tags: + Name: "{{ tag }}" + register: ec2 + + - name: Create local temporary directory + file: + path: "{{ temp_dir }}" + state: directory + + - name: Set CC node + set_fact: + cc: "{{ ec2.instances[0] }}" + + - name: Populate CC to the cluster inventory file {{ inventory }} + shell: printf "[cc]\n{{ cc.public_dns_name }}\n" > "{{ inventory }}" + + - name: Start the NC section in the cluster inventory file {{ inventory }} + shell: printf "\n[ncs]\n" >> "{{ inventory }}" + + - name: Populate CC to the NC section (if needed) in the cluster configuration file "{{ ccconf }}" + shell: printf "[cc]\n{{ cc.private_ip }}\n\n" > "{{ ccconf }}" + when: cc_on_nc + + - name: Populate NCs to the cluster inventory file {{ inventory }} + shell: printf "{{ node.1.public_dns_name }}\n" >> "{{ inventory }}" + when: node.0 != 0 + with_indexed_items: "{{ ec2.instances }}" + loop_control: + loop_var: node + + - name: Populate NCs to the cluster inventory file {{ inventory }} + shell: printf "\n[all:vars]\nansible_ssh_user={{ user }}\n" >> "{{ inventory }}" + + - name: Populate CC to the cluster configuration file "{{ ccconf }}" + shell: printf "[cc]\ncluster.address={{ cc.private_ip }}\n\n" > "{{ ccconf }}" + + - name: Populate NCs to the cluster configuration file + shell: printf "[nc/1]\naddress={{ cc.private_ip }}\n\n" >> "{{ ccconf }}" + when: cc_on_nc + + - set_fact: + start: 0 + + - set_fact: + start: 1 + when: cc_on_nc + + - name: Populate NCs to the cluster configuration file + shell: printf "[nc/{{ node.0 + start }}]\naddress={{ node.1.private_ip }}\n\n" >> "{{ ccconf }}" + when: node.0 != 0 + with_indexed_items: "{{ ec2.instances }}" + loop_control: + loop_var: node + + - name: Populate the NC txnlogdir to the cluster configuration file {{ ccconf }} + shell: printf "[nc]\ntxnlogdir={{ home_dir }}/txnlog\n" >> "{{ ccconf }}" + + - name: Populate NC iodevices to the cluster configuration file {{ ccconf }} + shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}" + + - name: Populate the NC command to the cluster configuration file {{ ccconf }} + shell: printf "command=asterixnc\n" >> "{{ ccconf }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_stop.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_stop.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_stop.yml new file mode 100644 index 0000000..b466ad4 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_stop.yml @@ -0,0 +1,37 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Stop AWS cluster + hosts: localhost + gather_facts: false + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Stop all instance + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + exact_count: 0 + count_tag: + Name: "{{ tag }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/yaml/install_jdk.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/install_jdk.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/install_jdk.yml new file mode 100644 index 0000000..05838bd --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/install_jdk.yml @@ -0,0 +1,26 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Download JDK + shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \ + \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \ + \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""; + +- name: Install JDK + shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm http://git-wip-us.apache.org/repos/asf/asterixdb/blob/235b2186/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml new file mode 100644 index 0000000..7a5a840 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml @@ -0,0 +1,44 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + + +- hosts: [localhost,] + tasks: + - include: wait_ssh.yml + +- hosts: all + tasks: + - include: install_jdk.yml + + - include_vars: ../conf/instance_settings.yml + + - include: deploy.yml + +- hosts: ncs + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: start_ncservice.yml + +- hosts: cc + tasks: + - include_vars: ../conf/instance_settings.yml + + - include: start_cc.yml +
