Yingyi Bu has submitted this change and it was merged. Change subject: Auto-config SSDs for the AWS installtion. ......................................................................
Auto-config SSDs for the AWS installtion. Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1598 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Ian Maxon <ima...@apache.org> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- A asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml M asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml M asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml A asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml 4 files changed, 97 insertions(+), 0 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Jenkins: Verified; No violations found; No violations found; Verified diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml new file mode 100644 index 0000000..e0a060e --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml @@ -0,0 +1,37 @@ +one_ephemeral: + - { device_name: /dev/xvdb, ephemeral: ephemeral0 } +two_ephemeral: + - { device_name: /dev/xvdb, ephemeral: ephemeral0 } + - { device_name: /dev/xvdc, ephemeral: ephemeral1 } +three_ephemeral: + - { device_name: /dev/xvdb, ephemeral: ephemeral0 } + - { device_name: /dev/xvdc, ephemeral: ephemeral1 } + - { device_name: /dev/xvdd, ephemeral: ephemeral2 } +four_ephemeral: + - { device_name: /dev/xvdb, ephemeral: ephemeral0 } + - { device_name: /dev/xvdc, ephemeral: ephemeral1 } + - { device_name: /dev/xvdd, ephemeral: ephemeral2 } + - { device_name: /dev/xvde, ephemeral: ephemeral3 } + +ephemeral_volumes: + c1.medium: "{{ one_ephemeral }}" + c1.xlarge: "{{ four_ephemeral }}" + c3.large: "{{ two_ephemeral }}" + c3.xlarge: "{{ two_ephemeral }}" + c3.2xlarge: "{{ two_ephemeral }}" + c3.4xlarge: "{{ two_ephemeral }}" + c3.8xlarge: "{{ two_ephemeral }}" + i2.xlarge: "{{ one_ephemeral }}" + i2.2xlarge: "{{ two_ephemeral }}" + i2.4xlarge: "{{ four_ephemeral }}" + m1.small: "{{ one_ephemeral }}" + m1.medium: "{{ one_ephemeral }}" + m1.large: "{{ two_ephemeral }}" + m1.xlarge: "{{ four_ephemeral }}" + m2.xlarge: "{{ one_ephemeral }}" + m2.2xlarge: "{{ one_ephemeral }}" + m2.4xlarge: "{{ two_ephemeral }}" + m3.medium: "{{ one_ephemeral }}" + m3.large: "{{ one_ephemeral }}" + m3.xlarge: "{{ two_ephemeral }}" + m3.2xlarge: "{{ two_ephemeral }}" diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml index 802abcd..5287e34 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml @@ -25,8 +25,12 @@ inventory: "{{ temp_dir }}/inventory" ccconf: "{{ temp_dir }}/cc.conf" home_dir: "/home/{{ user }}" + vols: "{{ ephemeral_volumes[instance_type]|default([]) }}" tasks: - include_vars: ../conf/aws_settings.yml + - include_vars: ../conf/aws_types.yml + - debug: + msg: "{{ vols }}" - name: Launch all instances ec2: key_name: "{{ keypair }}" @@ -40,6 +44,7 @@ count: "{{ count }}" instance_tags: Name: "{{ tag }}" + volumes: "{{ vols }}" register: ec2 - name: Create local temporary directory @@ -97,6 +102,25 @@ - name: Populate NC iodevices to the cluster configuration file {{ ccconf }} shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}" + when: vols|length == 0 + + - name: Generates multiple iodevice paths + shell: printf "devices:\n" >> "{{ temp_dir }}/devices" + when: vols|length > 0 + + - name: Populate multiple iodevices + shell: printf " - /data{{ vol.0 + 1 }}/iodevice\n" >> "{{ temp_dir }}/devices" + with_indexed_items: vols + loop_control: + loop_var: vol + when: vols|length > 0 + + - include_vars: "{{ temp_dir }}/devices" + when: vols|length > 0 + + - name: Populate multiple NC iodevices to the cluster configuration file {{ ccconf }} + shell: printf "iodevices={{ devices| join(',') }}\n" >> "{{ ccconf }}" + when: vols|length > 0 - name: Populate the NC command to the cluster configuration file {{ ccconf }} shell: printf "command=asterixnc\n" >> "{{ ccconf }}" diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml index 7a5a840..cf09b69 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml @@ -24,6 +24,14 @@ - hosts: all tasks: + - include_vars: ../conf/aws_settings.yml + - include_vars: ../conf/aws_types.yml + + - name: Mount instance local stores + include: mount.yml + +- hosts: all + tasks: - include: install_jdk.yml - include_vars: ../conf/instance_settings.yml diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml new file mode 100644 index 0000000..2e16476 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml @@ -0,0 +1,28 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Set up all instance storage devices + shell: | + sudo umount "{{ vol.1.device_name }}" + sudo mkdir "/data{{ vol.0 + 1 }}" + sudo mount "{{ vol.1.device_name }}" "/data{{ vol.0 + 1 }}" + sudo chown -R $USER "/data{{ vol.0 + 1 }}" + with_indexed_items: "{{ ephemeral_volumes[instance_type]|default([]) }}" + loop_control: + loop_var: vol -- To view, visit https://asterix-gerrit.ics.uci.edu/1598 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <buyin...@gmail.com> Gerrit-Reviewer: Ian Maxon <ima...@apache.org> Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Yingyi Bu <buyin...@gmail.com>
