Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Yingyi Bu (Code Review)]

Yingyi Bu has submitted this change and it was merged.

Change subject: Auto-config SSDs for the AWS installtion.
..


Auto-config SSDs for the AWS installtion.

Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1598
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
BAD: Jenkins 
Reviewed-by: Ian Maxon 
Integration-Tests: Jenkins 
---
A asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
A asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
4 files changed, 97 insertions(+), 0 deletions(-)

Approvals:
  Ian Maxon: Looks good to me, approved
  Jenkins: Verified; No violations found; No violations found; Verified



diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml 
b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
new file mode 100644
index 000..e0a060e
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
@@ -0,0 +1,37 @@
+one_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+two_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+three_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
+four_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
+  - { device_name: /dev/xvde, ephemeral: ephemeral3 }
+
+ephemeral_volumes:
+  c1.medium: "{{ one_ephemeral }}"
+  c1.xlarge: "{{ four_ephemeral }}"
+  c3.large: "{{ two_ephemeral }}"
+  c3.xlarge: "{{ two_ephemeral }}"
+  c3.2xlarge: "{{ two_ephemeral }}"
+  c3.4xlarge: "{{ two_ephemeral }}"
+  c3.8xlarge: "{{ two_ephemeral }}"
+  i2.xlarge: "{{ one_ephemeral }}"
+  i2.2xlarge: "{{ two_ephemeral }}"
+  i2.4xlarge: "{{ four_ephemeral }}"
+  m1.small: "{{ one_ephemeral }}"
+  m1.medium: "{{ one_ephemeral }}"
+  m1.large: "{{ two_ephemeral }}"
+  m1.xlarge: "{{ four_ephemeral }}"
+  m2.xlarge: "{{ one_ephemeral }}"
+  m2.2xlarge: "{{ one_ephemeral }}"
+  m2.4xlarge: "{{ two_ephemeral }}"
+  m3.medium: "{{ one_ephemeral }}"
+  m3.large: "{{ one_ephemeral }}"
+  m3.xlarge: "{{ two_ephemeral }}"
+  m3.2xlarge: "{{ two_ephemeral }}"
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml 
b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
index 802abcd..5287e34 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
@@ -25,8 +25,12 @@
  inventory: "{{ temp_dir }}/inventory"
  ccconf: "{{ temp_dir }}/cc.conf"
  home_dir: "/home/{{ user }}"
+ vols: "{{ ephemeral_volumes[instance_type]|default([]) }}"
   tasks:
 - include_vars: ../conf/aws_settings.yml
+- include_vars: ../conf/aws_types.yml
+- debug:
+msg: "{{ vols }}"
 - name: Launch all instances
   ec2:
 key_name: "{{ keypair }}"
@@ -40,6 +44,7 @@
 count: "{{ count }}"
 instance_tags:
 Name: "{{ tag }}"
+volumes: "{{ vols }}"
   register: ec2
 
 - name: Create local temporary directory
@@ -97,6 +102,25 @@
 
 - name: Populate NC iodevices to the cluster configuration file {{ ccconf 
}}
   shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}"
+  when: vols|length == 0
+
+- name: Generates multiple iodevice paths
+  shell: printf "devices:\n" >> "{{ temp_dir }}/devices"
+  when: vols|length > 0
+
+- name: Populate multiple iodevices
+  shell: printf "  - /data{{ vol.0 + 1 }}/iodevice\n" >> "{{ temp_dir 
}}/devices"
+  with_indexed_items: vols
+  loop_control:
+loop_var: vol
+  when: vols|length > 0
+
+- include_vars: "{{ temp_dir }}/devices"
+  when: vols|length > 0
+
+- name: Populate multiple NC iodevices to the cluster configuration file 
{{ ccconf }}
+  shell: printf "iodevices={{ devices| join(',') }}\n" >> "{{ ccconf }}"
+  when: vols|length > 0
 
 - name: Populate the NC command to the cluster configuration file {{ 
ccconf }}
   shell: printf "command=asterixnc\n" >> "{{ ccconf }}"
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml 
b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
index 7a5a840..cf09b69 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
@@ -24,6 

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/2252/ 
: SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1:

Integration Tests Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/2252/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Ian Maxon (Code Review)]

Ian Maxon has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1: Code-Review+2

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1: Integration-Tests-1

Integration Tests Failed

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/2251/ 
: UNSTABLE

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1: BAD+1

BAD Compatibility Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/847/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1:

Integration Tests Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/2251/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1:

Build Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/72/ (2/3)

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Jenkins (Code Review)]

Jenkins has posted comments on this change.

Change subject: Auto-config SSDs for the AWS installtion.
..


Patch Set 1:

Build Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4751/ (1/3)

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1598
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Auto-config SSDs for the AWS installtion.

[Yingyi Bu (Code Review)]

Yingyi Bu has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1598

Change subject: Auto-config SSDs for the AWS installtion.
..

Auto-config SSDs for the AWS installtion.

Change-Id: I584cc0a9e7420b2b9144635fee107494cb3fa487
---
A asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
A asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
4 files changed, 97 insertions(+), 0 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/98/1598/1

diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml 
b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
new file mode 100644
index 000..e0a060e
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
@@ -0,0 +1,37 @@
+one_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+two_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+three_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
+four_ephemeral:
+  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
+  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
+  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
+  - { device_name: /dev/xvde, ephemeral: ephemeral3 }
+
+ephemeral_volumes:
+  c1.medium: "{{ one_ephemeral }}"
+  c1.xlarge: "{{ four_ephemeral }}"
+  c3.large: "{{ two_ephemeral }}"
+  c3.xlarge: "{{ two_ephemeral }}"
+  c3.2xlarge: "{{ two_ephemeral }}"
+  c3.4xlarge: "{{ two_ephemeral }}"
+  c3.8xlarge: "{{ two_ephemeral }}"
+  i2.xlarge: "{{ one_ephemeral }}"
+  i2.2xlarge: "{{ two_ephemeral }}"
+  i2.4xlarge: "{{ four_ephemeral }}"
+  m1.small: "{{ one_ephemeral }}"
+  m1.medium: "{{ one_ephemeral }}"
+  m1.large: "{{ two_ephemeral }}"
+  m1.xlarge: "{{ four_ephemeral }}"
+  m2.xlarge: "{{ one_ephemeral }}"
+  m2.2xlarge: "{{ one_ephemeral }}"
+  m2.4xlarge: "{{ two_ephemeral }}"
+  m3.medium: "{{ one_ephemeral }}"
+  m3.large: "{{ one_ephemeral }}"
+  m3.xlarge: "{{ two_ephemeral }}"
+  m3.2xlarge: "{{ two_ephemeral }}"
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml 
b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
index 802abcd..5287e34 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
@@ -25,8 +25,12 @@
  inventory: "{{ temp_dir }}/inventory"
  ccconf: "{{ temp_dir }}/cc.conf"
  home_dir: "/home/{{ user }}"
+ vols: "{{ ephemeral_volumes[instance_type]|default([]) }}"
   tasks:
 - include_vars: ../conf/aws_settings.yml
+- include_vars: ../conf/aws_types.yml
+- debug:
+msg: "{{ vols }}"
 - name: Launch all instances
   ec2:
 key_name: "{{ keypair }}"
@@ -40,6 +44,7 @@
 count: "{{ count }}"
 instance_tags:
 Name: "{{ tag }}"
+volumes: "{{ vols }}"
   register: ec2
 
 - name: Create local temporary directory
@@ -97,6 +102,25 @@
 
 - name: Populate NC iodevices to the cluster configuration file {{ ccconf 
}}
   shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}"
+  when: vols|length == 0
+
+- name: Generates multiple iodevice paths
+  shell: printf "devices:\n" >> "{{ temp_dir }}/devices"
+  when: vols|length > 0
+
+- name: Populate multiple iodevices
+  shell: printf "  - /data{{ vol.0 + 1 }}/iodevice\n" >> "{{ temp_dir 
}}/devices"
+  with_indexed_items: vols
+  loop_control:
+loop_var: vol
+  when: vols|length > 0
+
+- include_vars: "{{ temp_dir }}/devices"
+  when: vols|length > 0
+
+- name: Populate multiple NC iodevices to the cluster configuration file 
{{ ccconf }}
+  shell: printf "iodevices={{ devices| join(',') }}\n" >> "{{ ccconf }}"
+  when: vols|length > 0
 
 - name: Populate the NC command to the cluster configuration file {{ 
ccconf }}
   shell: printf "command=asterixnc\n" >> "{{ ccconf }}"
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml 
b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
index 7a5a840..cf09b69 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
@@ -24,6 +24,14 @@
 
 - hosts: all
   tasks:
+- include_vars: ../conf/aws_settings.yml
+- include_vars: ../conf/aws_types.yml
+
+- name: Mount instance local stores
+  include: mount.yml
+
+- hosts: all
+  tasks:
 - include: install_jdk.yml
 
 - include_vars: ../conf/instance_settings.yml
diff --git