Yingyi Bu has submitted this change and it was merged. Change subject: Add AWS automation scripts to asterix-server. ......................................................................
Add AWS automation scripts to asterix-server. - Allows users to customize an AWS-based instance by editing conf/aws_settings.yml and conf/instance_settings.yml. - Starts an AWS cluster, install JDK and AsterixDB automatically. - Let the default value of storage.metadata.memorycomponent.numpages adjust to the available JVM heap size. Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d Reviewed-on: https://asterix-gerrit.ics.uci.edu/1475 Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-server/pom.xml M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml A asterixdb/asterix-server/src/main/aws/README A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml A asterixdb/asterix-server/src/main/aws/bin/start.sh A asterixdb/asterix-server/src/main/aws/bin/stop.sh A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java 17 files changed, 595 insertions(+), 8 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; Verified Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm index 384d759..42fb7c3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm @@ -104,7 +104,7 @@ "storage.memorycomponent.numcomponents" : 2, "storage.memorycomponent.numpages" : 8, "storage.memorycomponent.pagesize" : 131072, - "storage.metadata.memorycomponent.numpages" : 256, + "storage.metadata.memorycomponent.numpages" : 85, "transaction.log.dirs" : { "asterix_nc1" : "target/txnLogDir/asterix_nc1", "asterix_nc2" : "target/txnLogDir/asterix_nc2" diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm index 2476197..75c4d3e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm @@ -104,7 +104,7 @@ "storage.memorycomponent.numcomponents" : 2, "storage.memorycomponent.numpages" : 8, "storage.memorycomponent.pagesize" : 131072, - "storage.metadata.memorycomponent.numpages" : 256, + "storage.metadata.memorycomponent.numpages" : 85, "transaction.log.dirs" : { "asterix_nc1" : "target/txnLogDir/asterix_nc1", "asterix_nc2" : "target/txnLogDir/asterix_nc2" diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm index 5d124e7..76219aa 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm @@ -104,7 +104,7 @@ "storage.memorycomponent.numcomponents" : 2, "storage.memorycomponent.numpages" : 8, "storage.memorycomponent.pagesize" : 131072, - "storage.metadata.memorycomponent.numpages" : 256, + "storage.metadata.memorycomponent.numpages" : 85, "transaction.log.dirs" : { "asterix_nc1" : "target/txnLogDir/asterix_nc1", "asterix_nc2" : "target/txnLogDir/asterix_nc2" diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index faacc64..68ad80c 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -18,10 +18,10 @@ */ package org.apache.asterix.common.config; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; + import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.util.StorageUtil; - -import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; public class StorageProperties extends AbstractProperties { @@ -40,7 +40,6 @@ private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.metadata.memorycomponent.numpages"; - private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY = "storage.memorycomponent.numcomponents"; private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components @@ -53,6 +52,7 @@ private final long storageBufferCacheSizeDefault; private final int storageMemoryComponentNumPages; + private final int storageMetadataMemoryComponentNumPages; private final long storageMemorycomponentGlobalbudgetDefault; public StorageProperties(PropertiesAccessor accessor) { @@ -68,6 +68,10 @@ // for a dataset, including data and indexes. storageMemoryComponentNumPages = (int) (storageMemorycomponentGlobalbudgetDefault / (16 * getMemoryComponentPageSize())); + // By default, uses the min of 1/64 of the storageMemorycomponentGlobalbudgetDefault and 256 pages + // for the write buffer budget for a metadata dataset, including data and indexes. + storageMetadataMemoryComponentNumPages = Math + .min((int) (storageMemorycomponentGlobalbudgetDefault / (64 * getMemoryComponentPageSize())), 256); } @PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY) @@ -107,8 +111,7 @@ @PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY) public int getMetadataMemoryComponentNumPages() { return accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY, - STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + storageMetadataMemoryComponentNumPages, PropertyInterpreters.getIntegerPropertyInterpreter()); } @PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY) diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml index 3454428..f91ed64 100644 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@ -493,5 +493,9 @@ <artifactId>hadoop-minicluster</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> </dependencies> </project> diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml index b968ae1..32a0180 100644 --- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml +++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml @@ -52,6 +52,23 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> + <directory>src/main/aws</directory> + <outputDirectory>aws</outputDirectory> + <excludes> + <exclude>**/*.sh</exclude> + </excludes> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>src/main/aws</directory> + <outputDirectory>aws</outputDirectory> + <includes> + <include>**/*.sh</include> + </includes> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> <directory>target/appassembler/repo</directory> <outputDirectory>repo</outputDirectory> </fileSet> diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/aws/README new file mode 100644 index 0000000..ebd8fee --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/README @@ -0,0 +1,54 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +To start an AWS cluster, you need to do the following steps: + +1. Create an AWS account and an IAM user. + Sets up a security group that you'd like to use for your AWS cluster. The security group should at least allow + all TCP connection from anywhere. + +2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml; + retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml; + retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml. + Note that you can only read or download "access key ID" and "secret access key" once from your AWS console. + If you forget them, you have to create new keys again and delete the old ones. + +3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want. + Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup + in step 1. + +4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry: + Host *.amazonaws.com + IdentityFile <path_of_private_key> + Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key + pair that you uploaded to AWS and used in step 2. For example: + Host *.amazonaws.com + IdentityFile ~/.ssh/id_rsa + +5. Install ansible and boto3: + ansible: http://docs.ansible.com/ansible/intro_installation.html + boto3: pip install boto3 + +6. Launch your cluster instance on AWS: + bin/start.sh + Now you can use the AWS-based instance. + +7. Terminate the cluster instance on AWS: + bin/stop.sh + Note that it will destroy the instance and terminate all ec2 instances that you launched in step 6. diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml new file mode 100644 index 0000000..3deab82 --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml @@ -0,0 +1,48 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Start AWS cluster + hosts: localhost + gather_facts: false + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Launch all instances + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + exact_count: "{{ count }}" + count_tag: + Name: "{{ tag }}" + instance_tags: + Name: "{{ tag }}" + register: ec2 + - name: Create local temporary directory + file: + path: /tmp/asterixdb + state: directory + mode: 0755 + - name: Output the information of all nodes + local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes + diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml new file mode 100644 index 0000000..e1bdc86 --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Stop AWS cluster + hosts: localhost + gather_facts: false + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Stop all instance + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + exact_count: 0 diff --git a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml new file mode 100644 index 0000000..bb69b68 --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml @@ -0,0 +1,28 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Ensure the working directory exists + file: + path: "{{ binarydir }}" + state: directory + +- name: Unzip binary + unarchive: + src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}" + dest: "{{ binarydir }}" diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml new file mode 100644 index 0000000..743b70e --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml @@ -0,0 +1,63 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: ncs + tasks: + - include_vars: ../conf/instance_settings.yml + - name: Download JDK + shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \ + \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \ + \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""; + + - name: Install JDK + shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm + + - include: customize_deployment.yml + + - name: Ensure the log directory exists + file: + path: "{{ binarydir }}/logs" + state: directory + + - name: Ensure the io device directory exit + file: + path: "{{ basedir }}/iodevice" + state: directory + + - name: Start NC Service + shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" & + async: 10 + poll: 0 + +- hosts: cc + tasks: + - include_vars: ../conf/instance_settings.yml + - name: Copy cluster config to CC + copy: + src: /tmp/asterixdb/cc.conf + dest: "{{ basedir }}/cc.conf" + + - name: Update cluster config + shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g' + + - name: Start CC + shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" & + async: 10 + poll: 0 + diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/aws/bin/start.sh new file mode 100755 index 0000000..3779952 --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/bin/start.sh @@ -0,0 +1,38 @@ +#!/bin/bash -x +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + + +# Starts an AWS cluster. +ansible-playbook -i "localhost," ansible/aws_start.yml + +# Generates an Ansible inventory file and an AsterixDB configuration file. +temp=/tmp/asterixdb +inventory=$temp/inventory +conf=$temp/cc.conf +java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf + +# Waits a while so that all instances are up and running. +# TODO(yingyi) pull the "status check" field of each instance. +sleep 90 + +# Installs asterixdb on all AWS instances. +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $inventory ansible/instance_start.yml + diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh new file mode 100755 index 0000000..fd2c85b --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/bin/stop.sh @@ -0,0 +1,23 @@ +#!/bin/bash -x +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + + +# Terminates an AWS cluster +ansible-playbook -i "localhost," ansible/aws_stop.yml diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml new file mode 100644 index 0000000..06a7d27 --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml @@ -0,0 +1,46 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# The OS image id for ec2 instances. +image: ami-76fa4116 + +# The data center region for ec2 instances. +region: us-west-2 + +# The tag for each ec2 machine. +tag: scale_test + +# The name of a security group that appears in your AWS console. +group: default + +# The name of a key pair that appears in your AWS console. +keypair: <to be filled> + +# The AWS access key id for your IAM user. +access_key_id: <to be filled> + +# The AWS secrety key for your IAM user. +secret_access_key: <to be filled> + +# The AWS instance type. A full list of available types are listed at: +# https://aws.amazon.com/ec2/instance-types/ +instance_type: t2.micro + +# The number of ec2 instances that construct a cluster. +count: 2 diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml new file mode 100644 index 0000000..de83b7c --- /dev/null +++ b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml @@ -0,0 +1,48 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# The name of the product (or extension) being used. +product: asterixdb + +# The server binary zip. +binary: asterix-server-*-binary-assembly.zip + +# The script that starts a nc service. +ncsbin: "asterixncservice" + +# The script that starts a nc. +ncbin: "asterixnc" + +# The script that starts a cc. +ccbin: "asterixcc" + +# The parent directory for the working directory. +basedir: /home/ec2-user + +# The working directory. +binarydir: "{{ basedir }}/{{ product }}" + +# The pattern for retrieving the sever binary zip from the current build. +srcpattern: "../../../{{ binary }}" + +# The nc service command (script). +ncservice: "{{ binarydir}}/bin/{{ ncsbin }}" + +# The cc service command (script). +cc: "{{ binarydir}}/bin/{{ ccbin }}" diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java new file mode 100644 index 0000000..fb45643 --- /dev/null +++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterixdb.aws; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * The useful information for an AWS node. + */ +public class AwsNode { + + private final String privateIp; + private final String dnsName; + + public AwsNode(ObjectNode node) { + this.privateIp = node.get("private_ip").asText(); + this.dnsName = node.get("dns_name").asText(); + } + + public String getPrivateIp() { + return privateIp; + } + + public String getDnsName() { + return dnsName; + } +} diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java new file mode 100644 index 0000000..25895e8 --- /dev/null +++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterixdb.aws; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * This class is the binary that automatically generates : + * 1. an Ansible inventory file for the AWS cluster; + * 2. an AsterixDB configuration file for the AWS cluster + * from a JSON description file returned by the Ansible AWS startup script. + */ +public class ConfigGenerator { + + private ConfigGenerator() { + } + + public static void main(String[] args) throws Exception { + if (args.length < 3) { + System.err.println("The usage of ConfigGenerator: "); + System.err.println("<input node json file> <output inventory file> <output config file>"); + System.exit(0); + } + String source = args[0]; + String inventory = args[1]; + String config = args[2]; + + // Read Json file data to String + byte[] jsonData = Files.readAllBytes(Paths.get(source)); + + // Get a list of cluster nodes + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode root = (ObjectNode) objectMapper.readTree(jsonData); + ArrayNode nodes = (ArrayNode) root.get("tagged_instances"); + Iterator<JsonNode> nodeIterator = nodes.iterator(); + List<AwsNode> cluster = new ArrayList<>(); + while (nodeIterator.hasNext()) { + ObjectNode node = (ObjectNode) nodeIterator.next(); + cluster.add(new AwsNode(node)); + } + + if (cluster.isEmpty()) { + return; + } + + // Generates inventory file. + generateInventoryFile(cluster, inventory); + + // Generates asterixdb config + generateConfig(cluster, config); + } + + private static void generateInventoryFile(List<AwsNode> cluster, String inventoryPath) throws IOException { + Iterator<AwsNode> nodeIterator = cluster.iterator(); + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(inventoryPath)))) { + // Prints the cc section + writer.println("[cc]"); + String masterIp = nodeIterator.next().getDnsName(); + writer.println(masterIp); + writer.println(); + + // Prints the nc section + writer.println("[ncs]"); + writer.println(masterIp); // There is a NC that co-locates with CC. + while (nodeIterator.hasNext()) { + writer.println(nodeIterator.next().getDnsName()); + } + writer.println(); + + // Prints the user + writer.println("[all:vars]"); + writer.println("ansible_ssh_user=ec2-user"); + } + } + + private static void generateConfig(List<AwsNode> cluster, String configPath) throws IOException { + Iterator<AwsNode> nodeIterator = cluster.iterator(); + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(configPath)))) { + // Prints the cc section + writer.println("[cc]"); + String masterIp = nodeIterator.next().getPrivateIp(); + writer.println("cluster.address=" + masterIp); + writer.println(); + + // Prints the nc section + writer.println("[nc/1]"); + writer.println("address=" + masterIp); // There is a NC that co-locates with CC. + int ncCounter = 2; + while (nodeIterator.hasNext()) { + writer.println("[nc/" + ncCounter++ + "]"); + writer.println("address=" + nodeIterator.next().getPrivateIp()); + writer.println(); + } + + // Prints the nc parameter section. + writer.println("[nc]"); + String rootDirectory = "/home/ec2-user/"; + writer.println("txnlogdir=" + rootDirectory + "txnlog"); + + // TODO(yingyi): figure out how to get device mapping for SSD-based instances. + writer.println("iodevices=" + rootDirectory + "iodevice"); + writer.println("command=asterixnc"); + writer.println(); + } + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/1475 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d Gerrit-PatchSet: 10 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
