Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1475
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A
asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
13 files changed, 550 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/75/1475/1
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index faacc64..68ad80c 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,10 +18,10 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.StorageUtil;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
public class StorageProperties extends AbstractProperties {
@@ -40,7 +40,6 @@
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
- private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT
= 256; // ... so 32MB components
private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY =
"storage.memorycomponent.numcomponents";
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT =
2; // 2 components
@@ -53,6 +52,7 @@
private final long storageBufferCacheSizeDefault;
private final int storageMemoryComponentNumPages;
+ private final int storageMetadataMemoryComponentNumPages;
private final long storageMemorycomponentGlobalbudgetDefault;
public StorageProperties(PropertiesAccessor accessor) {
@@ -68,6 +68,10 @@
// for a dataset, including data and indexes.
storageMemoryComponentNumPages = (int)
(storageMemorycomponentGlobalbudgetDefault
/ (16 * getMemoryComponentPageSize()));
+ // By default, uses the min of 1/64 of the
storageMemorycomponentGlobalbudgetDefault and 256 pages
+ // for the write buffer budget for a metadata dataset, including data
and indexes.
+ storageMetadataMemoryComponentNumPages = Math
+ .min((int) (storageMemorycomponentGlobalbudgetDefault / (64 *
getMemoryComponentPageSize())), 256);
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
@@ -107,8 +111,7 @@
@PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY)
public int getMetadataMemoryComponentNumPages() {
return
accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY,
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ storageMetadataMemoryComponentNumPages,
PropertyInterpreters.getIntegerPropertyInterpreter());
}
@PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 3454428..f91ed64 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -493,5 +493,9 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
index b968ae1..32a0180 100644
--- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
+++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
@@ -52,6 +52,23 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <excludes>
+ <exclude>**/*.sh</exclude>
+ </excludes>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <includes>
+ <include>**/*.sh</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>target/appassembler/repo</directory>
<outputDirectory>repo</outputDirectory>
</fileSet>
diff --git a/asterixdb/asterix-server/src/main/aws/README
b/asterixdb/asterix-server/src/main/aws/README
new file mode 100644
index 0000000..73204fe
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/README
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+To start an AWS cluster, you need to do the following steps:
+
+1. Create an AWS account and an IAM user.
+
+2. Retrieve your AWS key pair name and fill that after "keypair:" in
conf/aws_settings.yml;
+ retrieve your AWS "access key ID" and fill that after "access_key_id:" in
conf/aws_settings.yml;
+ retrieve your AWS "secret access key" and fill that after
"secret_access_key:" in conf/aws_settings.yml.
+ Note that you can only read or download "access key ID" and "secret access
key" once from your AWS console.
+ If you forget them, you have to create new keys again and delete the old
ones.
+
+3. Customize other settings in conf/aws_settings.yml and
conf/instance_settings.yml to whatever you want.
+
+4. Configure your ssh setting by editing ~/.ssh/config and adding the
following entry:
+ Host *.amazonaws.com
+ IdentityFile <path_of_private_key>
+ Note that <path_of_private_key> should be replaced by the path to the file
that stores the private key for the key
+ pair that you uploaded to AWS and used in step 2. For example:
+ Host *.amazonaws.com
+ IdentityFile ~/.ssh/id_rsa
+
+5. Launch your cluster instance on AWS:
+ bin/start.sh
+ Now you can use the AWS-based instance.
+
+6. Terminate the cluster instance on AWS:
+ bin/stop.sh
+ Note that it will destroy the instance and terminate all ec2 instances that
you launched in step 5.
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
new file mode 100644
index 0000000..e30731c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
@@ -0,0 +1,47 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Start AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Launch all instances
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: "{{ count }}"
+ count_tag:
+ Name: "{{ tag }}"
+ instance_tags:
+ Name: "{{ tag }}"
+ register: ec2
+ - file:
+ path: /tmp/asterixdb
+ state: directory
+ mode: 0755
+ - name: Output the information of all nodes for future processing
+ local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes
+
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
new file mode 100644
index 0000000..e1bdc86
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
@@ -0,0 +1,35 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Stop AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Stop all instance
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: 0
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
new file mode 100644
index 0000000..1718ade
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
@@ -0,0 +1,68 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- hosts: ncs
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Download JDK
+ shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \
+ \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie\" \
+
\"http://download.oracle.com/otn-pub/java/jdk/8u60-b27/jre-8u60-linux-x64.rpm\"";
+
+ - name: Install JDK
+ shell: sudo yum -y localinstall jre-8u60-linux-x64.rpm
+
+ - name: Ensure the working directory exists
+ file:
+ path: "{{ binarydir }}"
+ state: directory
+
+ - name: Unzip binary
+ unarchive:
+ src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}"
+ dest: "{{ binarydir }}"
+
+ - name: Ensure the log directory exists
+ file:
+ path: "{{ binarydir }}/logs"
+ state: directory
+
+ - name: Ensure the io device directory exit
+ file:
+ path: "{{ basedir }}/iodevice"
+ state: directory
+
+ - name: Start NC Service
+ shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" &
+ async: 10
+ poll: 0
+
+- hosts: cc
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Copy cluster config to CC
+ copy:
+ src: /tmp/asterixdb/cc.conf
+ dest: "{{ basedir }}/cc.conf"
+
+ - name: Start CC
+ shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{
binarydir }}/logs/cc.log" &
+ async: 10
+ poll: 0
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh
b/asterixdb/asterix-server/src/main/aws/bin/start.sh
new file mode 100755
index 0000000..be33428
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/start.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# starts an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_start.yml
+
+# generates an Ansible inventory file and an AsterixDB configuration file
+temp=/tmp/asterixdb
+inventory=$temp/inventory
+conf=$temp/cc.conf
+java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes
$inventory $conf
+
+# wait a while so that all instances are up and running
+# TODO(yingyi) pull the "status check" field of each instance.
+sleep 60
+
+# install asterixdb on all AWS instances
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i $inventory ansible/instance_start.yml
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh
b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
new file mode 100755
index 0000000..44d5a3c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# starts an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
new file mode 100644
index 0000000..4d62667
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The OS image id for ec2 instances.
+image: ami-dce3b1bc
+
+# The data center region for ec2 instances.
+region: us-west-1
+
+# The tag for each ec2 machine.
+tag: scale_test
+
+# The name of a security group that appears in your AWS console.
+group: default
+
+# The name of a key pair that appears in your AWS console.
+keypair: <to be filled>
+
+# The AWS access key id for your IAM user.
+access_key_id: <to be filled>
+
+# The AWS secrety key for your IAM user.
+secret_access_key: <to be filled>
+
+# The AWS instance type. A full list of available types are listed at:
+# https://aws.amazon.com/ec2/instance-types/
+instance_type: t2.micro
+
+# The number of ec2 instances that construct a cluster.
+count: 2
diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
new file mode 100644
index 0000000..1008105
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
@@ -0,0 +1,45 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The name of the product (or extension) being used.
+product: asterixdb
+
+# The server binary zip.
+binary: asterix-server-*-binary-assembly.zip
+
+# The script that starts a nc service.
+ncsbin: "asterixncservice"
+
+# The script that starts a cc.
+ccbin: "asterixcc"
+
+# The parent directory for the working directory.
+basedir: /home/ec2-user
+
+# The working directory.
+binarydir: "{{ basedir }}/{{ product }}"
+
+# The pattern for retrieving the sever binary zip from the current build.
+srcpattern: "../../../{{ binary }}"
+
+# The nc service command (script).
+ncservice: "{{ binarydir}}/bin/{{ ncsbin }}"
+
+# The cc service command (script).
+cc: "{{ binarydir}}/bin/{{ ccbin }}"
diff --git
a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
new file mode 100644
index 0000000..8f66e08
--- /dev/null
+++
b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class AwsNode {
+
+ private final String publicIp;
+ private final String privateIp;
+ private final String dnsName;
+
+ public AwsNode(ObjectNode node) {
+ this.publicIp = node.get("public_ip").asText();
+ this.privateIp = node.get("private_ip").asText();
+ this.dnsName = node.get("dns_name").asText();
+ }
+
+ public String getPublicIp() {
+ return publicIp;
+ }
+
+ public String getPrivateIp() {
+ return privateIp;
+ }
+
+ public String getDnsName() {
+ return dnsName;
+ }
+}
diff --git
a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
new file mode 100644
index 0000000..6b6ee54
--- /dev/null
+++
b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ConfigGenerator {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.out.println("The usage of ConfigGenerator: ");
+ System.out.println("<input node json file> <output inventory file>
<output config file>");
+ System.exit(0);
+ }
+ String source = args[0];
+ String inventory = args[1];
+ String config = args[2];
+
+ // Read Json file data to String
+ byte[] jsonData = Files.readAllBytes(Paths.get(source));
+
+ // Get a list of cluster nodes
+ ObjectMapper objectMapper = new ObjectMapper();
+ ObjectNode root = (ObjectNode) objectMapper.readTree(jsonData);
+ ArrayNode nodes = (ArrayNode) root.get("tagged_instances");
+ Iterator<JsonNode> nodeIterator = nodes.iterator();
+ List<AwsNode> cluster = new ArrayList<>();
+ while (nodeIterator.hasNext()) {
+ ObjectNode node = (ObjectNode) nodeIterator.next();
+ cluster.add(new AwsNode(node));
+ }
+
+ if (cluster.size() <= 0) {
+ return;
+ }
+
+ // Generates inventory file.
+ generateInventoryFile(cluster, inventory);
+
+ // Generates asterixdb config
+ generateConfig(cluster, config);
+ }
+
+ private static void generateInventoryFile(List<AwsNode> cluster, String
inventoryPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new
FileOutputStream(inventoryPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getDnsName();
+ writer.println(masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[ncs]");
+ writer.println(masterIp); // There is a NC that co-locates with CC.
+ while (nodeIterator.hasNext()) {
+ writer.println(nodeIterator.next().getDnsName());
+ }
+ writer.println();
+
+ // Prints the user
+ writer.println("[all:vars]");
+ writer.println("ansible_ssh_user=ec2-user");
+ }
+ }
+
+ private static void generateConfig(List<AwsNode> cluster, String
configPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new
FileOutputStream(configPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getPrivateIp();
+ writer.println("cluster.address=" + masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[nc/1]");
+ writer.println("address=" + masterIp); // There is a NC that
co-locates with CC.
+ int ncCounter = 2;
+ while (nodeIterator.hasNext()) {
+ writer.println("[nc/" + ncCounter + "]");
+ writer.println("address=" +
nodeIterator.next().getPrivateIp());
+ writer.println();
+ }
+
+ // Prints the nc parameter section.
+ writer.println("[nc]");
+ String rootDirectory = "/home/ec2-user/";
+ writer.println("txnlogdir=" + rootDirectory + "txnlog");
+ writer.println("iodevices=" + rootDirectory + "iodevice");
+ writer.println("command=asterixnc");
+
writer.println("app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint");
+ writer.println();
+ }
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
