Repository: asterixdb Updated Branches: refs/heads/master 4bd3f5899 -> 7648b480c
Change "samples" to "opt" in the distribution package. This change also does the following things: - make AWS script more robust: (1) can be run from anywhere, (2) eliminate the need of customize_deployment.yml. - cleanup documentation to reflect the directory structure change. - fix Managix zip to be able to directories deeper than 2 levels. Change-Id: Ie7876376d089b9ebebba3d37ac90ac346a741be3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1482 Sonar-Qube: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Tested-by: Ian Maxon <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7648b480 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7648b480 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7648b480 Branch: refs/heads/master Commit: 7648b480cd682b77cec2fd945f76245350d63a10 Parents: 4bd3f58 Author: Yingyi Bu <[email protected]> Authored: Wed Feb 8 16:40:47 2017 -0800 Committer: Till Westmann <[email protected]> Committed: Wed Feb 8 23:58:52 2017 -0800 ---------------------------------------------------------------------- README.md | 2 +- .../event/service/AsterixEventServiceUtil.java | 17 +-- .../src/main/assembly/binary-assembly.xml | 25 +--- asterixdb/asterix-server/src/main/aws/README | 54 ------- .../src/main/aws/ansible/aws_start.yml | 48 ------ .../src/main/aws/ansible/aws_stop.yml | 35 ----- .../main/aws/ansible/customize_deployment.yml | 28 ---- .../src/main/aws/ansible/instance_start.yml | 63 -------- .../asterix-server/src/main/aws/bin/start.sh | 38 ----- .../asterix-server/src/main/aws/bin/stop.sh | 23 --- .../src/main/aws/conf/aws_settings.yml | 46 ------ .../src/main/aws/conf/instance_settings.yml | 48 ------ .../asterix-server/src/main/opt/aws/README | 55 +++++++ .../src/main/opt/aws/ansible/aws_start.yml | 48 ++++++ .../src/main/opt/aws/ansible/aws_stop.yml | 35 +++++ .../src/main/opt/aws/ansible/instance_start.yml | 65 ++++++++ .../src/main/opt/aws/bin/start.sh | 45 ++++++ .../asterix-server/src/main/opt/aws/bin/stop.sh | 28 ++++ .../src/main/opt/aws/conf/aws_settings.yml | 46 ++++++ .../src/main/opt/aws/conf/instance_settings.yml | 45 ++++++ .../main/opt/local/bin/start-sample-cluster.bat | 150 +++++++++++++++++++ .../main/opt/local/bin/start-sample-cluster.sh | 100 +++++++++++++ .../main/opt/local/bin/stop-sample-cluster.bat | 135 +++++++++++++++++ .../main/opt/local/bin/stop-sample-cluster.sh | 93 ++++++++++++ .../src/main/opt/local/conf/blue.conf | 19 +++ .../src/main/opt/local/conf/cc.conf | 38 +++++ .../samples/local/bin/start-sample-cluster.bat | 150 ------------------- .../samples/local/bin/start-sample-cluster.sh | 100 ------------- .../samples/local/bin/stop-sample-cluster.bat | 135 ----------------- .../samples/local/bin/stop-sample-cluster.sh | 93 ------------ .../src/main/samples/local/conf/blue.conf | 19 --- .../src/main/samples/local/conf/cc.conf | 38 ----- .../server/test/SampleLocalClusterIT.java | 2 +- 33 files changed, 916 insertions(+), 950 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index f6a1c2b..7f28757 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Here are steps to get AsterixDB running on your local machine: * Start a single-machine AsterixDB instance: $cd asterixdb/asterix-server/target/asterix-server-*-binary-assembly/ - $./samples/local/bin/start-sample-cluster.sh + $./opt/local/bin/start-sample-cluster.sh * Good to go and run queries in your browser at: http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java index 9b22396..b6cc339 100644 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java +++ b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java @@ -71,6 +71,7 @@ public class AsterixEventServiceUtil { public static final int CLUSTER_NET_PORT_DEFAULT = 1098; public static final int CLIENT_NET_PORT_DEFAULT = 1099; public static final int HTTP_PORT_DEFAULT = 8888; + private static final String EMPTY_STRING = ""; public static final String MANAGIX_CONF_XML = "conf" + File.separator + "managix-conf.xml"; @@ -345,25 +346,23 @@ public class AsterixEventServiceUtil { public static void zipDir(File sourceDir, File destFile) throws IOException { FileOutputStream fos = new FileOutputStream(destFile); ZipOutputStream zos = new ZipOutputStream(fos); - zipDir(sourceDir, destFile, zos); + zipDir(EMPTY_STRING, sourceDir, destFile, zos); zos.close(); } - private static void zipDir(File sourceDir, final File destFile, ZipOutputStream zos) throws IOException { - File[] dirList = sourceDir.listFiles(new FileFilter() { - public boolean accept(File f) { - return !f.getName().endsWith(destFile.getName()); - } - }); + private static void zipDir(String prefix, File sourceDir, final File destFile, ZipOutputStream zos) + throws IOException { + File[] dirList = sourceDir.listFiles(f->!f.getName().endsWith(destFile.getName())); + String newPrefix = prefix.equals(EMPTY_STRING) ? prefix : prefix + File.separator; for (int i = 0; i < dirList.length; i++) { File f = dirList[i]; if (f.isDirectory()) { - zipDir(f, destFile, zos); + zipDir(newPrefix + f.getName(), f, destFile, zos); } else { int bytesIn = 0; byte[] readBuffer = new byte[2156]; FileInputStream fis = new FileInputStream(f); - ZipEntry entry = new ZipEntry(sourceDir.getName() + File.separator + f.getName()); + ZipEntry entry = new ZipEntry(newPrefix + f.getName()); zos.putNextEntry(entry); while ((bytesIn = fis.read(readBuffer)) != -1) { zos.write(readBuffer, 0, bytesIn); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml index 32a0180..48aa168 100644 --- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml +++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml @@ -35,33 +35,16 @@ </includes> </fileSet> <fileSet> - <directory>src/main/samples</directory> - <outputDirectory>samples</outputDirectory> + <directory>src/main/opt</directory> + <outputDirectory>opt</outputDirectory> <excludes> <exclude>**/*.sh</exclude> </excludes> <filtered>true</filtered> </fileSet> <fileSet> - <directory>src/main/samples</directory> - <outputDirectory>samples</outputDirectory> - <includes> - <include>**/*.sh</include> - </includes> - <filtered>true</filtered> - <fileMode>0755</fileMode> - </fileSet> - <fileSet> - <directory>src/main/aws</directory> - <outputDirectory>aws</outputDirectory> - <excludes> - <exclude>**/*.sh</exclude> - </excludes> - <filtered>true</filtered> - </fileSet> - <fileSet> - <directory>src/main/aws</directory> - <outputDirectory>aws</outputDirectory> + <directory>src/main/opt</directory> + <outputDirectory>opt</outputDirectory> <includes> <include>**/*.sh</include> </includes> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/README ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/aws/README deleted file mode 100644 index ebd8fee..0000000 --- a/asterixdb/asterix-server/src/main/aws/README +++ /dev/null @@ -1,54 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -To start an AWS cluster, you need to do the following steps: - -1. Create an AWS account and an IAM user. - Sets up a security group that you'd like to use for your AWS cluster. The security group should at least allow - all TCP connection from anywhere. - -2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml; - retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml; - retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml. - Note that you can only read or download "access key ID" and "secret access key" once from your AWS console. - If you forget them, you have to create new keys again and delete the old ones. - -3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want. - Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup - in step 1. - -4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry: - Host *.amazonaws.com - IdentityFile <path_of_private_key> - Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key - pair that you uploaded to AWS and used in step 2. For example: - Host *.amazonaws.com - IdentityFile ~/.ssh/id_rsa - -5. Install ansible and boto3: - ansible: http://docs.ansible.com/ansible/intro_installation.html - boto3: pip install boto3 - -6. Launch your cluster instance on AWS: - bin/start.sh - Now you can use the AWS-based instance. - -7. Terminate the cluster instance on AWS: - bin/stop.sh - Note that it will destroy the instance and terminate all ec2 instances that you launched in step 6. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml deleted file mode 100644 index 3deab82..0000000 --- a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml +++ /dev/null @@ -1,48 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- name: Start AWS cluster - hosts: localhost - gather_facts: false - tasks: - - include_vars: ../conf/aws_settings.yml - - name: Launch all instances - ec2: - key_name: "{{ keypair }}" - group: "{{ group }}" - instance_type: "{{ instance_type }}" - image: "{{ image }}" - wait: true - region: "{{ region }}" - aws_access_key: "{{ access_key_id }}" - aws_secret_key: "{{ secret_access_key }}" - exact_count: "{{ count }}" - count_tag: - Name: "{{ tag }}" - instance_tags: - Name: "{{ tag }}" - register: ec2 - - name: Create local temporary directory - file: - path: /tmp/asterixdb - state: directory - mode: 0755 - - name: Output the information of all nodes - local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes - http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml deleted file mode 100644 index e1bdc86..0000000 --- a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- name: Stop AWS cluster - hosts: localhost - gather_facts: false - tasks: - - include_vars: ../conf/aws_settings.yml - - name: Stop all instance - ec2: - key_name: "{{ keypair }}" - group: "{{ group }}" - instance_type: "{{ instance_type }}" - image: "{{ image }}" - wait: true - region: "{{ region }}" - aws_access_key: "{{ access_key_id }}" - aws_secret_key: "{{ secret_access_key }}" - exact_count: 0 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml deleted file mode 100644 index bb69b68..0000000 --- a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml +++ /dev/null @@ -1,28 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- name: Ensure the working directory exists - file: - path: "{{ binarydir }}" - state: directory - -- name: Unzip binary - unarchive: - src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}" - dest: "{{ binarydir }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml deleted file mode 100644 index 743b70e..0000000 --- a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml +++ /dev/null @@ -1,63 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -- hosts: ncs - tasks: - - include_vars: ../conf/instance_settings.yml - - name: Download JDK - shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \ - \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \ - \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""; - - - name: Install JDK - shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm - - - include: customize_deployment.yml - - - name: Ensure the log directory exists - file: - path: "{{ binarydir }}/logs" - state: directory - - - name: Ensure the io device directory exit - file: - path: "{{ basedir }}/iodevice" - state: directory - - - name: Start NC Service - shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" & - async: 10 - poll: 0 - -- hosts: cc - tasks: - - include_vars: ../conf/instance_settings.yml - - name: Copy cluster config to CC - copy: - src: /tmp/asterixdb/cc.conf - dest: "{{ basedir }}/cc.conf" - - - name: Update cluster config - shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g' - - - name: Start CC - shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" & - async: 10 - poll: 0 - http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/bin/start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/aws/bin/start.sh deleted file mode 100755 index 3779952..0000000 --- a/asterixdb/asterix-server/src/main/aws/bin/start.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -x -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - - -# Starts an AWS cluster. -ansible-playbook -i "localhost," ansible/aws_start.yml - -# Generates an Ansible inventory file and an AsterixDB configuration file. -temp=/tmp/asterixdb -inventory=$temp/inventory -conf=$temp/cc.conf -java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf - -# Waits a while so that all instances are up and running. -# TODO(yingyi) pull the "status check" field of each instance. -sleep 90 - -# Installs asterixdb on all AWS instances. -export ANSIBLE_HOST_KEY_CHECKING=false -ansible-playbook -i $inventory ansible/instance_start.yml - http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/bin/stop.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh deleted file mode 100755 index fd2c85b..0000000 --- a/asterixdb/asterix-server/src/main/aws/bin/stop.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -x -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - - -# Terminates an AWS cluster -ansible-playbook -i "localhost," ansible/aws_stop.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml deleted file mode 100644 index 06a7d27..0000000 --- a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml +++ /dev/null @@ -1,46 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -# The OS image id for ec2 instances. -image: ami-76fa4116 - -# The data center region for ec2 instances. -region: us-west-2 - -# The tag for each ec2 machine. -tag: scale_test - -# The name of a security group that appears in your AWS console. -group: default - -# The name of a key pair that appears in your AWS console. -keypair: <to be filled> - -# The AWS access key id for your IAM user. -access_key_id: <to be filled> - -# The AWS secrety key for your IAM user. -secret_access_key: <to be filled> - -# The AWS instance type. A full list of available types are listed at: -# https://aws.amazon.com/ec2/instance-types/ -instance_type: t2.micro - -# The number of ec2 instances that construct a cluster. -count: 2 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml deleted file mode 100644 index de83b7c..0000000 --- a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml +++ /dev/null @@ -1,48 +0,0 @@ -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -# The name of the product (or extension) being used. -product: asterixdb - -# The server binary zip. -binary: asterix-server-*-binary-assembly.zip - -# The script that starts a nc service. -ncsbin: "asterixncservice" - -# The script that starts a nc. -ncbin: "asterixnc" - -# The script that starts a cc. -ccbin: "asterixcc" - -# The parent directory for the working directory. -basedir: /home/ec2-user - -# The working directory. -binarydir: "{{ basedir }}/{{ product }}" - -# The pattern for retrieving the sever binary zip from the current build. -srcpattern: "../../../{{ binary }}" - -# The nc service command (script). -ncservice: "{{ binarydir}}/bin/{{ ncsbin }}" - -# The cc service command (script). -cc: "{{ binarydir}}/bin/{{ ccbin }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/README ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/README b/asterixdb/asterix-server/src/main/opt/aws/README new file mode 100644 index 0000000..b071ca2 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/README @@ -0,0 +1,55 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +To start an AWS cluster, you need to do the following steps: + +1. Create an AWS account and an IAM user. + Set up a security group that you'd like to use for your AWS cluster. The security group should at least allow + all TCP connection from anywhere. + +2. Retrieve your AWS EC2 key pair name and fill that after "keypair:" in conf/aws_settings.yml; + retrieve your AWS IAM "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml; + retrieve your AWS IAM "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml. + Note that you can only read or download "access key ID" and "secret access key" once from your AWS console. + If you forget them, you have to create new keys again and delete the old ones. + +3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want. + Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup + in step 1. + +4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry: + Host *.amazonaws.com + IdentityFile <path_of_private_key> + Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key + pair that you uploaded to AWS and used in step 2. For example: + Host *.amazonaws.com + IdentityFile ~/.ssh/id_rsa + +5. Install ansible, boto, and boto3: + ansible: pip install ansible + boto: pip install boto + boto3: pip install boto3 + +6. Launch your cluster instance on AWS: + bin/start.sh + Now you can use the AWS-based instance. + +7. Terminate the cluster instance on AWS: + bin/stop.sh + Note that it will destroy the instance and terminate all ec2 instances that you launched in step 6. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml new file mode 100644 index 0000000..3deab82 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml @@ -0,0 +1,48 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Start AWS cluster + hosts: localhost + gather_facts: false + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Launch all instances + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + exact_count: "{{ count }}" + count_tag: + Name: "{{ tag }}" + instance_tags: + Name: "{{ tag }}" + register: ec2 + - name: Create local temporary directory + file: + path: /tmp/asterixdb + state: directory + mode: 0755 + - name: Output the information of all nodes + local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml new file mode 100644 index 0000000..e1bdc86 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Stop AWS cluster + hosts: localhost + gather_facts: false + tasks: + - include_vars: ../conf/aws_settings.yml + - name: Stop all instance + ec2: + key_name: "{{ keypair }}" + group: "{{ group }}" + instance_type: "{{ instance_type }}" + image: "{{ image }}" + wait: true + region: "{{ region }}" + aws_access_key: "{{ access_key_id }}" + aws_secret_key: "{{ secret_access_key }}" + exact_count: 0 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml new file mode 100644 index 0000000..27554a9 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml @@ -0,0 +1,65 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- hosts: ncs + tasks: + - include_vars: ../conf/instance_settings.yml + - name: Download JDK + shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \ + \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \ + \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""; + + - name: Install JDK + shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm + + - name: Deploy binaries + synchronize: + src: "{{ dist }}" + dest: "{{ binarydir }}" + + - name: Ensure the log directory exists + file: + path: "{{ binarydir }}/logs" + state: directory + + - name: Ensure the io device directory exit + file: + path: "{{ basedir }}/iodevice" + state: directory + + - name: Start NC Service + shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" & + async: 10 + poll: 0 + +- hosts: cc + tasks: + - include_vars: ../conf/instance_settings.yml + - name: Copy cluster config to CC + synchronize: + src: /tmp/asterixdb/cc.conf + dest: "{{ basedir }}/cc.conf" + + - name: Update cluster config + shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g' + + - name: Start CC + shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" & + async: 10 + poll: 0 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh new file mode 100755 index 0000000..eebdb9c --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +AWS_PATH=`dirname "${SCRIPT_PATH}"` +OPT_PATH=`dirname "${AWS_PATH}"` +DIST_PATH=`dirname "${OPT_PATH}"` + +# Starts an AWS cluster. +ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_start.yml + +# Generates an Ansible inventory file and an AsterixDB configuration file. +temp=/tmp/asterixdb +inventory=$temp/inventory +conf=$temp/cc.conf +java -cp "${DIST_PATH}/repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf + +# Waits a while so that all instances are up and running. +# TODO(yingyi) pull the "status check" field of each instance. +sleep 90 + +# Installs asterixdb on all AWS instances. +export ANSIBLE_HOST_KEY_CHECKING=false +ansible-playbook -i $inventory $AWS_PATH/ansible/instance_start.yml + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh new file mode 100755 index 0000000..0f8bf98 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +AWS_PATH=`dirname "${SCRIPT_PATH}"` + +# Terminates an AWS cluster. +ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_stop.yml http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml new file mode 100644 index 0000000..06a7d27 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml @@ -0,0 +1,46 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# The OS image id for ec2 instances. +image: ami-76fa4116 + +# The data center region for ec2 instances. +region: us-west-2 + +# The tag for each ec2 machine. +tag: scale_test + +# The name of a security group that appears in your AWS console. +group: default + +# The name of a key pair that appears in your AWS console. +keypair: <to be filled> + +# The AWS access key id for your IAM user. +access_key_id: <to be filled> + +# The AWS secrety key for your IAM user. +secret_access_key: <to be filled> + +# The AWS instance type. A full list of available types are listed at: +# https://aws.amazon.com/ec2/instance-types/ +instance_type: t2.micro + +# The number of ec2 instances that construct a cluster. +count: 2 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml new file mode 100644 index 0000000..d2a9ace --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml @@ -0,0 +1,45 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# The name of the product (or extension) being used. +product: asterixdb + +# The script that starts a nc service. +ncsbin: "asterixncservice" + +# The script that starts a nc. +ncbin: "asterixnc" + +# The script that starts a cc. +ccbin: "asterixcc" + +# The parent directory for the working directory. +basedir: /home/ec2-user + +# The working directory. +binarydir: "{{ basedir }}/{{ product }}" + +# The full binary distribution directory. +dist: "../../../" + +# The nc service command (script). +ncservice: "{{ binarydir}}/bin/{{ ncsbin }}" + +# The cc service command (script). +cc: "{{ binarydir}}/bin/{{ ccbin }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat new file mode 100644 index 0000000..580e1e0 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat @@ -0,0 +1,150 @@ +@REM ------------------------------------------------------------ +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ------------------------------------------------------------ + +@echo off +setlocal +goto opts + +:usage +echo. +echo Usage: %~nx0 [-f[orce]] +echo. +echo -f[orce] : Forces a start attempt when ${PRODUCT} processes are found to be running +exit /B 0 + +:opts +if "%1" == "" goto postopts + +if "%1" == "-f" ( + set force=1 +) else if "%1" == "-force" ( + set force=1 +) else if "%1" == "-usage" ( + goto :usage +) else if "%1" == "-help" ( + goto :usage +) else if "%1" == "--help" ( + goto :usage +) else if "%1" == "--usage" ( + goto :usage +) else ( + echo ERROR: unknown argument '%1' + call :usage + exit /B 1 +) +shift +goto opts +:postopts + +if NOT DEFINED JAVA_HOME ( + echo ERROR: JAVA_HOME not defined + goto :ERROR +) +REM ensure JAVA_HOME has no spaces nor quotes, since appassembler can't handle them +set JAVA_HOME=%JAVA_HOME:"=% +for %%I in ("%JAVA_HOME%") do ( + set JAVA_HOME=%%~sI +) + +set JAVACMD=%JAVA_HOME%\bin\java + +REM TODO(mblow): check java version, spaces in CWD + +set DIRNAME=%~dp0 + +pushd %DIRNAME%\.. +set CLUSTERDIR=%cd% +cd %CLUSTERDIR%\..\.. +set INSTALLDIR=%cd% +set LOGSDIR=%CLUSTERDIR%\logs + +echo CLUSTERDIR=%CLUSTERDIR% +echo INSTALLDIR=%INSTALLDIR% +echo LOGSDIR=%LOGSDIR% +echo. +cd %CLUSTERDIR% +if NOT EXIST %LOGSDIR% ( + mkdir %LOGSDIR% +) +call %INSTALLDIR%\bin\${HELPER_COMMAND} get_cluster_state -quiet + +IF %ERRORLEVEL% EQU 0 ( + echo ERROR: sample cluster address [localhost:${LISTEN_PORT}] already in use + goto :ERROR +) +set tempfile="%TEMP%\start-sample-cluster-%random%" + +wmic process where ^ + "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ + GET processid > %tempfile% 2>/dev/null + +set severity=ERROR +if "%force%" == "1" set severity=WARNING + +for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 + +if "%found%" == "1" ( + if "%force%" == "1" ( + echo %severity%: ${PRODUCT} processes are already running; -f[orce] specified, ignoring + del %tempfile% + ) else ( + echo %severity%: ${PRODUCT} processes are already running; aborting" + echo. + echo Re-run with -f to ignore, or run stop-sample-cluster.bat -f to forcibly terminate all running ${PRODUCT} processes: + for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P + del %tempfile% + exit /B 1 + ) +) + +goto :post_timestamp + +:timestamp +if "%1" == "" exit /B 0 +echo "--------------------------" >> %1 +echo "%date% %time%" >> %1 +echo "--------------------------" >> %1 +shift +goto :timestamp + +:post_timestamp +echo Starting sample cluster... + +call :timestamp %LOGSDIR%\blue-service.log %LOGSDIR%\red-service.log %LOGSDIR%\cc.log + +start /MIN "blue-nc" cmd /c "echo See output in %LOGSDIR%\blue-service.log && %INSTALLDIR%\bin\${NC_SERVICE_COMMAND} -logdir - -config-file %CLUSTERDIR%\conf\blue.conf >> %LOGSDIR%\blue-service.log 2>&1" +start /MIN "red-nc" cmd /c "echo See output in %LOGSDIR%\red-service.log && %INSTALLDIR%\bin\${NC_SERVICE_COMMAND} -logdir - >> %LOGSDIR%\red-service.log 2>&1" +start /MIN "cc" cmd /c "echo See output in %LOGSDIR%\cc.log && %INSTALLDIR%\bin\${CC_COMMAND} -config-file %CLUSTERDIR%\conf\cc.conf >>%LOGSDIR%\cc.log 2>&1" + +echo. +call %INSTALLDIR%\bin\${HELPER_COMMAND} wait_for_cluster -timeout 30 +if %ERRORLEVEL% EQU 0 ( + goto :END +) + +:ERROR +echo. +popd +endlocal +exit /B 1 + +:END +echo. +popd +endlocal http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh new file mode 100755 index 0000000..d702e8a --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +function usage() { + echo + echo Usage: $(basename $0) [-f[orce]] + echo + echo " -f[orce] : Forces a start attempt when ${PRODUCT} processes are found to be running" +} + +while [ -n "$1" ]; do + case $1 in + -f|-force) force=1;; + -help|--help|-usage|--usage) usage; exit 0;; + *) echo "ERROR: unknown argument '$1'"; usage; exit 1;; + esac + shift +done + +if [ -z "$JAVA_HOME" -a -x /usr/libexec/java_home ]; then + JAVA_HOME=$(/usr/libexec/java_home) + export JAVA_HOME +fi + +[ -z "$JAVA_HOME" ] && { + echo "JAVA_HOME not set" + exit 1 +} +"$JAVA_HOME/bin/java" -version 2>&1 | grep -q '1\.[89]' || { + echo "JAVA_HOME must be at version 1.8 or later:" + "$JAVA_HOME/bin/java" -version + exit 2 +} +DIRNAME=$(dirname $0) +[ $(echo $DIRNAME | wc -l) -ne 1 ] && { + echo "Paths with spaces are not supported" + exit 3 +} + +CLUSTERDIR=$(cd $DIRNAME/..; echo $PWD) +INSTALLDIR=$(cd $CLUSTERDIR/../..; echo $PWD) +LOGSDIR=$CLUSTERDIR/logs + +echo "CLUSTERDIR=$CLUSTERDIR" +echo "INSTALLDIR=$INSTALLDIR" +echo "LOGSDIR=$LOGSDIR" +echo +cd $CLUSTERDIR +mkdir -p $LOGSDIR +$INSTALLDIR/bin/${HELPER_COMMAND} get_cluster_state -quiet \ + && echo "ERROR: sample cluster address (localhost:${LISTEN_PORT}) already in use" && exit 1 + +if $JAVA_HOME/bin/jps | grep ' \(CCDriver\|NCDriver\|NCService\)$' > /tmp/$$_jps; then + if [ $force ]; then + severity=WARNING + else + severity=ERROR + fi + echo -n "${severity}: ${PRODUCT} processes are already running; " + if [ $force ]; then + echo "-f[orce] specified, ignoring" + else + echo "aborting" + echo + echo "Re-run with -f to ignore, or run stop-sample-cluster.sh -f to forcibly terminate all running ${PRODUCT} processes:" + cat /tmp/$$_jps | sed 's/^/ - /' + rm /tmp/$$_jps + exit 1 + fi +fi + +rm /tmp/$$_jps +( + echo "--------------------------" + date + echo "--------------------------" +) | tee -a $LOGSDIR/blue-service.log | tee -a $LOGSDIR/red-service.log >> $LOGSDIR/cc.log +echo "INFO: Starting sample cluster..." +$INSTALLDIR/bin/${NC_SERVICE_COMMAND} -logdir - -config-file $CLUSTERDIR/conf/blue.conf >> $LOGSDIR/blue-service.log 2>&1 & +$INSTALLDIR/bin/${NC_SERVICE_COMMAND} -logdir - >> $LOGSDIR/red-service.log 2>&1 & +$INSTALLDIR/bin/${CC_COMMAND} -config-file $CLUSTERDIR/conf/cc.conf >> $LOGSDIR/cc.log 2>&1 & +$INSTALLDIR/bin/${HELPER_COMMAND} wait_for_cluster -timeout 30 +exit $? http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat new file mode 100644 index 0000000..7ac9483 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat @@ -0,0 +1,135 @@ +@REM ------------------------------------------------------------ +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ------------------------------------------------------------ +@echo off +setlocal + +goto opts + +:usage +echo. +echo Usage: %~nx0 [-f[orce]] +echo. +echo -f[orce] : Forcibly terminates any running ${PRODUCT} processes (after shutting down cluster, if running) +exit /B 0 + +:kill +echo Killing %1... +TASKKILL /F /PID %1 +echo %1...killed +exit /B 0 + +:opts +if "%1" == "" goto postopts + +if "%1" == "-f" ( + set force=1 +) else if "%1" == "-force" ( + set force=1 +) else if "%1" == "-usage" ( + goto :usage +) else if "%1" == "-help" ( + goto :usage +) else if "%1" == "--help" ( + goto :usage +) else if "%1" == "--usage" ( + goto :usage +) else ( + echo ERROR: unknown argument '%1' + call :usage + exit /B 1 +) +shift +goto opts +:postopts + +if NOT DEFINED JAVA_HOME ( + echo ERROR: JAVA_HOME not defined + goto :ERROR +) + +REM ensure JAVA_HOME has no spaces nor quotes, since appassembler can't handle them +set JAVA_HOME=%JAVA_HOME:"=% +for %%I in ("%JAVA_HOME%") do ( + set JAVA_HOME=%%~sI +) + +set DIRNAME=%~dp0 + +pushd %DIRNAME%\.. +set CLUSTERDIR=%cd% +cd %CLUSTERDIR%\..\.. +set INSTALLDIR=%cd% + +set tempfile="%TEMP%\stop-sample-cluster-%random%" + +call %INSTALLDIR%\bin\${HELPER_COMMAND} get_cluster_state -quiet +if %ERRORLEVEL% EQU 1 ( + echo WARNING: sample cluster does not appear to be running + goto :post_shutdown +) +call %INSTALLDIR%\bin\${HELPER_COMMAND} shutdown_cluster_all +echo INFO: Waiting up for cluster to shutdown... + +set tries=0 +:wait_loop +set /A tries=%tries% + 1 +if "%tries%" == "60" goto :timed_out +wmic process where ^ + "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ + GET processid >%tempfile% 2>/dev/null + +set found= +for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 +if "%found%" == "1" ( + timeout /T 1 /NOBREAK >/dev/null + goto :wait_loop +) +goto :post_shutdown + +:timed_out +echo timed out! + +:post_shutdown +echo. + +wmic process where ^ + "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ + GET processid > %tempfile% 2>/dev/null + +set found= +for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 + +if "%found%" == "1" ( + if "%force%" == "1" ( + echo WARNING: ${PRODUCT} processes remain after cluster shutdown; -f[orce] specified, forcibly terminating ${PRODUCT} processes: + for /F "skip=1" %%P in ('type %tempfile%') DO call :kill %%P + ) else ( + echo WARNING: ${PRODUCT} processes remain after cluster shutdown; re-run with -f[orce] to forcibly terminate all ${PRODUCT} processes: + for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P + ) +) +del %tempfile% + +goto :END +:ERROR +popd +exit /B 1 + +:END +popd http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh new file mode 100755 index 0000000..7ee5fdf --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +function usage() { + echo + echo Usage: $(basename $0) [-f[orce]] + echo + echo " -f[orce] : Forcibly terminates any running ${PRODUCT} processes (after shutting down cluster, if running)" +} + +while [ -n "$1" ]; do + case $1 in + -f|-force) force=1;; + -help|--help|-usage|--usage) usage; exit 0;; + *) echo "ERROR: unknown argument '$1'"; usage; exit 1;; + esac + shift +done + +if [ -z "$JAVA_HOME" -a -x /usr/libexec/java_home ]; then + JAVA_HOME=$(/usr/libexec/java_home) + export JAVA_HOME +fi + +[ -z "$JAVA_HOME" ] && { + echo "JAVA_HOME not set" + exit 1 +} +"$JAVA_HOME/bin/java" -version 2>&1 | grep -q '1\.[89]' || { + echo "JAVA_HOME must be at version 1.8 or later:" + "$JAVA_HOME/bin/java" -version + exit 2 +} +DIRNAME=$(dirname $0) +[ $(echo $DIRNAME | wc -l) -ne 1 ] && { + echo "Paths with spaces are not supported" + exit 3 +} + +CLUSTERDIR=$(cd $DIRNAME/..; echo $PWD) +INSTALLDIR=$(cd $CLUSTERDIR/../..; echo $PWD) +$INSTALLDIR/bin/${HELPER_COMMAND} get_cluster_state -quiet +if [ $? -ne 1 ]; then + $INSTALLDIR/bin/${HELPER_COMMAND} shutdown_cluster_all + first=1 + tries=0 + echo -n "INFO: Waiting up to 60s for cluster to shutdown" + while [ -n "$($JAVA_HOME/bin/jps | awk '/ (CCDriver|NCDriver|NCService)$/')" ]; do + if [ $tries -ge 60 ]; then + echo "...timed out!" + break + fi + sleep 1s + echo -n . + tries=$(expr $tries + 1) + done + echo ".done." || true +else + echo "WARNING: sample cluster does not appear to be running" +fi + +if $JAVA_HOME/bin/jps | grep ' \(CCDriver\|NCDriver\|NCService\)$' > /tmp/$$_jps; then + echo -n "WARNING: ${PRODUCT} processes remain after cluster shutdown; " + if [ $force ]; then + echo "-f[orce] specified, forcibly terminating ${PRODUCT} processes:" + cat /tmp/$$_jps | while read line; do + echo -n " - $line..." + echo $line | awk '{ print $1 }' | xargs -n1 kill -9 + echo "killed" + done + else + echo "re-run with -f|-force to forcibly terminate all ${PRODUCT} processes:" + cat /tmp/$$_jps | sed 's/^/ - /' + fi +fi +rm /tmp/$$_jps http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/conf/blue.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/blue.conf b/asterixdb/asterix-server/src/main/opt/local/conf/blue.conf new file mode 100644 index 0000000..1cb265e --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/conf/blue.conf @@ -0,0 +1,19 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[ncservice] +port=9091 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf new file mode 100644 index 0000000..71b7514 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf @@ -0,0 +1,38 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[nc/red] +txnlogdir=data/red/txnlog +coredumpdir=data/red/coredump +iodevices=data/red + +[nc/blue] +port=9091 +txnlogdir=data/blue/txnlog +coredumpdir=data/blue/coredump +iodevices=data/blue + +[nc] +storagedir=storage +address=127.0.0.1 +command=${NC_COMMAND} + +[cc] +cluster.address = 127.0.0.1 + +[app] +log.level=INFO http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat b/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat deleted file mode 100644 index 580e1e0..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat +++ /dev/null @@ -1,150 +0,0 @@ -@REM ------------------------------------------------------------ -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ------------------------------------------------------------ - -@echo off -setlocal -goto opts - -:usage -echo. -echo Usage: %~nx0 [-f[orce]] -echo. -echo -f[orce] : Forces a start attempt when ${PRODUCT} processes are found to be running -exit /B 0 - -:opts -if "%1" == "" goto postopts - -if "%1" == "-f" ( - set force=1 -) else if "%1" == "-force" ( - set force=1 -) else if "%1" == "-usage" ( - goto :usage -) else if "%1" == "-help" ( - goto :usage -) else if "%1" == "--help" ( - goto :usage -) else if "%1" == "--usage" ( - goto :usage -) else ( - echo ERROR: unknown argument '%1' - call :usage - exit /B 1 -) -shift -goto opts -:postopts - -if NOT DEFINED JAVA_HOME ( - echo ERROR: JAVA_HOME not defined - goto :ERROR -) -REM ensure JAVA_HOME has no spaces nor quotes, since appassembler can't handle them -set JAVA_HOME=%JAVA_HOME:"=% -for %%I in ("%JAVA_HOME%") do ( - set JAVA_HOME=%%~sI -) - -set JAVACMD=%JAVA_HOME%\bin\java - -REM TODO(mblow): check java version, spaces in CWD - -set DIRNAME=%~dp0 - -pushd %DIRNAME%\.. -set CLUSTERDIR=%cd% -cd %CLUSTERDIR%\..\.. -set INSTALLDIR=%cd% -set LOGSDIR=%CLUSTERDIR%\logs - -echo CLUSTERDIR=%CLUSTERDIR% -echo INSTALLDIR=%INSTALLDIR% -echo LOGSDIR=%LOGSDIR% -echo. -cd %CLUSTERDIR% -if NOT EXIST %LOGSDIR% ( - mkdir %LOGSDIR% -) -call %INSTALLDIR%\bin\${HELPER_COMMAND} get_cluster_state -quiet - -IF %ERRORLEVEL% EQU 0 ( - echo ERROR: sample cluster address [localhost:${LISTEN_PORT}] already in use - goto :ERROR -) -set tempfile="%TEMP%\start-sample-cluster-%random%" - -wmic process where ^ - "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ - GET processid > %tempfile% 2>/dev/null - -set severity=ERROR -if "%force%" == "1" set severity=WARNING - -for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 - -if "%found%" == "1" ( - if "%force%" == "1" ( - echo %severity%: ${PRODUCT} processes are already running; -f[orce] specified, ignoring - del %tempfile% - ) else ( - echo %severity%: ${PRODUCT} processes are already running; aborting" - echo. - echo Re-run with -f to ignore, or run stop-sample-cluster.bat -f to forcibly terminate all running ${PRODUCT} processes: - for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P - del %tempfile% - exit /B 1 - ) -) - -goto :post_timestamp - -:timestamp -if "%1" == "" exit /B 0 -echo "--------------------------" >> %1 -echo "%date% %time%" >> %1 -echo "--------------------------" >> %1 -shift -goto :timestamp - -:post_timestamp -echo Starting sample cluster... - -call :timestamp %LOGSDIR%\blue-service.log %LOGSDIR%\red-service.log %LOGSDIR%\cc.log - -start /MIN "blue-nc" cmd /c "echo See output in %LOGSDIR%\blue-service.log && %INSTALLDIR%\bin\${NC_SERVICE_COMMAND} -logdir - -config-file %CLUSTERDIR%\conf\blue.conf >> %LOGSDIR%\blue-service.log 2>&1" -start /MIN "red-nc" cmd /c "echo See output in %LOGSDIR%\red-service.log && %INSTALLDIR%\bin\${NC_SERVICE_COMMAND} -logdir - >> %LOGSDIR%\red-service.log 2>&1" -start /MIN "cc" cmd /c "echo See output in %LOGSDIR%\cc.log && %INSTALLDIR%\bin\${CC_COMMAND} -config-file %CLUSTERDIR%\conf\cc.conf >>%LOGSDIR%\cc.log 2>&1" - -echo. -call %INSTALLDIR%\bin\${HELPER_COMMAND} wait_for_cluster -timeout 30 -if %ERRORLEVEL% EQU 0 ( - goto :END -) - -:ERROR -echo. -popd -endlocal -exit /B 1 - -:END -echo. -popd -endlocal http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh deleted file mode 100755 index d702e8a..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -function usage() { - echo - echo Usage: $(basename $0) [-f[orce]] - echo - echo " -f[orce] : Forces a start attempt when ${PRODUCT} processes are found to be running" -} - -while [ -n "$1" ]; do - case $1 in - -f|-force) force=1;; - -help|--help|-usage|--usage) usage; exit 0;; - *) echo "ERROR: unknown argument '$1'"; usage; exit 1;; - esac - shift -done - -if [ -z "$JAVA_HOME" -a -x /usr/libexec/java_home ]; then - JAVA_HOME=$(/usr/libexec/java_home) - export JAVA_HOME -fi - -[ -z "$JAVA_HOME" ] && { - echo "JAVA_HOME not set" - exit 1 -} -"$JAVA_HOME/bin/java" -version 2>&1 | grep -q '1\.[89]' || { - echo "JAVA_HOME must be at version 1.8 or later:" - "$JAVA_HOME/bin/java" -version - exit 2 -} -DIRNAME=$(dirname $0) -[ $(echo $DIRNAME | wc -l) -ne 1 ] && { - echo "Paths with spaces are not supported" - exit 3 -} - -CLUSTERDIR=$(cd $DIRNAME/..; echo $PWD) -INSTALLDIR=$(cd $CLUSTERDIR/../..; echo $PWD) -LOGSDIR=$CLUSTERDIR/logs - -echo "CLUSTERDIR=$CLUSTERDIR" -echo "INSTALLDIR=$INSTALLDIR" -echo "LOGSDIR=$LOGSDIR" -echo -cd $CLUSTERDIR -mkdir -p $LOGSDIR -$INSTALLDIR/bin/${HELPER_COMMAND} get_cluster_state -quiet \ - && echo "ERROR: sample cluster address (localhost:${LISTEN_PORT}) already in use" && exit 1 - -if $JAVA_HOME/bin/jps | grep ' \(CCDriver\|NCDriver\|NCService\)$' > /tmp/$$_jps; then - if [ $force ]; then - severity=WARNING - else - severity=ERROR - fi - echo -n "${severity}: ${PRODUCT} processes are already running; " - if [ $force ]; then - echo "-f[orce] specified, ignoring" - else - echo "aborting" - echo - echo "Re-run with -f to ignore, or run stop-sample-cluster.sh -f to forcibly terminate all running ${PRODUCT} processes:" - cat /tmp/$$_jps | sed 's/^/ - /' - rm /tmp/$$_jps - exit 1 - fi -fi - -rm /tmp/$$_jps -( - echo "--------------------------" - date - echo "--------------------------" -) | tee -a $LOGSDIR/blue-service.log | tee -a $LOGSDIR/red-service.log >> $LOGSDIR/cc.log -echo "INFO: Starting sample cluster..." -$INSTALLDIR/bin/${NC_SERVICE_COMMAND} -logdir - -config-file $CLUSTERDIR/conf/blue.conf >> $LOGSDIR/blue-service.log 2>&1 & -$INSTALLDIR/bin/${NC_SERVICE_COMMAND} -logdir - >> $LOGSDIR/red-service.log 2>&1 & -$INSTALLDIR/bin/${CC_COMMAND} -config-file $CLUSTERDIR/conf/cc.conf >> $LOGSDIR/cc.log 2>&1 & -$INSTALLDIR/bin/${HELPER_COMMAND} wait_for_cluster -timeout 30 -exit $? http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat b/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat deleted file mode 100644 index 7ac9483..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat +++ /dev/null @@ -1,135 +0,0 @@ -@REM ------------------------------------------------------------ -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ------------------------------------------------------------ -@echo off -setlocal - -goto opts - -:usage -echo. -echo Usage: %~nx0 [-f[orce]] -echo. -echo -f[orce] : Forcibly terminates any running ${PRODUCT} processes (after shutting down cluster, if running) -exit /B 0 - -:kill -echo Killing %1... -TASKKILL /F /PID %1 -echo %1...killed -exit /B 0 - -:opts -if "%1" == "" goto postopts - -if "%1" == "-f" ( - set force=1 -) else if "%1" == "-force" ( - set force=1 -) else if "%1" == "-usage" ( - goto :usage -) else if "%1" == "-help" ( - goto :usage -) else if "%1" == "--help" ( - goto :usage -) else if "%1" == "--usage" ( - goto :usage -) else ( - echo ERROR: unknown argument '%1' - call :usage - exit /B 1 -) -shift -goto opts -:postopts - -if NOT DEFINED JAVA_HOME ( - echo ERROR: JAVA_HOME not defined - goto :ERROR -) - -REM ensure JAVA_HOME has no spaces nor quotes, since appassembler can't handle them -set JAVA_HOME=%JAVA_HOME:"=% -for %%I in ("%JAVA_HOME%") do ( - set JAVA_HOME=%%~sI -) - -set DIRNAME=%~dp0 - -pushd %DIRNAME%\.. -set CLUSTERDIR=%cd% -cd %CLUSTERDIR%\..\.. -set INSTALLDIR=%cd% - -set tempfile="%TEMP%\stop-sample-cluster-%random%" - -call %INSTALLDIR%\bin\${HELPER_COMMAND} get_cluster_state -quiet -if %ERRORLEVEL% EQU 1 ( - echo WARNING: sample cluster does not appear to be running - goto :post_shutdown -) -call %INSTALLDIR%\bin\${HELPER_COMMAND} shutdown_cluster_all -echo INFO: Waiting up for cluster to shutdown... - -set tries=0 -:wait_loop -set /A tries=%tries% + 1 -if "%tries%" == "60" goto :timed_out -wmic process where ^ - "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ - GET processid >%tempfile% 2>/dev/null - -set found= -for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 -if "%found%" == "1" ( - timeout /T 1 /NOBREAK >/dev/null - goto :wait_loop -) -goto :post_shutdown - -:timed_out -echo timed out! - -:post_shutdown -echo. - -wmic process where ^ - "name='java.exe' and CommandLine like '%%org.codehaus.mojo.appassembler.booter.AppassemblerBooter%%' and (CommandLine like '%%app.name=\"%%[cn]c\"%%' or CommandLine like '%%app.name=\"%%ncservice\"%%')" ^ - GET processid > %tempfile% 2>/dev/null - -set found= -for /F "skip=1" %%P in ('type %tempfile%') DO set found=1 - -if "%found%" == "1" ( - if "%force%" == "1" ( - echo WARNING: ${PRODUCT} processes remain after cluster shutdown; -f[orce] specified, forcibly terminating ${PRODUCT} processes: - for /F "skip=1" %%P in ('type %tempfile%') DO call :kill %%P - ) else ( - echo WARNING: ${PRODUCT} processes remain after cluster shutdown; re-run with -f[orce] to forcibly terminate all ${PRODUCT} processes: - for /F "skip=1" %%P in ('type %tempfile%') DO @echo - %%P - ) -) -del %tempfile% - -goto :END -:ERROR -popd -exit /B 1 - -:END -popd http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh deleted file mode 100755 index 7ee5fdf..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# ------------------------------------------------------------ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ------------------------------------------------------------ - -function usage() { - echo - echo Usage: $(basename $0) [-f[orce]] - echo - echo " -f[orce] : Forcibly terminates any running ${PRODUCT} processes (after shutting down cluster, if running)" -} - -while [ -n "$1" ]; do - case $1 in - -f|-force) force=1;; - -help|--help|-usage|--usage) usage; exit 0;; - *) echo "ERROR: unknown argument '$1'"; usage; exit 1;; - esac - shift -done - -if [ -z "$JAVA_HOME" -a -x /usr/libexec/java_home ]; then - JAVA_HOME=$(/usr/libexec/java_home) - export JAVA_HOME -fi - -[ -z "$JAVA_HOME" ] && { - echo "JAVA_HOME not set" - exit 1 -} -"$JAVA_HOME/bin/java" -version 2>&1 | grep -q '1\.[89]' || { - echo "JAVA_HOME must be at version 1.8 or later:" - "$JAVA_HOME/bin/java" -version - exit 2 -} -DIRNAME=$(dirname $0) -[ $(echo $DIRNAME | wc -l) -ne 1 ] && { - echo "Paths with spaces are not supported" - exit 3 -} - -CLUSTERDIR=$(cd $DIRNAME/..; echo $PWD) -INSTALLDIR=$(cd $CLUSTERDIR/../..; echo $PWD) -$INSTALLDIR/bin/${HELPER_COMMAND} get_cluster_state -quiet -if [ $? -ne 1 ]; then - $INSTALLDIR/bin/${HELPER_COMMAND} shutdown_cluster_all - first=1 - tries=0 - echo -n "INFO: Waiting up to 60s for cluster to shutdown" - while [ -n "$($JAVA_HOME/bin/jps | awk '/ (CCDriver|NCDriver|NCService)$/')" ]; do - if [ $tries -ge 60 ]; then - echo "...timed out!" - break - fi - sleep 1s - echo -n . - tries=$(expr $tries + 1) - done - echo ".done." || true -else - echo "WARNING: sample cluster does not appear to be running" -fi - -if $JAVA_HOME/bin/jps | grep ' \(CCDriver\|NCDriver\|NCService\)$' > /tmp/$$_jps; then - echo -n "WARNING: ${PRODUCT} processes remain after cluster shutdown; " - if [ $force ]; then - echo "-f[orce] specified, forcibly terminating ${PRODUCT} processes:" - cat /tmp/$$_jps | while read line; do - echo -n " - $line..." - echo $line | awk '{ print $1 }' | xargs -n1 kill -9 - echo "killed" - done - else - echo "re-run with -f|-force to forcibly terminate all ${PRODUCT} processes:" - cat /tmp/$$_jps | sed 's/^/ - /' - fi -fi -rm /tmp/$$_jps http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/conf/blue.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/conf/blue.conf b/asterixdb/asterix-server/src/main/samples/local/conf/blue.conf deleted file mode 100644 index 1cb265e..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/conf/blue.conf +++ /dev/null @@ -1,19 +0,0 @@ -; Licensed to the Apache Software Foundation (ASF) under one -; or more contributor license agreements. See the NOTICE file -; distributed with this work for additional information -; regarding copyright ownership. The ASF licenses this file -; to you under the Apache License, Version 2.0 (the -; "License"); you may not use this file except in compliance -; with the License. You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, -; software distributed under the License is distributed on an -; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -; KIND, either express or implied. See the License for the -; specific language governing permissions and limitations -; under the License. - -[ncservice] -port=9091 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/main/samples/local/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/samples/local/conf/cc.conf b/asterixdb/asterix-server/src/main/samples/local/conf/cc.conf deleted file mode 100644 index 71b7514..0000000 --- a/asterixdb/asterix-server/src/main/samples/local/conf/cc.conf +++ /dev/null @@ -1,38 +0,0 @@ -; Licensed to the Apache Software Foundation (ASF) under one -; or more contributor license agreements. See the NOTICE file -; distributed with this work for additional information -; regarding copyright ownership. The ASF licenses this file -; to you under the Apache License, Version 2.0 (the -; "License"); you may not use this file except in compliance -; with the License. You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, -; software distributed under the License is distributed on an -; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -; KIND, either express or implied. See the License for the -; specific language governing permissions and limitations -; under the License. - -[nc/red] -txnlogdir=data/red/txnlog -coredumpdir=data/red/coredump -iodevices=data/red - -[nc/blue] -port=9091 -txnlogdir=data/blue/txnlog -coredumpdir=data/blue/coredump -iodevices=data/blue - -[nc] -storagedir=storage -address=127.0.0.1 -command=${NC_COMMAND} - -[cc] -cluster.address = 127.0.0.1 - -[app] -log.level=INFO http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7648b480/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java index 0991d4f..c926ec0 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java @@ -54,7 +54,7 @@ public class SampleLocalClusterIT { // src/test/resources/NCServiceExecutionIT/cc.conf. private static final String OUTPUT_DIR = joinPath(TARGET_DIR, "sample-local-cluster"); - private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "samples", "local"); + private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "opt", "local"); @Rule public TestRule watcher = new TestMethodTracer();
