IGNITE-3293 AWS bootstrap scripts patch for Ignite-Cassandra - Fixes #794. Signed-off-by: AKuznetsov <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ee8ac73f Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ee8ac73f Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ee8ac73f Branch: refs/heads/master Commit: ee8ac73f2e048d3bc15525378630c27738a2981a Parents: 59fb5ef Author: Igor <[email protected]> Authored: Fri Aug 5 11:33:58 2016 +0700 Committer: AKuznetsov <[email protected]> Committed: Fri Aug 5 11:33:58 2016 +0700 ---------------------------------------------------------------------- modules/cassandra/pom.xml | 24 +- .../cassandra/src/test/bootstrap/aws/README.txt | 13 + .../aws/cassandra/cassandra-bootstrap.sh | 449 +++--- .../bootstrap/aws/cassandra/cassandra-env.sh | 4 + .../bootstrap/aws/cassandra/cassandra-start.sh | 483 +----- .../aws/cassandra/cassandra-template.yaml | 9 +- .../cassandra/src/test/bootstrap/aws/common.sh | 1481 ++++++++++++++++++ modules/cassandra/src/test/bootstrap/aws/env.sh | 113 ++ .../test/bootstrap/aws/ganglia/agent-start.sh | 75 + .../bootstrap/aws/ganglia/ganglia-bootstrap.sh | 417 +++++ .../bootstrap/aws/ignite/ignite-bootstrap.sh | 456 +++--- .../ignite/ignite-cassandra-server-template.xml | 27 +- .../src/test/bootstrap/aws/ignite/ignite-env.sh | 6 +- .../test/bootstrap/aws/ignite/ignite-start.sh | 537 +------ .../src/test/bootstrap/aws/logs-collector.sh | 179 ++- .../tests/ignite-cassandra-client-template.xml | 21 +- .../test/bootstrap/aws/tests/tests-bootstrap.sh | 448 +++--- .../test/bootstrap/aws/tests/tests-manager.sh | 458 ++++++ .../test/bootstrap/aws/tests/tests-report.sh | 189 +-- .../src/test/bootstrap/aws/tests/tests-run.sh | 715 --------- .../LoadTestsCassandraArtifactsCreator.java | 104 ++ .../tests/cassandra/connection-settings.xml | 6 +- .../tests/persistence/blob/ignite-config.xml | 6 - .../tests/persistence/pojo/ignite-config.xml | 6 - .../persistence/primitive/ignite-config.xml | 6 - .../primitive/ignite-remote-client-config.xml | 6 +- .../primitive/ignite-remote-server-config.xml | 6 +- .../scripts/recreate-cassandra-artifacts.bat | 39 + .../scripts/recreate-cassandra-artifacts.sh | 39 + 29 files changed, 3756 insertions(+), 2566 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/pom.xml ---------------------------------------------------------------------- diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml index dc8b08c..9fc880b 100644 --- a/modules/cassandra/pom.xml +++ b/modules/cassandra/pom.xml @@ -43,11 +43,7 @@ <minlog.version>1.3.0</minlog.version> <asm.version>5.0.3</asm.version> <objenesis.version>2.1</objenesis.version> - <netty-handler.version>4.0.27.Final</netty-handler.version> - <netty-buffer.version>4.0.27.Final</netty-buffer.version> - <netty-common.version>4.0.27.Final</netty-common.version> - <netty-transport.version>4.0.27.Final</netty-transport.version> - <netty-codec.version>4.0.27.Final</netty-codec.version> + <netty.version>4.0.33.Final</netty.version> <guava.version>19.0</guava.version> <metrics-core.version>3.0.2</metrics-core.version> </properties> @@ -121,31 +117,31 @@ <dependency> <groupId>io.netty</groupId> <artifactId>netty-handler</artifactId> - <version>${netty-handler.version}</version> + <version>${netty.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-buffer</artifactId> - <version>${netty-buffer.version}</version> + <version>${netty.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-common</artifactId> - <version>${netty-common.version}</version> + <version>${netty.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-transport</artifactId> - <version>${netty-transport.version}</version> + <version>${netty.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-codec</artifactId> - <version>${netty-codec.version}</version> + <version>${netty.version}</version> </dependency> <dependency> @@ -217,6 +213,14 @@ <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> + <excludeArtifactIds> + netty-all,cassandra-all,snappy-java,lz4,compress-lzf,commons-codec,commons-lang3,commons-math3, + concurrentlinkedhashmap-lru,antlr,ST4,antlr-runtime,jcl-over-slf4j,jackson-core-asl, + jackson-mapper-asl,json-simple,high-scale-lib,snakeyaml,jbcrypt,reporter-config3, + reporter-config-base,hibernate-validator,validation-api,jboss-logging,thrift-server, + disruptor,stream,fastutil,logback-core,logback-classic,libthrift,httpclient,httpcore, + cassandra-thrift,jna,jamm,joda-time,sigar,ecj,tools + </excludeArtifactIds> </configuration> </execution> <!-- --> http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/README.txt ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/README.txt b/modules/cassandra/src/test/bootstrap/aws/README.txt new file mode 100644 index 0000000..4457d81 --- /dev/null +++ b/modules/cassandra/src/test/bootstrap/aws/README.txt @@ -0,0 +1,13 @@ +Shell scripts to spin up Ignite, Cassandra and Load tests clusters in AWS. + +1) cassandra - bootstrap scripts for Cassandra cluster nodes +2) ganglia - bootstrap scripts for Ganglia master and agents +3) ignite - bootstrap scripts for Ignite cluster nodes +4) tests - bootstrap scripts for Load Tests cluster nodes +5) common.sh - definitions for common functions +6) env.sh - definitions for common variables +7) log-collector.sh - log collector daemon script, to collect logs and upload them to S3 + +For more details please look at the documentation: + + https://apacheignite.readme.io/docs/aws-infrastructure-deployment \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-bootstrap.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-bootstrap.sh b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-bootstrap.sh index 298c1b4..017b1b1 100644 --- a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-bootstrap.sh +++ b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,41 +17,31 @@ # limitations under the License. # -AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip - -S3_ROOT=s3://bucket/folder -S3_DOWNLOADS=$S3_ROOT/test -S3_SYSTEM=$S3_ROOT/test1 - -CASSANDRA_DOWNLOAD_URL=http://www-eu.apache.org/dist/cassandra/3.5/apache-cassandra-3.5-bin.tar.gz -CASSANDRA_TARBALL=apache-cassandra-3.5-bin.tar.gz -CASSANDRA_UNTAR_DIR=apache-cassandra-3.5 +# ----------------------------------------------------------------------------------------------- +# Bootstrap script to spin up Cassandra cluster +# ----------------------------------------------------------------------------------------------- -TESTS_PACKAGE_DONLOAD_URL=$S3_DOWNLOADS/ignite-cassandra-tests-1.6.0-SNAPSHOT.zip -TESTS_PACKAGE_ZIP=ignite-cassandra-tests-1.6.0-SNAPSHOT.zip -TESTS_PACKAGE_UNZIP_DIR=ignite-cassandra-tests +# URL to download AWS CLI tools +AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -S3_LOGS_URL=$S3_SYSTEM/logs/c-logs -S3_LOGS_TRIGGER_URL=$S3_SYSTEM/logs-trigger -S3_BOOTSTRAP_SUCCESS_URL=$S3_SYSTEM/c-success -S3_BOOTSTRAP_FAILURE_URL=$S3_SYSTEM/c-failure -S3_CASSANDRA_NODES_DISCOVERY_URL=$S3_SYSTEM/c-discovery -S3_CASSANDRA_FIRST_NODE_LOCK_URL=$S3_SYSTEM/c-first-node-lock -S3_CASSANDRA_NODES_JOIN_LOCK_URL=$S3_SYSTEM/c-join-lock +# URL to download JDK +JDK_DOWNLOAD_URL=http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz -INSTANCE_REGION=us-west-2 -INSTANCE_NAME_TAG=CASSANDRA-SERVER [email protected] -INSTANCE_PROJECT_TAG=ignite +# URL to download Ignite-Cassandra tests package - you should previously package and upload it to this place +TESTS_PACKAGE_DONLOAD_URL=s3://<bucket>/<folder>/ignite-cassandra-tests-<version>.zip +# Terminates script execution and upload logs to S3 terminate() { - if [[ "$S3_BOOTSTRAP_SUCCESS_URL" != */ ]]; then - S3_BOOTSTRAP_SUCCESS_URL=${S3_BOOTSTRAP_SUCCESS_URL}/ + SUCCESS_URL=$S3_CASSANDRA_BOOTSTRAP_SUCCESS + FAILURE_URL=$S3_CASSANDRA_BOOTSTRAP_FAILURE + + if [ -n "$SUCCESS_URL" ] && [[ "$SUCCESS_URL" != */ ]]; then + SUCCESS_URL=${SUCCESS_URL}/ fi - if [[ "$S3_BOOTSTRAP_FAILURE_URL" != */ ]]; then - S3_BOOTSTRAP_FAILURE_URL=${S3_BOOTSTRAP_FAILURE_URL}/ + if [ -n "$FAILURE_URL" ] && [[ "$FAILURE_URL" != */ ]]; then + FAILURE_URL=${FAILURE_URL}/ fi host_name=$(hostname -f | tr '[:upper:]' '[:lower:]') @@ -63,13 +53,23 @@ terminate() echo "[ERROR] Cassandra node bootstrap failed" echo "[ERROR]-----------------------------------------------------" msg=$1 - reportFolder=${S3_BOOTSTRAP_FAILURE_URL}${host_name} + + if [ -z "$FAILURE_URL" ]; then + exit 1 + fi + + reportFolder=${FAILURE_URL}${host_name} reportFile=$reportFolder/__error__ else echo "[INFO]-----------------------------------------------------" echo "[INFO] Cassandra node bootstrap successfully completed" echo "[INFO]-----------------------------------------------------" - reportFolder=${S3_BOOTSTRAP_SUCCESS_URL}${host_name} + + if [ -z "$SUCCESS_URL" ]; then + exit 0 + fi + + reportFolder=${SUCCESS_URL}${host_name} reportFile=$reportFolder/__success__ fi @@ -77,7 +77,7 @@ terminate() aws s3 rm --recursive $reportFolder if [ $? -ne 0 ]; then - echo "[ERROR] Failed drop report folder: $reportFolder" + echo "[ERROR] Failed to drop report folder: $reportFolder" fi aws s3 cp --sse AES256 /opt/bootstrap-result $reportFile @@ -94,165 +94,67 @@ terminate() exit 0 } -tagInstance() -{ - export EC2_HOME=/opt/aws/apitools/ec2 - export JAVA_HOME=/opt/jdk1.8.0_77 - export PATH=$JAVA_HOME/bin:$EC2_HOME/bin:$PATH - - INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) - if [ $? -ne 0 ]; then - terminate "Failed to get instance metadata to tag it" - fi - - if [ -n "$INSTANCE_NAME_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag Name=${INSTANCE_NAME_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: Name=${INSTANCE_NAME_TAG}" - fi - fi - - if [ -n "$INSTANCE_OWNER_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag owner=${INSTANCE_OWNER_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: owner=${INSTANCE_OWNER_TAG}" - fi - fi - - if [ -n "$INSTANCE_PROJECT_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag project=${INSTANCE_PROJECT_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: project=${INSTANCE_PROJECT_TAG}" - fi - fi -} - +# Downloads specified package downloadPackage() { echo "[INFO] Downloading $3 package from $1 into $2" - if [[ "$1" == s3* ]]; then - aws s3 cp $1 $2 - - if [ $? -ne 0 ]; then - echo "[WARN] Failed to download $3 package from first attempt" - rm -Rf $2 - sleep 10s - - echo "[INFO] Trying second attempt to download $3 package" + for i in 0 9; + do + if [[ "$1" == s3* ]]; then aws s3 cp $1 $2 + code=$? + else + curl "$1" -o "$2" + code=$? + fi - if [ $? -ne 0 ]; then - echo "[WARN] Failed to download $3 package from second attempt" - rm -Rf $2 - sleep 10s - - echo "[INFO] Trying third attempt to download $3 package" - aws s3 cp $1 $2 - - if [ $? -ne 0 ]; then - terminate "All three attempts to download $3 package from $1 are failed" - fi - fi + if [ $code -eq 0 ]; then + echo "[INFO] $3 package successfully downloaded from $1 into $2" + return 0 fi - else - curl "$1" -o "$2" - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - echo "[WARN] Failed to download $3 package from first attempt" - rm -Rf $2 - sleep 10s + echo "[WARN] Failed to download $3 package from $i attempt, sleeping extra 5sec" + sleep 5s + done - echo "[INFO] Trying second attempt to download $3 package" - curl "$1" -o "$2" + terminate "All 10 attempts to download $3 package from $1 are failed" +} - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - echo "[WARN] Failed to download $3 package from second attempt" - rm -Rf $2 - sleep 10s +# Downloads and setup JDK +setupJava() +{ + rm -Rf /opt/java /opt/jdk.tar.gz - echo "[INFO] Trying third attempt to download $3 package" - curl "$1" -o "$2" + echo "[INFO] Downloading 'jdk'" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "$JDK_DOWNLOAD_URL" -O /opt/jdk.tar.gz + if [ $? -ne 0 ]; then + terminate "Failed to download 'jdk'" + fi - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - terminate "All three attempts to download $3 package from $1 are failed" - fi - fi - fi + echo "[INFO] Untaring 'jdk'" + tar -xvzf /opt/jdk.tar.gz -C /opt + if [ $? -ne 0 ]; then + terminate "Failed to untar 'jdk'" fi - echo "[INFO] $3 package successfully downloaded from $1 into $2" -} + rm -Rf /opt/jdk.tar.gz -if [[ "$S3_CASSANDRA_NODES_DISCOVERY_URL" != */ ]]; then - S3_CASSANDRA_NODES_DISCOVERY_URL=${S3_CASSANDRA_NODES_DISCOVERY_URL}/ -fi + unzipDir=$(ls /opt | grep "jdk") + if [ "$unzipDir" != "java" ]; then + mv /opt/$unzipDir /opt/java + fi +} -echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Bootstrapping Cassandra node" -echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Cassandra download URL: $CASSANDRA_DOWNLOAD_URL" -echo "[INFO] Tests package download URL: $TESTS_PACKAGE_DONLOAD_URL" -echo "[INFO] Logs URL: $S3_LOGS_URL" -echo "[INFO] Logs trigger URL: $S3_LOGS_TRIGGER_URL" -echo "[INFO] Cassandra nodes discovery URL: $S3_CASSANDRA_NODES_DISCOVERY_URL" -echo "[INFO] Cassandra first node lock URL: $S3_CASSANDRA_FIRST_NODE_LOCK_URL" -echo "[INFO] Cassandra nodes join lock URL: $S3_CASSANDRA_NODES_JOIN_LOCK_URL" -echo "[INFO] Bootsrap success URL: $S3_BOOTSTRAP_SUCCESS_URL" -echo "[INFO] Bootsrap failure URL: $S3_BOOTSTRAP_FAILURE_URL" -echo "[INFO]-----------------------------------------------------------------" +# Downloads and setup AWS CLI +setupAWSCLI() +{ + echo "[INFO] Installing 'awscli'" + pip install --upgrade awscli + if [ $? -eq 0 ]; then + return 0 + fi -echo "[INFO] Installing 'wget' package" -yum -y install wget -if [ $? -ne 0 ]; then - terminate "Failed to install 'wget' package" -fi - -echo "[INFO] Installing 'net-tools' package" -yum -y install net-tools -if [ $? -ne 0 ]; then - terminate "Failed to install 'net-tools' package" -fi - -echo "[INFO] Installing 'python' package" -yum -y install python -if [ $? -ne 0 ]; then - terminate "Failed to install 'python' package" -fi - -echo "[INFO] Installing 'unzip' package" -yum -y install unzip -if [ $? -ne 0 ]; then - terminate "Failed to install 'unzip' package" -fi - -rm -Rf /opt/jdk1.8.0_77 /opt/jdk-8u77-linux-x64.tar.gz - -echo "[INFO] Downloading 'jdk-8u77'" -wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz"; -O /opt/jdk-8u77-linux-x64.tar.gz -if [ $? -ne 0 ]; then - terminate "Failed to download 'jdk-8u77'" -fi - -echo "[INFO] Unzipping 'jdk-8u77'" -tar -xvzf /opt/jdk-8u77-linux-x64.tar.gz -C /opt -if [ $? -ne 0 ]; then - terminate "Failed to untar 'jdk-8u77'" -fi - -rm -Rf /opt/jdk-8u77-linux-x64.tar.gz - -downloadPackage "https://bootstrap.pypa.io/get-pip.py"; "/opt/get-pip.py" "get-pip.py" - -echo "[INFO] Installing 'pip'" -python /opt/get-pip.py -if [ $? -ne 0 ]; then - terminate "Failed to install 'pip'" -fi - -echo "[INFO] Installing 'awscli'" -pip install --upgrade awscli -if [ $? -ne 0 ]; then echo "[ERROR] Failed to install 'awscli' using pip" echo "[INFO] Trying to install awscli using zip archive" echo "[INFO] Downloading awscli zip" @@ -274,100 +176,161 @@ if [ $? -ne 0 ]; then fi echo "[INFO] Successfully installed awscli from zip archive" -fi +} + +# Setup all the pre-requisites (packages, settings and etc.) +setupPreRequisites() +{ + echo "[INFO] Installing 'wget' package" + yum -y install wget + if [ $? -ne 0 ]; then + terminate "Failed to install 'wget' package" + fi + + echo "[INFO] Installing 'net-tools' package" + yum -y install net-tools + if [ $? -ne 0 ]; then + terminate "Failed to install 'net-tools' package" + fi -tagInstance + echo "[INFO] Installing 'python' package" + yum -y install python + if [ $? -ne 0 ]; then + terminate "Failed to install 'python' package" + fi -echo "[INFO] Creating 'cassandra' group" -exists=$(cat /etc/group | grep cassandra) -if [ -z "$exists" ]; then - groupadd cassandra + echo "[INFO] Installing 'unzip' package" + yum -y install unzip if [ $? -ne 0 ]; then - terminate "Failed to create 'cassandra' group" + terminate "Failed to install 'unzip' package" fi -fi -echo "[INFO] Creating 'cassandra' user" -exists=$(cat /etc/passwd | grep cassandra) -if [ -z "$exists" ]; then - useradd -g cassandra cassandra + downloadPackage "https://bootstrap.pypa.io/get-pip.py"; "/opt/get-pip.py" "get-pip.py" + + echo "[INFO] Installing 'pip'" + python /opt/get-pip.py if [ $? -ne 0 ]; then - terminate "Failed to create 'cassandra' user" + terminate "Failed to install 'pip'" fi -fi +} + +# Downloads and setup tests package +setupTestsPackage() +{ + downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/ignite-cassandra-tests.zip" "Tests" -rm -Rf /storage/cassandra /opt/cassandra /opt/$CASSANDRA_TARBALL + rm -Rf /opt/ignite-cassandra-tests -echo "[INFO] Creating '/storage/cassandra' storage" -mkdir -p /storage/cassandra -chown -R cassandra:cassandra /storage/cassandra -if [ $? -ne 0 ]; then - terminate "Failed to setup Cassandra storage dir: /storage/cassandra" -fi + unzip /opt/ignite-cassandra-tests.zip -d /opt + if [ $? -ne 0 ]; then + terminate "Failed to unzip tests package" + fi -downloadPackage "$CASSANDRA_DOWNLOAD_URL" "/opt/$CASSANDRA_TARBALL" "Cassandra" + rm -f /opt/ignite-cassandra-tests.zip -echo "[INFO] Unzipping Cassandra package" -tar -xvzf /opt/$CASSANDRA_TARBALL -C /opt -if [ $? -ne 0 ]; then - terminate "Failed to untar Cassandra package" -fi + unzipDir=$(ls /opt | grep "ignite-cassandra") + if [ "$unzipDir" != "ignite-cassandra-tests" ]; then + mv /opt/$unzipDir /opt/ignite-cassandra-tests + fi -rm -f /opt/$CASSANDRA_TARBALL /opt/cassandra -mv /opt/$CASSANDRA_UNTAR_DIR /opt/cassandra -chown -R cassandra:cassandra /opt/cassandra + find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \; -downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/$TESTS_PACKAGE_ZIP" "Tests" + . /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "cassandra" -unzip /opt/$TESTS_PACKAGE_ZIP -d /opt -if [ $? -ne 0 ]; then - terminate "Failed to unzip tests package: $TESTS_PACKAGE_DONLOAD_URL" -fi + setupNTP -chown -R cassandra:cassandra /opt/$TESTS_PACKAGE_UNZIP_DIR -find /opt/$TESTS_PACKAGE_UNZIP_DIR -type f -name "*.sh" -exec chmod ug+x {} \; + echo "[INFO] Starting logs collector daemon" -if [ ! -f "/opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-env.sh" ]; then - terminate "There are no cassandra-env.sh in tests package" -fi + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh "$S3_LOGS_TRIGGER" "$S3_CASSANDRA_LOGS/$HOST_NAME" "/opt/cassandra/logs" "/opt/cassandra/cassandra-start.log" > /opt/logs-collector.log & -if [ ! -f "/opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-start.sh" ]; then - terminate "There are no cassandra-start.sh in tests package" -fi + echo "[INFO] Logs collector daemon started: $!" -if [ ! -f "/opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-template.yaml" ]; then - terminate "There are no cassandra-start.sh in tests package" -fi + echo "----------------------------------------------------------------------------------------" + printInstanceInfo + echo "----------------------------------------------------------------------------------------" + tagInstance + bootstrapGangliaAgent "cassandra" 8641 +} -if [ ! -f "/opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/logs-collector.sh" ]; then - terminate "There are no logs-collector.sh in tests package" -fi +# Downloads Cassandra package +downloadCassandra() +{ + downloadPackage "$CASSANDRA_DOWNLOAD_URL" "/opt/apache-cassandra.tar.gz" "Cassandra" -mv -f /opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-start.sh /opt -mv -f /opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-env.sh /opt/cassandra/conf -mv -f /opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/cassandra/cassandra-template.yaml /opt/cassandra/conf -mv -f /opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/logs-collector.sh /opt -rm -Rf /opt/$TESTS_PACKAGE_UNZIP_DIR -chown -R cassandra:cassandra /opt/cassandra /opt/cassandra-start.sh /opt/logs-collector.sh + rm -Rf /opt/cassandra -#profile=/home/cassandra/.bash_profile -profile=/root/.bash_profile + echo "[INFO] Untaring Cassandra package" + tar -xvzf /opt/apache-cassandra.tar.gz -C /opt + if [ $? -ne 0 ]; then + terminate "Failed to untar Cassandra package" + fi -echo "export JAVA_HOME=/opt/jdk1.8.0_77" >> $profile -echo "export CASSANDRA_HOME=/opt/cassandra" >> $profile -echo "export PATH=\$JAVA_HOME/bin:\$CASSANDRA_HOME/bin:\$PATH" >> $profile -echo "export S3_BOOTSTRAP_SUCCESS_URL=$S3_BOOTSTRAP_SUCCESS_URL" >> $profile -echo "export S3_BOOTSTRAP_FAILURE_URL=$S3_BOOTSTRAP_FAILURE_URL" >> $profile -echo "export S3_CASSANDRA_NODES_DISCOVERY_URL=$S3_CASSANDRA_NODES_DISCOVERY_URL" >> $profile -echo "export S3_CASSANDRA_NODES_JOIN_LOCK_URL=$S3_CASSANDRA_NODES_JOIN_LOCK_URL" >> $profile -echo "export S3_CASSANDRA_FIRST_NODE_LOCK_URL=$S3_CASSANDRA_FIRST_NODE_LOCK_URL" >> $profile + rm -f /opt/apache-cassandra.tar.gz -HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + unzipDir=$(ls /opt | grep "cassandra" | grep "apache") + if [ "$unzipDir" != "cassandra" ]; then + mv /opt/$unzipDir /opt/cassandra + fi +} + +# Setups Cassandra +setupCassandra() +{ + echo "[INFO] Creating 'cassandra' group" + exists=$(cat /etc/group | grep cassandra) + if [ -z "$exists" ]; then + groupadd cassandra + if [ $? -ne 0 ]; then + terminate "Failed to create 'cassandra' group" + fi + fi + + echo "[INFO] Creating 'cassandra' user" + exists=$(cat /etc/passwd | grep cassandra) + if [ -z "$exists" ]; then + useradd -g cassandra cassandra + if [ $? -ne 0 ]; then + terminate "Failed to create 'cassandra' user" + fi + fi + + rm -f /opt/cassandra/conf/cassandra-env.sh /opt/cassandra/conf/cassandra-template.yaml + + cp /opt/ignite-cassandra-tests/bootstrap/aws/cassandra/cassandra-env.sh /opt/cassandra/conf + cp /opt/ignite-cassandra-tests/bootstrap/aws/cassandra/cassandra-template.yaml /opt/cassandra/conf + + chown -R cassandra:cassandra /opt/cassandra /opt/ignite-cassandra-tests + + createCassandraStorageLayout + + cat /opt/cassandra/conf/cassandra-template.yaml | sed -r "s/\\\$\{CASSANDRA_DATA_DIR\}/$CASSANDRA_DATA_DIR/g" > /opt/cassandra/conf/cassandra-template-1.yaml + cat /opt/cassandra/conf/cassandra-template-1.yaml | sed -r "s/\\\$\{CASSANDRA_COMMITLOG_DIR\}/$CASSANDRA_COMMITLOG_DIR/g" > /opt/cassandra/conf/cassandra-template-2.yaml + cat /opt/cassandra/conf/cassandra-template-2.yaml | sed -r "s/\\\$\{CASSANDRA_CACHES_DIR\}/$CASSANDRA_CACHES_DIR/g" > /opt/cassandra/conf/cassandra-template-3.yaml + + rm -f /opt/cassandra/conf/cassandra-template.yaml /opt/cassandra/conf/cassandra-template-1.yaml /opt/cassandra/conf/cassandra-template-2.yaml + mv /opt/cassandra/conf/cassandra-template-3.yaml /opt/cassandra/conf/cassandra-template.yaml + + echo "export JAVA_HOME=/opt/java" >> $1 + echo "export CASSANDRA_HOME=/opt/cassandra" >> $1 + echo "export PATH=\$JAVA_HOME/bin:\$CASSANDRA_HOME/bin:\$PATH" >> $1 +} + +################################################################################################################### + +echo "[INFO]-----------------------------------------------------------------" +echo "[INFO] Bootstrapping Cassandra node" +echo "[INFO]-----------------------------------------------------------------" -/opt/logs-collector.sh "/opt/cassandra/logs" "$S3_LOGS_URL/$HOST_NAME" "$S3_LOGS_TRIGGER_URL" > /opt/cassandra/logs-collector.log & +setupPreRequisites +setupJava +setupAWSCLI +setupTestsPackage +downloadCassandra +setupCassandra "/root/.bash_profile" -cmd="/opt/cassandra-start.sh" +cmd="/opt/ignite-cassandra-tests/bootstrap/aws/cassandra/cassandra-start.sh" -#sudo -u cassandra -g cassandra sh -c "$cmd | tee /opt/cassandra/start.log" +#sudo -u cassandra -g cassandra sh -c "$cmd | tee /opt/cassandra/cassandra-start.log" -$cmd | tee /opt/cassandra/start.log \ No newline at end of file +$cmd | tee /opt/cassandra/cassandra-start.log \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-env.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-env.sh b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-env.sh index 11dfc50..ba76401 100644 --- a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-env.sh +++ b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-env.sh @@ -15,6 +15,10 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Environment setup script from Cassandra distribution +# ----------------------------------------------------------------------------------------------- + calculate_heap_sizes() { case "`uname`" in http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-start.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-start.sh b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-start.sh index c73c509..4a6daef 100644 --- a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-start.sh +++ b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-start.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,377 +17,40 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Script to start Cassandra daemon (used by cassandra-bootstrap.sh) +# ----------------------------------------------------------------------------------------------- + #profile=/home/cassandra/.bash_profile profile=/root/.bash_profile . $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "cassandra" -terminate() -{ - if [[ "$S3_BOOTSTRAP_SUCCESS_URL" != */ ]]; then - S3_BOOTSTRAP_SUCCESS_URL=${S3_BOOTSTRAP_SUCCESS_URL}/ - fi - - if [[ "$S3_BOOTSTRAP_FAILURE_URL" != */ ]]; then - S3_BOOTSTRAP_FAILURE_URL=${S3_BOOTSTRAP_FAILURE_URL}/ - fi - - msg=$HOST_NAME - - if [ -n "$1" ]; then - echo "[ERROR] $1" - echo "[ERROR]-----------------------------------------------------" - echo "[ERROR] Failed to start Cassandra node" - echo "[ERROR]-----------------------------------------------------" - msg=$1 - reportFolder=${S3_BOOTSTRAP_FAILURE_URL}${HOST_NAME} - reportFile=$reportFolder/__error__ - else - echo "[INFO]-----------------------------------------------------" - echo "[INFO] Cassandra node successfully started" - echo "[INFO]-----------------------------------------------------" - reportFolder=${S3_BOOTSTRAP_SUCCESS_URL}${HOST_NAME} - reportFile=$reportFolder/__success__ - fi - - echo $msg > /opt/cassandra/start_result - - aws s3 rm --recursive $reportFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed drop report folder: $reportFolder" - fi - - if [ -d "/opt/cassandra/logs" ]; then - aws s3 sync --sse AES256 /opt/cassandra/logs $reportFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to export Cassandra logs to: $reportFolder" - fi - fi - - aws s3 cp --sse AES256 /opt/cassandra/start_result $reportFile - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to export node start result to: $reportFile" - fi - - rm -f /opt/cassandra/start_result /opt/cassandra/join-lock /opt/cassandra/remote-join-lock - - if [ -n "$1" ]; then - exit 1 - fi - - exit 0 -} - -registerNode() -{ - echo "[INFO] Registering Cassandra node seed: ${S3_CASSANDRA_NODES_DISCOVERY_URL}$HOST_NAME" - - aws s3 cp --sse AES256 /opt/cassandra/join-lock ${S3_CASSANDRA_NODES_DISCOVERY_URL}$HOST_NAME - if [ $? -ne 0 ]; then - terminate "Failed to register Cassandra seed info in: ${S3_CASSANDRA_NODES_DISCOVERY_URL}$HOST_NAME" - fi - - echo "[INFO] Cassandra node seed successfully registered" -} - -unregisterNode() -{ - echo "[INFO] Removing Cassandra node registration from: ${S3_CASSANDRA_NODES_DISCOVERY_URL}$HOST_NAME" - aws s3 rm ${S3_CASSANDRA_NODES_DISCOVERY_URL}$HOST_NAME - echo "[INFO] Cassandra node registration removed" -} - -cleanupMetadata() -{ - echo "[INFO] Running cleanup" - aws s3 rm $S3_CASSANDRA_NODES_JOIN_LOCK_URL - aws s3 rm --recursive $S3_CASSANDRA_NODES_DISCOVERY_URL - aws s3 rm --recursive $S3_BOOTSTRAP_SUCCESS_URL - aws s3 rm --recursive $S3_BOOTSTRAP_FAILURE_URL - echo "[INFO] Cleanup completed" -} - +# Setups Cassandra seeds for this EC2 node. Looks for the information in S3 about +# already up and running Cassandra cluster nodes setupCassandraSeeds() { - echo "[INFO] Setting up Cassandra seeds" + if [ "$FIRST_NODE_LOCK" == "true" ]; then + echo "[INFO] Setting up Cassandra seeds" - if [ "$FIRST_NODE" == "true" ]; then - CASSANDRA_SEEDS=$(hostname -f | tr '[:upper:]' '[:lower:]') - echo "[INFO] Using host address as a seed for the first Cassandra node: $CASSANDRA_SEEDS" - aws s3 rm --recursive ${S3_CASSANDRA_NODES_DISCOVERY_URL::-1} - if [ $? -ne 0 ]; then - terminate "Failed to clean Cassandra node discovery URL: $S3_CASSANDRA_NODES_DISCOVERY_URL" - fi + CLUSTER_SEEDS=$(hostname -f | tr '[:upper:]' '[:lower:]') - cat /opt/cassandra/conf/cassandra-template.yaml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/cassandra/conf/cassandra.yaml - - return 0 - fi - - echo "[INFO] Looking for Cassandra seeds in: $S3_CASSANDRA_NODES_DISCOVERY_URL" - - startTime=$(date +%s) - - while true; do - seeds=$(aws s3 ls $S3_CASSANDRA_NODES_DISCOVERY_URL | grep -v PRE | sed -r "s/^.* //g") - if [ -n "$seeds" ]; then - seeds=($seeds) - length=${#seeds[@]} - - if [ $length -lt 4 ]; then - seed1=${seeds[0]} - seed2=${seeds[1]} - seed3=${seeds[2]} - else - pos1=$(($RANDOM%$length)) - pos2=$(($RANDOM%$length)) - pos3=$(($RANDOM%$length)) - seed1=${seeds[${pos1}]} - seed2=${seeds[${pos2}]} - seed3=${seeds[${pos3}]} - fi - - CASSANDRA_SEEDS=$seed1 - CASSANDRA_SEED=$seed1 - - if [ "$seed2" != "$seed1" ] && [ -n "$seed2" ]; then - CASSANDRA_SEEDS="$CASSANDRA_SEEDS,$seed2" - fi - - if [ "$seed3" != "$seed2" ] && [ "$seed3" != "$seed1" ] && [ -n "$seed3" ]; then - CASSANDRA_SEEDS="$CASSANDRA_SEEDS,$seed3" - fi - - echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS" - - cat /opt/cassandra/conf/cassandra-template.yaml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/cassandra/conf/cassandra.yaml - - return 0 - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $NODE_STARTUP_TIME ]; then - terminate "${NODE_STARTUP_TIME}min timeout expired, but first Cassandra node is still not up and running" - fi - - echo "[INFO] Waiting for the first Cassandra node to start and publish its seed, time passed ${duration}min" - - sleep 1m - done -} - -tryToGetFirstNodeLock() -{ - echo "[INFO] Trying to get first node lock" - - checkFirstNodeLockExist - if [ $? -ne 0 ]; then - return 1 - fi - - createFirstNodeLock - - sleep 5s - - rm -Rf /opt/cassandra/first-node-lock - - aws s3 cp $S3_CASSANDRA_FIRST_NODE_LOCK_URL /opt/cassandra/first-node-lock - if [ $? -ne 0 ]; then - echo "[WARN] Failed to check just created first node lock" - return 1 - fi - - first_host=$(cat /opt/cassandra/first-node-lock) - - rm -f /opt/cassandra/first-node-lock - - if [ "$first_host" != "$HOST_NAME" ]; then - echo "[INFO] Node $first_host has discarded previously created first node lock" - return 1 - fi - - echo "[INFO] Congratulations, got first node lock" - - return 0 -} - -checkFirstNodeLockExist() -{ - echo "[INFO] Checking for the first node lock" - - lockExists=$(aws s3 ls $S3_CASSANDRA_FIRST_NODE_LOCK_URL) - if [ -n "$lockExists" ]; then - echo "[INFO] First node lock already exists" - return 1 - fi - - echo "[INFO] First node lock doesn't exist" - - return 0 -} - -createFirstNodeLock() -{ - aws s3 cp --sse AES256 /opt/cassandra/join-lock $S3_CASSANDRA_FIRST_NODE_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to create first node lock" - fi - echo "[INFO] Created first node lock" -} - -removeFirstNodeLock() -{ - aws s3 rm $S3_CASSANDRA_FIRST_NODE_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to remove first node lock" - fi - echo "[INFO] Removed first node lock" -} - -tryToGetClusterJoinLock() -{ - echo "[INFO] Trying to get cluster join lock" - - checkClusterJoinLockExist - if [ $? -ne 0 ]; then - return 1 - fi - - createClusterJoinLock - - sleep 5s - - rm -Rf /opt/cassandra/remote-join-lock - - aws s3 cp $S3_CASSANDRA_NODES_JOIN_LOCK_URL /opt/cassandra/remote-join-lock - if [ $? -ne 0 ]; then - echo "[WARN] Failed to check just created cluster join lock" - return 1 - fi - - join_host=$(cat /opt/cassandra/remote-join-lock) - - if [ "$join_host" != "$HOST_NAME" ]; then - echo "[INFO] Node $first_host has discarded previously created cluster join lock" - return 1 - fi - - echo "[INFO] Congratulations, got cluster join lock" - - return 0 -} - -checkClusterJoinLockExist() -{ - echo "[INFO] Checking for the cluster join lock" - - lockExists=$(aws s3 ls $S3_CASSANDRA_NODES_JOIN_LOCK_URL) - if [ -n "$lockExists" ]; then - echo "[INFO] Cluster join lock already exists" - return 1 - fi - - status=$(/opt/cassandra/bin/nodetool -h $CASSANDRA_SEED status) - leaving=$(echo $status | grep UL) - moving=$(echo $status | grep UM) - joining=$(echo $status | grep UJ) - - if [ -n "$leaving" ] || [ -n "$moving" ] || [ -n "$joining" ]; then - echo "[INFO] Cluster join lock doesn't exist in S3, but some node still trying to join Cassandra cluster" - return 1 - fi - - echo "[INFO] Cluster join lock doesn't exist" - - return 0 -} - -createClusterJoinLock() -{ - aws s3 cp --sse AES256 /opt/cassandra/join-lock $S3_CASSANDRA_NODES_JOIN_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to create cluster join lock" - fi - echo "[INFO] Created cluster join lock" -} - -removeClusterJoinLock() -{ - aws s3 rm $S3_CASSANDRA_NODES_JOIN_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to remove cluster join lock" - fi - echo "[INFO] Removed cluster join lock" -} - -waitToJoinCassandraCluster() -{ - echo "[INFO] Waiting to join Cassandra cluster" - - while true; do - tryToGetClusterJoinLock + echo "[INFO] Using host address as a seed for the first Cassandra node: $CLUSTER_SEEDS" + aws s3 rm --recursive ${S3_CASSANDRA_NODES_DISCOVERY::-1} if [ $? -ne 0 ]; then - echo "[INFO] Another node is trying to join cluster. Waiting for extra 1min." - sleep 1m - else - echo "[INFO]-------------------------------------------------------------" - echo "[INFO] Congratulations, got lock to join Cassandra cluster" - echo "[INFO]-------------------------------------------------------------" - break + terminate "Failed to clean Cassandra node discovery URL: $S3_CASSANDRA_NODES_DISCOVERY" fi - done -} - -waitFirstCassandraNodeRegistered() -{ - echo "[INFO] Waiting for the first Cassandra node to register" - - startTime=$(date +%s) - - while true; do - first_host= - - exists=$(aws s3 ls $S3_CASSANDRA_FIRST_NODE_LOCK_URL) - if [ -n "$exists" ]; then - rm -Rf /opt/cassandra/first-node-lock - - aws s3 cp $S3_CASSANDRA_FIRST_NODE_LOCK_URL /opt/cassandra/first-node-lock - if [ $? -ne 0 ]; then - terminate "Failed to check existing first node lock" - fi - - first_host=$(cat /opt/cassandra/first-node-lock) - - rm -Rf /opt/cassandra/first-node-lock - fi - - if [ -n "$first_host" ]; then - exists=$(aws s3 ls ${S3_CASSANDRA_NODES_DISCOVERY_URL}${first_host}) - if [ -n "$exists" ]; then - break - fi - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $NODE_STARTUP_TIME ]; then - terminate "${NODE_STARTUP_TIME}min timeout expired, but first Cassandra node is still not up and running" - fi - - echo "[INFO] Waiting extra 1min" - - sleep 1m - done + else + setupClusterSeeds "cassandra" "true" + CLUSTER_SEEDS=$(echo $CLUSTER_SEEDS | sed -r "s/ /,/g") + fi - echo "[INFO] First Cassandra node registered" + cat /opt/cassandra/conf/cassandra-template.yaml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CLUSTER_SEEDS/g" > /opt/cassandra/conf/cassandra.yaml } +# Gracefully starts Cassandra daemon and waits until it joins Cassandra cluster startCassandra() { echo "[INFO]-------------------------------------------------------------" @@ -397,8 +60,13 @@ startCassandra() setupCassandraSeeds - if [ "$FIRST_NODE" != "true" ]; then - waitToJoinCassandraCluster + waitToJoinCluster + + if [ "$FIRST_NODE_LOCK" == "true" ]; then + aws s3 rm --recursive ${S3_CASSANDRA_NODES_DISCOVERY::-1} + if [ $? -ne 0 ]; then + terminate "Failed to clean Cassandra node discovery URL: $S3_IGNITE_NODES_DISCOVERY" + fi fi proc=$(ps -ef | grep java | grep "org.apache.cassandra.service.CassandraDaemon") @@ -420,64 +88,40 @@ startCassandra() START_ATTEMPT=$(( $START_ATTEMPT+1 )) } -# Time (in minutes) to wait for the Cassandra node up and running and register it in S3 -NODE_STARTUP_TIME=10 - -# Number of attempts to start (not first) Cassandra daemon -NODE_START_ATTEMPTS=3 - -HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') -echo $HOST_NAME > /opt/cassandra/join-lock +####################################################################################################### START_ATTEMPT=0 -FIRST_NODE="false" - +# Cleans all the previous metadata about this EC2 node unregisterNode +# Tries to get first-node lock tryToGetFirstNodeLock -if [ $? -eq 0 ]; then - FIRST_NODE="true" -fi - echo "[INFO]-----------------------------------------------------------------" -if [ "$FIRST_NODE" == "true" ]; then +if [ "$FIRST_NODE_LOCK" == "true" ]; then echo "[INFO] Starting first Cassandra node" else echo "[INFO] Starting Cassandra node" fi echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Cassandra nodes discovery URL: $S3_CASSANDRA_NODES_DISCOVERY_URL" -echo "[INFO] Cassandra first node lock URL: $S3_CASSANDRA_FIRST_NODE_LOCK_URL" -echo "[INFO] Cassandra nodes join lock URL: $S3_CASSANDRA_NODES_JOIN_LOCK_URL" -echo "[INFO] Start success URL: $S3_BOOTSTRAP_SUCCESS_URL" -echo "[INFO] Start failure URL: $S3_BOOTSTRAP_FAILURE_URL" -echo "[INFO] CASSANDRA_HOME: $CASSANDRA_HOME" -echo "[INFO] JAVA_HOME: $JAVA_HOME" -echo "[INFO] PATH: $PATH" +printInstanceInfo echo "[INFO]-----------------------------------------------------------------" -if [ -z "$S3_CASSANDRA_NODES_DISCOVERY_URL" ]; then - terminate "S3 discovery URL doesn't specified" -fi - -if [[ "$S3_CASSANDRA_NODES_DISCOVERY_URL" != */ ]]; then - S3_CASSANDRA_NODES_DISCOVERY_URL=${S3_CASSANDRA_NODES_DISCOVERY_URL}/ -fi - -if [ "$FIRST_NODE" != "true" ]; then - waitFirstCassandraNodeRegistered +if [ "$FIRST_NODE_LOCK" != "true" ]; then + waitFirstClusterNodeRegistered "true" else cleanupMetadata fi +# Start Cassandra daemon startCassandra startTime=$(date +%s) +# Trying multiple attempts to start Cassandra daemon while true; do proc=$(ps -ef | grep java | grep "org.apache.cassandra.service.CassandraDaemon") @@ -490,9 +134,9 @@ while true; do echo $proc echo "[INFO]-----------------------------------------------------" - if [ "$FIRST_NODE" != "true" ]; then - removeClusterJoinLock - fi + # Once node joined the cluster we need to remove cluster-join lock + # to allow other EC2 nodes to acquire it and join cluster sequentially + removeClusterJoinLock break fi @@ -501,43 +145,64 @@ while true; do duration=$(( $currentTime-$startTime )) duration=$(( $duration/60 )) - if [ $duration -gt $NODE_STARTUP_TIME ]; then - if [ "$FIRST_NODE" == "true" ]; then - removeFirstNodeLock - terminate "${NODE_STARTUP_TIME}min timeout expired, but first Cassandra daemon is still not up and running" + if [ $duration -gt $SERVICE_STARTUP_TIME ]; then + if [ "$FIRST_NODE_LOCK" == "true" ]; then + # If the first node of Cassandra cluster failed to start Cassandra daemon in SERVICE_STARTUP_TIME min, + # we will not try any other attempts and just terminate with error. Terminate function itself, will + # take care about removing all the locks holding by this node. + terminate "${SERVICE_STARTUP_TIME}min timeout expired, but first Cassandra daemon is still not up and running" else + # If node isn't the first node of Cassandra cluster and it failed to start we need to + # remove cluster-join lock to allow other EC2 nodes to acquire it removeClusterJoinLock - if [ $START_ATTEMPT -gt $NODE_START_ATTEMPTS ]; then - terminate "${NODE_START_ATTEMPTS} attempts exceed, but Cassandra daemon is still not up and running" + # If node failed all SERVICE_START_ATTEMPTS attempts to start Cassandra daemon we will not + # try anymore and terminate with error + if [ $START_ATTEMPT -gt $SERVICE_START_ATTEMPTS ]; then + terminate "${SERVICE_START_ATTEMPTS} attempts exceed, but Cassandra daemon is still not up and running" fi + # New attempt to start Cassandra daemon startCassandra fi continue fi + # Checking for the situation when two nodes trying to simultaneously join Cassandra cluster. + # This actually can happen only in not standard situation, when you are trying to start + # Cassandra daemon on some EC2 nodes manually and not using bootstrap script. concurrencyError=$(cat /opt/cassandra/logs/system.log | grep "java.lang.UnsupportedOperationException: Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while cassandra.consistent.rangemovement is true") - if [ -n "$concurrencyError" ] && [ "$FIRST_NODE" != "true" ]; then + if [ -n "$concurrencyError" ] && [ "$FIRST_NODE_LOCK" != "true" ]; then + # Remove cluster-join lock to allow other EC2 nodes to acquire it removeClusterJoinLock - echo "[WARN] Failed to concurrently start Cassandra daemon. Sleeping for extra 1min" - sleep 1m + + echo "[WARN] Failed to concurrently start Cassandra daemon. Sleeping for extra 30sec" + sleep 30s + + # New attempt to start Cassandra daemon startCassandra + continue fi + # Handling situation when Cassandra daemon process abnormally terminated if [ -z "$proc" ]; then - if [ "$FIRST_NODE" == "true" ]; then - removeFirstNodeLock + # If this is the first node of Cassandra cluster just terminating with error + if [ "$FIRST_NODE_LOCK" == "true" ]; then terminate "Failed to start Cassandra daemon" fi + # Remove cluster-join lock to allow other EC2 nodes to acquire it removeClusterJoinLock - echo "[WARN] Failed to start Cassandra daemon. Sleeping for extra 1min" - sleep 1m + + echo "[WARN] Failed to start Cassandra daemon. Sleeping for extra 30sec" + sleep 30s + + # New attempt to start Cassandra daemon startCassandra + continue fi @@ -545,6 +210,8 @@ while true; do sleep 30s done +# Once Cassandra daemon successfully started we registering new Cassandra node in S3 registerNode +# Terminating script with zero exit code terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-template.yaml ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-template.yaml b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-template.yaml index 965e34e..e621886 100644 --- a/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-template.yaml +++ b/modules/cassandra/src/test/bootstrap/aws/cassandra/cassandra-template.yaml @@ -154,13 +154,12 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner # will spread data evenly across them, subject to the granularity of # the configured compaction strategy. # If not set, the default directory is $CASSANDRA_HOME/data/data. -data_file_directories: - - /storage/cassandra/data +data_file_directories: ${CASSANDRA_DATA_DIR} # commit log. when running on magnetic HDD, this should be a # separate spindle than the data directories. # If not set, the default directory is $CASSANDRA_HOME/data/commitlog. -commitlog_directory: /storage/cassandra/commitlog +commitlog_directory: ${CASSANDRA_COMMITLOG_DIR} # policy for data disk failures: # die: shut down gossip and client transports and kill the JVM for any fs errors or @@ -285,7 +284,7 @@ counter_cache_save_period: 7200 # saved caches # If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. -saved_caches_directory: /storage/cassandra/saved_caches +saved_caches_directory: ${CASSANDRA_CACHES_DIR} # commitlog_sync may be either "periodic" or "batch." # @@ -757,7 +756,7 @@ cross_node_timeout: false # # You can use a custom Snitch by setting this to the full class name # of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: GossipingPropertyFileSnitch +endpoint_snitch: Ec2Snitch # controls how often to perform the more expensive part of host score # calculation
